From 14c8627ce097af72779bc625539ca83715d3d9f4 Mon Sep 17 00:00:00 2001 From: Maxime Beauchamp <15185355+baktun14@users.noreply.github.com> Date: Sun, 20 Sep 2026 14:37:15 +0400 Subject: [PATCH 1/5] feat(deployment): record accelerator and process origin evidence in workload probes The probe collects four more sections from each service it reaches: accelerator state, large files, established sockets and process origins. It parses them and writes one row per service into a new table, with the verdict and the detection id when there is one. The sweep that already runs deletes rows past the retention window. The new sections never reach the scanner, so verdicts do not change. --- .../drizzle/0058_workload_probe_evidence.sql | 19 + apps/api/drizzle/meta/0058_snapshot.json | 2053 +++++++++++++++++ apps/api/drizzle/meta/_journal.json | 7 + .../src/workload-abuse/config/env.config.ts | 4 +- .../workload-abuse.controller.spec.ts | 23 +- .../controllers/workload-abuse.controller.ts | 6 +- .../parse-probe-evidence.spec.ts | 152 ++ .../probe-evidence/parse-probe-evidence.ts | 183 ++ .../src/workload-abuse/model-schemas/index.ts | 1 + .../workload-probe-evidence.schema.ts | 48 + .../workload-probe-evidence.repository.ts | 51 + .../probe-evidence.service.spec.ts | 117 + .../probe-evidence/probe-evidence.service.ts | 70 + ...be-trial-deployment.handler.integration.ts | 54 +- .../probe-trial-deployment.handler.spec.ts | 59 +- .../probe-trial-deployment.handler.ts | 17 + .../provider-shell-probe.service.spec.ts | 122 +- .../provider-shell-probe.service.ts | 6 +- .../trial-workload-probe.service.spec.ts | 28 +- .../trial-workload-probe.service.ts | 22 +- .../workload-abuse-instrumentation.service.ts | 8 + 21 files changed, 3031 insertions(+), 19 deletions(-) create mode 100644 apps/api/drizzle/0058_workload_probe_evidence.sql create mode 100644 apps/api/drizzle/meta/0058_snapshot.json create mode 100644 apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.spec.ts create mode 100644 apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.ts create mode 100644 apps/api/src/workload-abuse/model-schemas/workload-probe-evidence/workload-probe-evidence.schema.ts create mode 100644 apps/api/src/workload-abuse/repositories/workload-probe-evidence/workload-probe-evidence.repository.ts create mode 100644 apps/api/src/workload-abuse/services/probe-evidence/probe-evidence.service.spec.ts create mode 100644 apps/api/src/workload-abuse/services/probe-evidence/probe-evidence.service.ts diff --git a/apps/api/drizzle/0058_workload_probe_evidence.sql b/apps/api/drizzle/0058_workload_probe_evidence.sql new file mode 100644 index 0000000000..1da7cede41 --- /dev/null +++ b/apps/api/drizzle/0058_workload_probe_evidence.sql @@ -0,0 +1,19 @@ +CREATE TABLE "workload_probe_evidence" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4() NOT NULL, + "wallet_id" integer NOT NULL, + "dseq" varchar NOT NULL, + "provider" text NOT NULL, + "service" varchar(255) NOT NULL, + "probe_status" varchar(64) NOT NULL, + "verdict" varchar(16) NOT NULL, + "detection_id" uuid, + "accelerator" jsonb, + "artifacts" jsonb, + "process_origins" jsonb, + "net_shape" jsonb, + "behavioural_findings" jsonb, + "created_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE INDEX "workload_probe_evidence_wallet_dseq_created_idx" ON "workload_probe_evidence" USING btree ("wallet_id","dseq","created_at");--> statement-breakpoint +CREATE INDEX "workload_probe_evidence_created_idx" ON "workload_probe_evidence" USING btree ("created_at"); \ No newline at end of file diff --git a/apps/api/drizzle/meta/0058_snapshot.json b/apps/api/drizzle/meta/0058_snapshot.json new file mode 100644 index 0000000000..2695cc76b6 --- /dev/null +++ b/apps/api/drizzle/meta/0058_snapshot.json @@ -0,0 +1,2053 @@ +{ + "id": "63bba39e-51aa-48a3-a83b-b019a98795f2", + "prevId": "929d72bc-0bb2-40b5-ae01-1289fda18bfa", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.user_wallets": { + "name": "user_wallets", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "varchar", + "primaryKey": false, + "notNull": false + }, + "deployment_allowance": { + "name": "deployment_allowance", + "type": "numeric(20, 2)", + "primaryKey": false, + "notNull": true, + "default": "'0.00'" + }, + "fee_allowance": { + "name": "fee_allowance", + "type": "numeric(20, 2)", + "primaryKey": false, + "notNull": true, + "default": "'0.00'" + }, + "trial": { + "name": "trial", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": true + }, + "activated_at": { + "name": "activated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "credits_low_notified_at": { + "name": "credits_low_notified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "credits_sufficient_since": { + "name": "credits_sufficient_since", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "credits_low_since": { + "name": "credits_low_since", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "abuse_locked_at": { + "name": "abuse_locked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "abuse_locked_reason": { + "name": "abuse_locked_reason", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "user_wallets_user_id_userSetting_id_fk": { + "name": "user_wallets_user_id_userSetting_id_fk", + "tableFrom": "user_wallets", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_wallets_user_id_unique": { + "name": "user_wallets_user_id_unique", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + }, + "user_wallets_address_unique": { + "name": "user_wallets_address_unique", + "nullsNotDistinct": false, + "columns": [ + "address" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.payment_methods": { + "name": "payment_methods", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "fingerprint": { + "name": "fingerprint", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "payment_method_id": { + "name": "payment_method_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "is_validated": { + "name": "is_validated", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "payment_methods_fingerprint_payment_method_id_unique": { + "name": "payment_methods_fingerprint_payment_method_id_unique", + "columns": [ + { + "expression": "fingerprint", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "payment_method_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "payment_methods_user_id_is_default_unique": { + "name": "payment_methods_user_id_is_default_unique", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_default", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"payment_methods\".\"is_default\" = true", + "concurrently": false, + "method": "btree", + "with": {} + }, + "payment_methods_fingerprint_idx": { + "name": "payment_methods_fingerprint_idx", + "columns": [ + { + "expression": "fingerprint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "payment_methods_user_id_idx": { + "name": "payment_methods_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "payment_methods_user_id_is_validated_idx": { + "name": "payment_methods_user_id_is_validated_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_validated", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "payment_methods_user_id_fingerprint_payment_method_id_idx": { + "name": "payment_methods_user_id_fingerprint_payment_method_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "fingerprint", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "payment_method_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "payment_methods_user_id_userSetting_id_fk": { + "name": "payment_methods_user_id_userSetting_id_fk", + "tableFrom": "payment_methods", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.stripe_transactions": { + "name": "stripe_transactions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "stripe_transaction_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "stripe_transaction_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'created'" + }, + "amount": { + "name": "amount", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "amount_refunded": { + "name": "amount_refunded", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "bonus_amount": { + "name": "bonus_amount", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "currency": { + "name": "currency", + "type": "varchar(3)", + "primaryKey": false, + "notNull": true, + "default": "'usd'" + }, + "stripe_payment_intent_id": { + "name": "stripe_payment_intent_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "stripe_charge_id": { + "name": "stripe_charge_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "stripe_coupon_id": { + "name": "stripe_coupon_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "stripe_promotion_code_id": { + "name": "stripe_promotion_code_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "stripe_invoice_id": { + "name": "stripe_invoice_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "stripe_idempotency_key": { + "name": "stripe_idempotency_key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "payment_method_type": { + "name": "payment_method_type", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "card_brand": { + "name": "card_brand", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "card_last4": { + "name": "card_last4", + "type": "varchar(4)", + "primaryKey": false, + "notNull": false + }, + "receipt_url": { + "name": "receipt_url", + "type": "varchar(2048)", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "varchar(500)", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "varchar(1000)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "stripe_transactions_stripe_invoice_id_unique": { + "name": "stripe_transactions_stripe_invoice_id_unique", + "columns": [ + { + "expression": "stripe_invoice_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"stripe_transactions\".\"stripe_invoice_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_stripe_idempotency_key_unique": { + "name": "stripe_transactions_stripe_idempotency_key_unique", + "columns": [ + { + "expression": "stripe_idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"stripe_transactions\".\"stripe_idempotency_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_user_id_idx": { + "name": "stripe_transactions_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_stripe_payment_intent_id_idx": { + "name": "stripe_transactions_stripe_payment_intent_id_idx", + "columns": [ + { + "expression": "stripe_payment_intent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_stripe_charge_id_idx": { + "name": "stripe_transactions_stripe_charge_id_idx", + "columns": [ + { + "expression": "stripe_charge_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_stripe_coupon_id_idx": { + "name": "stripe_transactions_stripe_coupon_id_idx", + "columns": [ + { + "expression": "stripe_coupon_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_stripe_promotion_code_id_idx": { + "name": "stripe_transactions_stripe_promotion_code_id_idx", + "columns": [ + { + "expression": "stripe_promotion_code_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_status_idx": { + "name": "stripe_transactions_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_created_at_idx": { + "name": "stripe_transactions_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_user_id_created_at_idx": { + "name": "stripe_transactions_user_id_created_at_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "stripe_transactions_user_id_userSetting_id_fk": { + "name": "stripe_transactions_user_id_userSetting_id_fk", + "tableFrom": "stripe_transactions", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.wallet_settings": { + "name": "wallet_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "wallet_id": { + "name": "wallet_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "auto_reload_enabled": { + "name": "auto_reload_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "auto_reload_mode": { + "name": "auto_reload_mode", + "type": "auto_reload_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'prediction'" + }, + "auto_reload_threshold": { + "name": "auto_reload_threshold", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 2000 + }, + "auto_reload_amount": { + "name": "auto_reload_amount", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 10000 + }, + "last_auto_charge_at": { + "name": "last_auto_charge_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "auto_reload_failure_count": { + "name": "auto_reload_failure_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "auto_reload_paused_at": { + "name": "auto_reload_paused_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "wallet_settings_user_id_idx": { + "name": "wallet_settings_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "wallet_settings_wallet_id_user_wallets_id_fk": { + "name": "wallet_settings_wallet_id_user_wallets_id_fk", + "tableFrom": "wallet_settings", + "tableTo": "user_wallets", + "columnsFrom": [ + "wallet_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "wallet_settings_user_id_userSetting_id_fk": { + "name": "wallet_settings_user_id_userSetting_id_fk", + "tableFrom": "wallet_settings", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "wallet_settings_wallet_id_unique": { + "name": "wallet_settings_wallet_id_unique", + "nullsNotDistinct": false, + "columns": [ + "wallet_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.userSetting": { + "name": "userSetting", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "userId": { + "name": "userId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "username": { + "name": "username", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "emailVerified": { + "name": "emailVerified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "stripeCustomerId": { + "name": "stripeCustomerId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "bio": { + "name": "bio", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "subscribedToNewsletter": { + "name": "subscribedToNewsletter", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "youtubeUsername": { + "name": "youtubeUsername", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "twitterUsername": { + "name": "twitterUsername", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "githubUsername": { + "name": "githubUsername", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "last_active_at": { + "name": "last_active_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "last_ip": { + "name": "last_ip", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "last_user_agent": { + "name": "last_user_agent", + "type": "varchar(500)", + "primaryKey": false, + "notNull": false + }, + "last_fingerprint": { + "name": "last_fingerprint", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "onboardingSkippedAt": { + "name": "onboardingSkippedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "fair_use_policy_accepted_at": { + "name": "fair_use_policy_accepted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "userSetting_userId_unique": { + "name": "userSetting_userId_unique", + "nullsNotDistinct": false, + "columns": [ + "userId" + ] + }, + "userSetting_username_unique": { + "name": "userSetting_username_unique", + "nullsNotDistinct": false, + "columns": [ + "username" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.template": { + "name": "template", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "userId": { + "name": "userId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "copiedFromId": { + "name": "copiedFromId", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "isPublic": { + "name": "isPublic", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "cpu": { + "name": "cpu", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "ram": { + "name": "ram", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "storage": { + "name": "storage", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "sdl": { + "name": "sdl", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "template_userId_idx": { + "name": "template_userId_idx", + "columns": [ + { + "expression": "userId", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.templateFavorite": { + "name": "templateFavorite", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "userId": { + "name": "userId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "templateId": { + "name": "templateId", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "addedDate": { + "name": "addedDate", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "templateFavorite_userId_templateId_unique": { + "name": "templateFavorite_userId_templateId_unique", + "columns": [ + { + "expression": "userId", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "templateId", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "templateFavorite_templateId_template_id_fk": { + "name": "templateFavorite_templateId_template_id_fk", + "tableFrom": "templateFavorite", + "tableTo": "template", + "columnsFrom": [ + "templateId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.deployment_settings": { + "name": "deployment_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "dseq": { + "name": "dseq", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "auto_top_up_enabled": { + "name": "auto_top_up_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "closed": { + "name": "closed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "last_funded_at": { + "name": "last_funded_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "runtime_limit_hours": { + "name": "runtime_limit_hours", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "sdl": { + "name": "sdl", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sealed_secrets": { + "name": "sealed_secrets", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "manifest_version": { + "name": "manifest_version", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "runtime_ends_at": { + "name": "runtime_ends_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "runtime_ending_notified_for": { + "name": "runtime_ending_notified_for", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "provider_unreachable_notified_for": { + "name": "provider_unreachable_notified_for", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "id_auto_top_up_enabled_closed_idx": { + "name": "id_auto_top_up_enabled_closed_idx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "auto_top_up_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "closed", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_id_id_sealed_secrets_idx": { + "name": "user_id_id_sealed_secrets_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"deployment_settings\".\"sealed_secrets\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "deployment_settings_user_id_userSetting_id_fk": { + "name": "deployment_settings_user_id_userSetting_id_fk", + "tableFrom": "deployment_settings", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "dseq_user_id_idx": { + "name": "dseq_user_id_idx", + "nullsNotDistinct": false, + "columns": [ + "dseq", + "user_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.api_keys": { + "name": "api_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "hashed_key": { + "name": "hashed_key", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "key_format": { + "name": "key_format", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "api_keys_user_id_userSetting_id_fk": { + "name": "api_keys_user_id_userSetting_id_fk", + "tableFrom": "api_keys", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "api_keys_hashed_key_unique": { + "name": "api_keys_hashed_key_unique", + "nullsNotDistinct": false, + "columns": [ + "hashed_key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.email_verification_codes": { + "name": "email_verification_codes", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "code": { + "name": "code", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "email_verification_codes_user_id_idx": { + "name": "email_verification_codes_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "email_verification_codes_user_id_userSetting_id_fk": { + "name": "email_verification_codes_user_id_userSetting_id_fk", + "tableFrom": "email_verification_codes", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_keys": { + "name": "data_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "wrapped_key": { + "name": "wrapped_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "wrapped_by_kid": { + "name": "wrapped_by_kid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "retired_at": { + "name": "retired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_keys_active_user_id_idx": { + "name": "data_keys_active_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_keys\".\"retired_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_keys_wrapped_by_kid_idx": { + "name": "data_keys_wrapped_by_kid_idx", + "columns": [ + { + "expression": "wrapped_by_kid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_keys_user_id_userSetting_id_fk": { + "name": "data_keys_user_id_userSetting_id_fk", + "tableFrom": "data_keys", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.blocked_email_domains": { + "name": "blocked_email_domains", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "domain": { + "name": "domain", + "type": "varchar(253)", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "blocked_email_domain_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'blocked'" + }, + "source": { + "name": "source", + "type": "blocked_email_domain_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'manual'" + }, + "reason": { + "name": "reason", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "triggered_by_user_id": { + "name": "triggered_by_user_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "blocked_email_domains_domain_unique": { + "name": "blocked_email_domains_domain_unique", + "columns": [ + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "blocked_email_domains_triggered_by_user_id_userSetting_id_fk": { + "name": "blocked_email_domains_triggered_by_user_id_userSetting_id_fk", + "tableFrom": "blocked_email_domains", + "tableTo": "userSetting", + "columnsFrom": [ + "triggered_by_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "blocked_email_domains_domain_normalized": { + "name": "blocked_email_domains_domain_normalized", + "value": "\"blocked_email_domains\".\"domain\" ~ '^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?(?:\\.[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)+$'" + } + }, + "isRLSEnabled": false + }, + "public.workload_abuse_detections": { + "name": "workload_abuse_detections", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "wallet_id": { + "name": "wallet_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "dseq": { + "name": "dseq", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "verdict": { + "name": "verdict", + "type": "workload_abuse_verdict", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "probe_status": { + "name": "probe_status", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "signals": { + "name": "signals", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "evidence_excerpt": { + "name": "evidence_excerpt", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "workload_abuse_action", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'detected'" + }, + "enforcement_error": { + "name": "enforcement_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workload_abuse_detections_user_id_idx": { + "name": "workload_abuse_detections_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workload_abuse_detections_dseq_idx": { + "name": "workload_abuse_detections_dseq_idx", + "columns": [ + { + "expression": "dseq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workload_abuse_detections_user_id_userSetting_id_fk": { + "name": "workload_abuse_detections_user_id_userSetting_id_fk", + "tableFrom": "workload_abuse_detections", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workload_probe_evidence": { + "name": "workload_probe_evidence", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "wallet_id": { + "name": "wallet_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "dseq": { + "name": "dseq", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service": { + "name": "service", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "probe_status": { + "name": "probe_status", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "verdict": { + "name": "verdict", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true + }, + "detection_id": { + "name": "detection_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "accelerator": { + "name": "accelerator", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "process_origins": { + "name": "process_origins", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "net_shape": { + "name": "net_shape", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "behavioural_findings": { + "name": "behavioural_findings", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workload_probe_evidence_wallet_dseq_created_idx": { + "name": "workload_probe_evidence_wallet_dseq_created_idx", + "columns": [ + { + "expression": "wallet_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dseq", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workload_probe_evidence_created_idx": { + "name": "workload_probe_evidence_created_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.stripe_transaction_status": { + "name": "stripe_transaction_status", + "schema": "public", + "values": [ + "created", + "pending", + "requires_action", + "succeeded", + "failed", + "refunded", + "canceled" + ] + }, + "public.stripe_transaction_type": { + "name": "stripe_transaction_type", + "schema": "public", + "values": [ + "payment_intent", + "coupon_claim", + "manual_credit" + ] + }, + "public.auto_reload_mode": { + "name": "auto_reload_mode", + "schema": "public", + "values": [ + "prediction", + "threshold" + ] + }, + "public.blocked_email_domain_source": { + "name": "blocked_email_domain_source", + "schema": "public", + "values": [ + "auto", + "manual" + ] + }, + "public.blocked_email_domain_status": { + "name": "blocked_email_domain_status", + "schema": "public", + "values": [ + "blocked", + "allowed" + ] + }, + "public.workload_abuse_action": { + "name": "workload_abuse_action", + "schema": "public", + "values": [ + "detected", + "enforcing", + "enforced", + "enforcement_failed" + ] + }, + "public.workload_abuse_verdict": { + "name": "workload_abuse_verdict", + "schema": "public", + "values": [ + "hard", + "soft", + "proxy" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/apps/api/drizzle/meta/_journal.json b/apps/api/drizzle/meta/_journal.json index 3484520e3b..4eb242cd4a 100644 --- a/apps/api/drizzle/meta/_journal.json +++ b/apps/api/drizzle/meta/_journal.json @@ -407,6 +407,13 @@ "when": 1789465677452, "tag": "0057_deployment_settings_user_sealed_secrets_index", "breakpoints": true + }, + { + "idx": 58, + "version": "7", + "when": 1789839246549, + "tag": "0058_workload_probe_evidence", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/api/src/workload-abuse/config/env.config.ts b/apps/api/src/workload-abuse/config/env.config.ts index 1f5dc1a664..1f7b9529a7 100644 --- a/apps/api/src/workload-abuse/config/env.config.ts +++ b/apps/api/src/workload-abuse/config/env.config.ts @@ -97,7 +97,9 @@ export const envSchema = z.object({ /** Bounds the damage of a domain match that turns out to be too broad. */ WORKLOAD_ABUSE_DOMAIN_BLOCK_MAX_SIBLINGS: z.number({ coerce: true }).int().positive().default(200), /** A domain with an account older than this predates the attack, so it is somebody's real domain. */ - WORKLOAD_ABUSE_DOMAIN_BLOCK_MIN_ACCOUNT_AGE_DAYS: z.number({ coerce: true }).int().positive().default(30) + WORKLOAD_ABUSE_DOMAIN_BLOCK_MIN_ACCOUNT_AGE_DAYS: z.number({ coerce: true }).int().positive().default(30), + /** Evidence rows feed the behavioural replay, so they must outlive its 30-day window with margin. */ + WORKLOAD_ABUSE_EVIDENCE_RETENTION_DAYS: z.number({ coerce: true }).int().positive().default(90) }); export type WorkloadAbuseConfig = z.infer; diff --git a/apps/api/src/workload-abuse/controllers/workload-abuse.controller.spec.ts b/apps/api/src/workload-abuse/controllers/workload-abuse.controller.spec.ts index d2c6b40e47..45ad2b1973 100644 --- a/apps/api/src/workload-abuse/controllers/workload-abuse.controller.spec.ts +++ b/apps/api/src/workload-abuse/controllers/workload-abuse.controller.spec.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { mock } from "vitest-mock-extended"; +import type { ProbeEvidenceService } from "@src/workload-abuse/services/probe-evidence/probe-evidence.service"; import type { TrialAbuseEnforcementJobService } from "@src/workload-abuse/services/trial-abuse-enforcement-job/trial-abuse-enforcement-job.service"; import type { TrialWorkloadProbeJobService } from "@src/workload-abuse/services/trial-workload-probe-job/trial-workload-probe-job.service"; import { WorkloadAbuseController } from "./workload-abuse.controller"; @@ -35,13 +36,31 @@ describe(WorkloadAbuseController.name, () => { await expect(controller.probeTrialDeployments({ dryRun: false })).rejects.toMatchObject({ errors: [probeError, enforcementError] }); }); + it("purges expired evidence once the sweeps settle", async () => { + const { controller, probeEvidenceService } = setup(); + + await controller.probeTrialDeployments({ dryRun: true }); + + expect(probeEvidenceService.purgeExpired).toHaveBeenCalledTimes(1); + }); + + it("leaves the purge to the next run when a sweep fails", async () => { + const { controller, probeJobService, probeEvidenceService } = setup(); + probeJobService.reconcile.mockRejectedValue(new Error("db down")); + + await expect(controller.probeTrialDeployments({ dryRun: false })).rejects.toThrow("db down"); + + expect(probeEvidenceService.purgeExpired).not.toHaveBeenCalled(); + }); + function setup() { const probeJobService = mock(); probeJobService.reconcile.mockResolvedValue(); const enforcementJobService = mock(); enforcementJobService.reconcile.mockResolvedValue(); - const controller = new WorkloadAbuseController(probeJobService, enforcementJobService); + const probeEvidenceService = mock(); + const controller = new WorkloadAbuseController(probeJobService, enforcementJobService, probeEvidenceService); - return { controller, probeJobService, enforcementJobService }; + return { controller, probeJobService, enforcementJobService, probeEvidenceService }; } }); diff --git a/apps/api/src/workload-abuse/controllers/workload-abuse.controller.ts b/apps/api/src/workload-abuse/controllers/workload-abuse.controller.ts index 425615cd3d..98467ac97f 100644 --- a/apps/api/src/workload-abuse/controllers/workload-abuse.controller.ts +++ b/apps/api/src/workload-abuse/controllers/workload-abuse.controller.ts @@ -1,6 +1,7 @@ import { singleton } from "tsyringe"; import type { DryRunOptions } from "@src/core/types/console"; +import { ProbeEvidenceService } from "@src/workload-abuse/services/probe-evidence/probe-evidence.service"; import { TrialAbuseEnforcementJobService } from "@src/workload-abuse/services/trial-abuse-enforcement-job/trial-abuse-enforcement-job.service"; import { TrialWorkloadProbeJobService } from "@src/workload-abuse/services/trial-workload-probe-job/trial-workload-probe-job.service"; @@ -8,7 +9,8 @@ import { TrialWorkloadProbeJobService } from "@src/workload-abuse/services/trial export class WorkloadAbuseController { constructor( private readonly probeJobService: TrialWorkloadProbeJobService, - private readonly enforcementJobService: TrialAbuseEnforcementJobService + private readonly enforcementJobService: TrialAbuseEnforcementJobService, + private readonly probeEvidenceService: ProbeEvidenceService ) {} /** Each sweep runs whether or not the other fails, so a probe outage does not leave stuck wipes waiting another run. */ @@ -18,5 +20,7 @@ export class WorkloadAbuseController { if (failures.length === 1) throw failures[0]; if (failures.length > 1) throw new AggregateError(failures, "Both the probe sweep and the enforcement sweep failed"); + + await this.probeEvidenceService.purgeExpired(); } } diff --git a/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.spec.ts b/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.spec.ts new file mode 100644 index 0000000000..54db21df06 --- /dev/null +++ b/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.spec.ts @@ -0,0 +1,152 @@ +import { describe, expect, it } from "vitest"; + +import { parseProbeEvidence, withoutEvidenceSections } from "./parse-probe-evidence"; + +describe("parseProbeEvidence", () => { + it("parses accelerator, processes, artifacts, process origins, and net shape from full probe output", () => { + const raw = [ + "--loadavg", + "0.52 0.58 0.59 1/902 12345", + "--nproc", + "4", + "--procs", + "999 cpu_s=12 rss_mb=512 comm=python3 exe=/usr/bin/python3 cwd=/ app.py", + "--net", + " 2 listen=8080", + " 3 st=01 remote=18.185.10.20:443", + "--tmp", + "total 8", + "--files", + "== /tmp/app.conf", + "| key=value", + "--authorized-keys", + "--recent-exec", + "/usr/bin/python3", + "--recent-conf", + "/home/user/app.ini", + "--accel", + "NVIDIA T4, 95, 15130, 15360", + "1234, python3, 15100", + "--netl", + " 3 0D05 140AB912:01BB 01", + "--disk", + "73014444032 /root/.cache/model.safetensors", + "--procorig", + "btime=1740000000", + "1234 ppid=1 starttime=650000 comm=python3", + "" + ].join("\n"); + + const features = parseProbeEvidence(raw); + + expect(features.accelerator).toEqual([ + { name: "NVIDIA T4", utilPct: 95, memUsedMb: 15130, memTotalMb: 15360, processes: [{ pid: 1234, name: "python3", vramMb: 15100 }] } + ]); + expect(features.artifacts).toEqual([{ path: "/root/.cache/model.safetensors", sizeBytes: 73014444032 }]); + expect(features.processOrigins).toEqual([{ pid: 1234, ppid: 1, comm: "python3", startedAtEpochMs: 1740006500000 }]); + expect(features.netShape).toEqual({ + listenPorts: [8080], + established: [{ localPort: 3333, remoteIp: "18.185.10.20", remotePort: 443, count: 3 }] + }); + }); + + it("returns null accelerator when the tooling is absent", () => { + const features = parseProbeEvidence("--accel\naccel: unavailable\n"); + + expect(features.accelerator).toBeNull(); + }); + + it("returns nulls when no evidence sections are present", () => { + const features = parseProbeEvidence("--loadavg\n0.10 0.20 0.30 1/900 1\n--nproc\n2\n"); + + expect(features).toEqual({ accelerator: null, artifacts: null, processOrigins: null, netShape: null }); + }); + + it("tolerates non-numeric vram placeholders in compute app lines", () => { + const features = parseProbeEvidence("--accel\nNVIDIA T4, 0, 0, 15360\n1234, python3, [N/A]\n"); + + expect(features.accelerator).toEqual([ + { name: "NVIDIA T4", utilPct: 0, memUsedMb: 0, memTotalMb: 15360, processes: [{ pid: 1234, name: "python3", vramMb: 0 }] } + ]); + }); + + it("keeps commas in process names within one process entry", () => { + const features = parseProbeEvidence("--accel\nNVIDIA T4, 5, 100, 15360\n42, trainer, extra, part, 96\n"); + + expect(features.accelerator).toEqual([ + { name: "NVIDIA T4", utilPct: 5, memUsedMb: 100, memTotalMb: 15360, processes: [{ pid: 42, name: "trainer, extra, part", vramMb: 96 }] } + ]); + }); + + it("keeps spaces in artifact paths and process comm fields", () => { + const features = parseProbeEvidence( + ["--disk", "104857600 /root/My Folder/model file.bin", "--procorig", "btime=100", "7 ppid=1 starttime=50 comm=my worker", ""].join("\n") + ); + + expect(features.artifacts).toEqual([{ path: "/root/My Folder/model file.bin", sizeBytes: 104857600 }]); + expect(features.processOrigins).toEqual([{ pid: 7, ppid: 1, comm: "my worker", startedAtEpochMs: 100500 }]); + }); + + it("decodes v4-mapped and native ipv6 remotes from netl lines", () => { + const raw = [ + "--net", + " 1 listen=22", + "--netl", + " 1 0016 0000000000000000FFFF00000100007F:01BB 01", + " 2 0EA7 F804012A8F0B170C0000000002000000:0050 01", + "" + ].join("\n"); + + const features = parseProbeEvidence(raw); + + expect(features.netShape?.listenPorts).toEqual([22]); + expect(features.netShape?.established).toEqual([ + { localPort: 22, remoteIp: "127.0.0.1", remotePort: 443, count: 1 }, + { localPort: 3751, remoteIp: "2a01:04f8:0c17:0b8f:0000:0000:0000:0002", remotePort: 80, count: 2 } + ]); + }); + + it("merges duplicate netl keys into one counted entry", () => { + const raw = ["--netl", " 2 0D05 140AB912:01BB 01", " 3 0D05 140AB912:01BB 01", ""].join("\n"); + + const features = parseProbeEvidence(raw); + + expect(features.netShape?.established).toEqual([{ localPort: 3333, remoteIp: "18.185.10.20", remotePort: 443, count: 5 }]); + }); + + it("ignores malformed section lines instead of throwing", () => { + const features = parseProbeEvidence( + ["--accel", "not,a,gpu,line", "--disk", "notasize /path", "--procorig", "nonsense line", "--netl", "garbage", ""].join("\n") + ); + + expect(features.accelerator).toEqual([]); + expect(features.artifacts).toEqual([]); + expect(features.processOrigins).toEqual([]); + expect(features.netShape).toEqual({ listenPorts: [], established: [] }); + }); + + it("reports zero startedAtEpochMs when the btime header is missing", () => { + const features = parseProbeEvidence("--procorig\n999 ppid=1 starttime=100 comm=sh\n"); + + expect(features.processOrigins).toEqual([{ pid: 999, ppid: 1, comm: "sh", startedAtEpochMs: 0 }]); + }); +}); + +describe("withoutEvidenceSections", () => { + it("strips everything from the first --accel marker onward", () => { + const legacy = ["--loadavg", "0.10", "--nproc", "2"].join("\n") + "\n"; + const evidence = ["--accel", "NVIDIA T4, 1, 2, 3", "--netl", " 1 0D05 140AB912:01BB 01", "--disk", "1 /x", "--procorig", "btime=1"].join("\n") + "\n"; + + expect(withoutEvidenceSections(legacy + evidence)).toBe(legacy); + }); + + it("returns empty output when --accel opens the output", () => { + expect(withoutEvidenceSections("--accel\nNVIDIA T4, 1, 2, 3\n")).toBe(""); + }); + + it("returns the output unchanged when no evidence markers exist", () => { + const legacyOnly = "--loadavg\n0.10\n--nproc\n2\n"; + + expect(withoutEvidenceSections(legacyOnly)).toBe(legacyOnly); + }); +}); diff --git a/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.ts b/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.ts new file mode 100644 index 0000000000..d1c87946bb --- /dev/null +++ b/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.ts @@ -0,0 +1,183 @@ +import type { ProbeEvidenceAccelerator, ProbeEvidenceArtifact, ProbeEvidenceNetShape, ProbeEvidenceProcessOrigin } from "@src/workload-abuse/model-schemas"; + +export type ProbeEvidenceFeatures = { + accelerator: ProbeEvidenceAccelerator[] | null; + artifacts: ProbeEvidenceArtifact[] | null; + processOrigins: ProbeEvidenceProcessOrigin[] | null; + netShape: ProbeEvidenceNetShape | null; +}; + +const SECTION_MARKERS = [ + "--loadavg", + "--nproc", + "--procs", + "--net", + "--tmp", + "--files", + "--authorized-keys", + "--recent-exec", + "--recent-conf", + "--accel", + "--netl", + "--disk", + "--procorig" +] as const; + +export function parseProbeEvidence(rawShellOutput: string): ProbeEvidenceFeatures { + const sections = splitSections(rawShellOutput); + + return { + accelerator: parseAccelerator(sections.get("--accel")), + artifacts: parseArtifacts(sections.get("--disk")), + processOrigins: parseProcessOrigins(sections.get("--procorig")), + netShape: parseNetShape(sections.get("--net"), sections.get("--netl")) + }; +} + +export function withoutEvidenceSections(rawShellOutput: string): string { + const boundary = rawShellOutput.indexOf("\n--accel\n"); + if (boundary !== -1) return rawShellOutput.slice(0, boundary + 1); + return rawShellOutput.startsWith("--accel\n") ? "" : rawShellOutput; +} + +function splitSections(output: string): Map { + const sections = new Map(); + let currentLines: string[] | null = null; + + for (const line of output.split("\n")) { + if ((SECTION_MARKERS as readonly string[]).includes(line)) { + currentLines = sections.get(line) ?? []; + sections.set(line, currentLines); + continue; + } + currentLines?.push(line); + } + + return sections; +} + +function parseAccelerator(lines: string[] | undefined): ProbeEvidenceAccelerator[] | null { + if (!lines) return null; + if (lines.some(line => line.trim() === "accel: unavailable")) return null; + + const accelerators: Array> = []; + const processes: ProbeEvidenceAccelerator["processes"] = []; + + for (const line of lines) { + const fields = line.split(",").map(field => field.trim()); + if (fields.length >= 4 && !isNumeric(fields[0]) && isNumeric(fields[1]) && isNumeric(fields[2]) && isNumeric(fields[3])) { + accelerators.push({ + name: fields[0], + utilPct: Number(fields[1]), + memUsedMb: Number(fields[2]), + memTotalMb: Number(fields[3]) + }); + } else if (fields.length >= 3 && isNumeric(fields[0])) { + processes.push({ + pid: Number(fields[0]), + name: fields.slice(1, -1).join(", "), + vramMb: toNumberOrZero(fields[fields.length - 1]) + }); + } + } + + return accelerators.map(accelerator => ({ ...accelerator, processes })); +} + +function parseArtifacts(lines: string[] | undefined): ProbeEvidenceArtifact[] | null { + if (!lines) return null; + + const artifacts: ProbeEvidenceArtifact[] = []; + for (const line of lines) { + const match = line.match(/^(\d+) (.+)$/); + if (match) artifacts.push({ sizeBytes: Number(match[1]), path: match[2] }); + } + + return artifacts; +} + +function parseProcessOrigins(lines: string[] | undefined): ProbeEvidenceProcessOrigin[] | null { + if (!lines) return null; + + let bootTimeSeconds: number | null = null; + const origins: ProbeEvidenceProcessOrigin[] = []; + + for (const line of lines) { + const bootTimeMatch = line.match(/^btime=(\d+)$/); + if (bootTimeMatch) { + bootTimeSeconds = Number(bootTimeMatch[1]); + continue; + } + const match = line.match(/^(\d+) ppid=(\d+) starttime=(\d+) comm=(.*)$/); + if (!match) continue; + const starttimeTicks = Number(match[3]); + origins.push({ + pid: Number(match[1]), + ppid: Number(match[2]), + comm: match[4], + startedAtEpochMs: bootTimeSeconds === null ? 0 : Math.round((bootTimeSeconds + starttimeTicks / 100) * 1000) + }); + } + + return origins; +} + +function parseNetShape(netLines: string[] | undefined, netlLines: string[] | undefined): ProbeEvidenceNetShape | null { + if (!netlLines) return null; + + const listenPorts = new Set(); + if (netLines) { + for (const line of netLines) { + const match = line.trim().match(/^\d+\s+listen=(\d+)$/); + if (match) listenPorts.add(Number(match[1])); + } + } + + const establishedByKey = new Map(); + for (const line of netlLines) { + const match = line.trim().match(/^(\d+)\s+([0-9A-Fa-f]+)\s+([0-9A-Fa-f]+):([0-9A-Fa-f]+)\s+\d+$/); + if (!match) continue; + const localPort = parseInt(match[2], 16); + const remoteIp = decodeHexIp(match[3]); + const remotePort = parseInt(match[4], 16); + if (!remoteIp) continue; + const key = `${localPort}|${remoteIp}|${remotePort}`; + const existing = establishedByKey.get(key); + if (existing) existing.count += Number(match[1]); + else establishedByKey.set(key, { localPort, remoteIp, remotePort, count: Number(match[1]) }); + } + + return { + listenPorts: [...listenPorts].sort((a, b) => a - b), + established: [...establishedByKey.values()] + }; +} + +function decodeHexIp(hex: string): string | null { + if (hex.length === 8) { + return `${parseInt(hex.slice(6, 8), 16)}.${parseInt(hex.slice(4, 6), 16)}.${parseInt(hex.slice(2, 4), 16)}.${parseInt(hex.slice(0, 2), 16)}`; + } + if (hex.length === 32) { + const lowered = hex.toLowerCase(); + if (lowered.startsWith("0000000000000000ffff0000")) return decodeHexIp(lowered.slice(24)); + return formatIpv6(lowered); + } + return null; +} + +function formatIpv6(hex: string): string { + const hextets: string[] = []; + for (let index = 0; index < 32; index += 8) { + const group = hex.slice(index, index + 8); + hextets.push(`${group.slice(6, 8)}${group.slice(4, 6)}`, `${group.slice(2, 4)}${group.slice(0, 2)}`); + } + return hextets.join(":").toLowerCase(); +} + +function toNumberOrZero(value: string): number { + return isNumeric(value) ? Number(value) : 0; +} + +function isNumeric(value: string): boolean { + return /^\d+(\.\d+)?$/.test(value); +} diff --git a/apps/api/src/workload-abuse/model-schemas/index.ts b/apps/api/src/workload-abuse/model-schemas/index.ts index 0d27584ae4..869922b71e 100644 --- a/apps/api/src/workload-abuse/model-schemas/index.ts +++ b/apps/api/src/workload-abuse/model-schemas/index.ts @@ -1,2 +1,3 @@ export * from "./blocked-email-domain/blocked-email-domain.schema"; export * from "./workload-abuse-detection/workload-abuse-detection.schema"; +export * from "./workload-probe-evidence/workload-probe-evidence.schema"; diff --git a/apps/api/src/workload-abuse/model-schemas/workload-probe-evidence/workload-probe-evidence.schema.ts b/apps/api/src/workload-abuse/model-schemas/workload-probe-evidence/workload-probe-evidence.schema.ts new file mode 100644 index 0000000000..fa03581ae1 --- /dev/null +++ b/apps/api/src/workload-abuse/model-schemas/workload-probe-evidence/workload-probe-evidence.schema.ts @@ -0,0 +1,48 @@ +import { sql } from "drizzle-orm"; +import { index, integer, jsonb, pgTable, text, timestamp, uuid, varchar } from "drizzle-orm/pg-core"; + +export type ProbeEvidenceAccelerator = { + name: string; + utilPct: number; + memUsedMb: number; + memTotalMb: number; + processes: Array<{ pid: number; name: string; vramMb: number }>; +}; + +export type ProbeEvidenceArtifact = { path: string; sizeBytes: number }; + +export type ProbeEvidenceProcessOrigin = { pid: number; ppid: number; comm: string; startedAtEpochMs: number }; + +export type ProbeEvidenceNetShape = { + listenPorts: number[]; + established: Array<{ localPort: number; remoteIp: string; remotePort: number; count: number }>; +}; + +export type ProbeEvidenceBehaviouralFinding = { signal: string; detail: Record }; + +export const WorkloadProbeEvidence = pgTable( + "workload_probe_evidence", + { + id: uuid("id") + .primaryKey() + .notNull() + .default(sql`uuid_generate_v4()`), + walletId: integer("wallet_id").notNull(), + dseq: varchar("dseq").notNull(), + provider: text("provider").notNull(), + service: varchar("service", { length: 255 }).notNull(), + probeStatus: varchar("probe_status", { length: 64 }).notNull(), + verdict: varchar("verdict", { length: 16 }).notNull(), + detectionId: uuid("detection_id"), + accelerator: jsonb("accelerator").$type(), + artifacts: jsonb("artifacts").$type(), + processOrigins: jsonb("process_origins").$type(), + netShape: jsonb("net_shape").$type(), + behaviouralFindings: jsonb("behavioural_findings").$type(), + createdAt: timestamp("created_at", { withTimezone: true }).defaultNow().notNull() + }, + table => ({ + walletDseqCreatedIdx: index("workload_probe_evidence_wallet_dseq_created_idx").on(table.walletId, table.dseq, table.createdAt), + createdIdx: index("workload_probe_evidence_created_idx").on(table.createdAt) + }) +); diff --git a/apps/api/src/workload-abuse/repositories/workload-probe-evidence/workload-probe-evidence.repository.ts b/apps/api/src/workload-abuse/repositories/workload-probe-evidence/workload-probe-evidence.repository.ts new file mode 100644 index 0000000000..8896df592a --- /dev/null +++ b/apps/api/src/workload-abuse/repositories/workload-probe-evidence/workload-probe-evidence.repository.ts @@ -0,0 +1,51 @@ +import { and, asc, eq, gt, lt, sql } from "drizzle-orm"; +import { singleton } from "tsyringe"; + +import { type ApiPgDatabase, type ApiPgTables, InjectPg, InjectPgTable } from "@src/core/providers"; +import { type AbilityParams, BaseRepository } from "@src/core/repositories/base.repository"; +import { TxService } from "@src/core/services"; + +type Table = ApiPgTables["WorkloadProbeEvidence"]; +export type WorkloadProbeEvidenceInput = Partial; +export type WorkloadProbeEvidenceInsert = Table["$inferInsert"]; +export type WorkloadProbeEvidenceOutput = Table["$inferSelect"]; + +@singleton() +export class WorkloadProbeEvidenceRepository extends BaseRepository { + constructor( + @InjectPg() protected readonly pg: ApiPgDatabase, + @InjectPgTable("WorkloadProbeEvidence") protected readonly table: Table, + protected readonly txManager: TxService + ) { + super(pg, table, txManager, "WorkloadProbeEvidence", "WorkloadProbeEvidence"); + } + + accessibleBy(...abilityParams: AbilityParams) { + return new WorkloadProbeEvidenceRepository(this.pg, this.table, this.txManager).withAbility(...abilityParams) as this; + } + + async insertMany(rows: WorkloadProbeEvidenceInsert[]): Promise { + if (!rows.length) return []; + return await this.cursor.insert(this.table).values(rows).returning(); + } + + async findRecentForDeployment({ walletId, dseq, since }: { walletId: number; dseq: string; since: Date }): Promise { + return await this.cursor + .select() + .from(this.table) + .where(and(eq(this.table.walletId, walletId), eq(this.table.dseq, dseq), gt(this.table.createdAt, since))) + .orderBy(asc(this.table.createdAt)); + } + + async countDistinctDeploymentsSince({ since }: { since: Date }): Promise { + const [row] = await this.cursor + .select({ deployments: sql`count(distinct (${this.table.walletId}, ${this.table.dseq}))` }) + .from(this.table) + .where(gt(this.table.createdAt, since)); + return Number(row?.deployments ?? 0); + } + + async deleteOlderThan({ before }: { before: Date }): Promise { + await this.cursor.delete(this.table).where(lt(this.table.createdAt, before)); + } +} diff --git a/apps/api/src/workload-abuse/services/probe-evidence/probe-evidence.service.spec.ts b/apps/api/src/workload-abuse/services/probe-evidence/probe-evidence.service.spec.ts new file mode 100644 index 0000000000..75120bbd8d --- /dev/null +++ b/apps/api/src/workload-abuse/services/probe-evidence/probe-evidence.service.spec.ts @@ -0,0 +1,117 @@ +import { describe, expect, it, vi } from "vitest"; +import { mock } from "vitest-mock-extended"; + +import { type CreateLogger } from "@src/core"; +import type { WorkloadProbeEvidenceRepository } from "@src/workload-abuse/repositories/workload-probe-evidence/workload-probe-evidence.repository"; +import type { WorkloadAbuseConfigService } from "@src/workload-abuse/services/workload-abuse-config/workload-abuse-config.service"; +import type { WorkloadAbuseInstrumentationService } from "@src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service"; +import { ProbeEvidenceService } from "./probe-evidence.service"; + +import { mockConfigService } from "@test/mocks/config-service.mock"; + +describe(ProbeEvidenceService.name, () => { + it("records one row per shell output with parsed features", async () => { + const { service, evidenceRepository } = setup(); + evidenceRepository.insertMany.mockResolvedValue([]); + + await service.recordEvidence({ + walletId: 42, + dseq: "1000001", + verdict: "clean", + probeStatus: "completed", + shellOutputs: [ + { + service: "web", + provider: "akash1provider", + output: "--accel\nNVIDIA T4, 90, 14000, 15360\n1234, python3, 14000\n--disk\n1073741824 /root/model.bin\n" + }, + { service: "sidecar", provider: "akash1provider", output: "--loadavg\n0.10\n" } + ] + }); + + expect(evidenceRepository.insertMany).toHaveBeenCalledWith([ + expect.objectContaining({ + walletId: 42, + dseq: "1000001", + provider: "akash1provider", + service: "web", + probeStatus: "completed", + verdict: "clean", + accelerator: [expect.objectContaining({ name: "NVIDIA T4", utilPct: 90 })], + artifacts: [{ path: "/root/model.bin", sizeBytes: 1073741824 }] + }), + expect.objectContaining({ service: "sidecar", accelerator: null, artifacts: null }) + ]); + }); + + it("skips the write when there are no shell outputs", async () => { + const { service, evidenceRepository } = setup(); + + await service.recordEvidence({ walletId: 42, dseq: "1000001", verdict: "clean", probeStatus: "completed", shellOutputs: [] }); + + expect(evidenceRepository.insertMany).not.toHaveBeenCalled(); + }); + + it("passes the detection id through to every row", async () => { + const { service, evidenceRepository } = setup(); + evidenceRepository.insertMany.mockResolvedValue([]); + + await service.recordEvidence({ + walletId: 42, + dseq: "1000001", + verdict: "hard", + probeStatus: "completed", + detectionId: "detection-uuid", + shellOutputs: [{ service: "web", provider: "akash1provider", output: "--accel\nNVIDIA T4, 1, 2, 3\n" }] + }); + + expect(evidenceRepository.insertMany).toHaveBeenCalledWith([expect.objectContaining({ detectionId: "detection-uuid" })]); + }); + + it("logs and counts a write failure without rethrowing", async () => { + const { service, evidenceRepository, instrumentation, logger } = setup(); + evidenceRepository.insertMany.mockRejectedValue(new Error("connection refused")); + + await expect( + service.recordEvidence({ + walletId: 42, + dseq: "1000001", + verdict: "clean", + probeStatus: "completed", + shellOutputs: [{ service: "web", provider: "akash1provider", output: "--disk\n1 /root/x\n" }] + }) + ).resolves.toBeUndefined(); + + expect(instrumentation.recordEvidenceWriteFailure).toHaveBeenCalledTimes(1); + expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ event: "WORKLOAD_EVIDENCE_WRITE_FAILED", walletId: 42, dseq: "1000001" })); + }); + + it("purges rows older than the retention window", async () => { + const { service, evidenceRepository } = setup(); + + await service.purgeExpired(); + + expect(evidenceRepository.deleteOlderThan).toHaveBeenCalledWith({ before: expect.any(Date) }); + }); + + it("logs a purge failure without rethrowing", async () => { + const { service, evidenceRepository, instrumentation, logger } = setup(); + evidenceRepository.deleteOlderThan.mockRejectedValue(new Error("deadlock")); + + await expect(service.purgeExpired()).resolves.toBeUndefined(); + + expect(instrumentation.recordEvidenceWriteFailure).toHaveBeenCalledTimes(1); + expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ event: "WORKLOAD_EVIDENCE_PURGE_FAILED" })); + }); + + function setup(input?: { retentionDays?: number }) { + const evidenceRepository = mock(); + const instrumentation = mock(); + const logger = mock>(); + const createLogger = vi.fn(() => logger); + const config = mockConfigService({ WORKLOAD_ABUSE_EVIDENCE_RETENTION_DAYS: input?.retentionDays ?? 90 }); + const service = new ProbeEvidenceService(evidenceRepository, instrumentation, config, createLogger); + + return { service, evidenceRepository, instrumentation, logger, config }; + } +}); diff --git a/apps/api/src/workload-abuse/services/probe-evidence/probe-evidence.service.ts b/apps/api/src/workload-abuse/services/probe-evidence/probe-evidence.service.ts new file mode 100644 index 0000000000..dc54a831f0 --- /dev/null +++ b/apps/api/src/workload-abuse/services/probe-evidence/probe-evidence.service.ts @@ -0,0 +1,70 @@ +import { inject, singleton } from "tsyringe"; + +import { type CreateLogger, LOGGER_FACTORY } from "@src/core"; +import { parseProbeEvidence } from "@src/workload-abuse/lib/probe-evidence/parse-probe-evidence"; +import { WorkloadProbeEvidenceRepository } from "@src/workload-abuse/repositories/workload-probe-evidence/workload-probe-evidence.repository"; +import { WorkloadAbuseConfigService } from "@src/workload-abuse/services/workload-abuse-config/workload-abuse-config.service"; +import { WorkloadAbuseInstrumentationService } from "@src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service"; + +type ProbeEvidenceShellOutput = { service: string; provider: string; output: string }; + +@singleton() +export class ProbeEvidenceService { + private readonly logger: ReturnType; + + constructor( + private readonly evidenceRepository: WorkloadProbeEvidenceRepository, + private readonly instrumentation: WorkloadAbuseInstrumentationService, + private readonly config: WorkloadAbuseConfigService, + @inject(LOGGER_FACTORY) createLogger: CreateLogger + ) { + this.logger = createLogger({ context: ProbeEvidenceService.name }); + } + + async recordEvidence(input: { + walletId: number; + dseq: string; + verdict: string; + probeStatus: string; + detectionId?: string; + shellOutputs: ProbeEvidenceShellOutput[]; + }): Promise { + if (!input.shellOutputs.length) return; + + try { + await this.evidenceRepository.insertMany( + input.shellOutputs.map(shellOutput => { + const features = parseProbeEvidence(shellOutput.output); + return { + walletId: input.walletId, + dseq: input.dseq, + provider: shellOutput.provider, + service: shellOutput.service, + probeStatus: input.probeStatus, + verdict: input.verdict, + detectionId: input.detectionId, + accelerator: features.accelerator, + artifacts: features.artifacts, + processOrigins: features.processOrigins, + netShape: features.netShape + }; + }) + ); + } catch (error) { + this.instrumentation.recordEvidenceWriteFailure(); + this.logger.warn({ event: "WORKLOAD_EVIDENCE_WRITE_FAILED", error, walletId: input.walletId, dseq: input.dseq }); + } + } + + async purgeExpired(): Promise { + const retentionDays = this.config.get("WORKLOAD_ABUSE_EVIDENCE_RETENTION_DAYS"); + const before = new Date(Date.now() - retentionDays * 24 * 60 * 60 * 1000); + + try { + await this.evidenceRepository.deleteOlderThan({ before }); + } catch (error) { + this.instrumentation.recordEvidenceWriteFailure(); + this.logger.warn({ event: "WORKLOAD_EVIDENCE_PURGE_FAILED", error, before }); + } + } +} diff --git a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.integration.ts b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.integration.ts index 148af985fa..45224b9aae 100644 --- a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.integration.ts +++ b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.integration.ts @@ -5,6 +5,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { JOB_NAME } from "@src/core"; import { WorkloadAbuseDetectionRepository } from "@src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository"; +import { WorkloadProbeEvidenceRepository } from "@src/workload-abuse/repositories/workload-probe-evidence/workload-probe-evidence.repository"; import { type ProbeReport, TrialWorkloadProbeService } from "@src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service"; import { ProbeTrialDeployment, probeTrialDeploymentKeyFor } from "@src/workload-abuse/services/trial-workload-probe-job/trial-workload-probe-job.service"; import { WorkloadAbuseConfigService } from "@src/workload-abuse/services/workload-abuse-config/workload-abuse-config.service"; @@ -16,6 +17,8 @@ import { expectJobCompleted, findJobRows, useJobWorkers } from "@test/services/j const MAX_ATTEMPTS = 30; +const ACCELERATED_SHELL_OUTPUT = "--loadavg\n0.10\n--accel\nNVIDIA A100, 95, 20480, 24576\n1234, python3, 18000"; + const jobWorkers = useJobWorkers(() => [container.resolve(ProbeTrialDeploymentHandler)]); describe(ProbeTrialDeploymentHandler.name, () => { @@ -64,6 +67,42 @@ describe(ProbeTrialDeploymentHandler.name, () => { expect(await findDetections()).toHaveLength(0); }); + it("records evidence for every probed service on a clean probe", async () => { + const { probeDeployment, findEvidence } = await setup({ verdict: "clean" }); + + await probeDeployment(); + + expect(await findEvidence()).toMatchObject([ + { + verdict: "clean", + detectionId: null, + provider: "akash1provider", + service: "ssh", + accelerator: [{ name: "NVIDIA A100", utilPct: 95, memUsedMb: 20480, memTotalMb: 24576, processes: [{ pid: 1234, name: "python3", vramMb: 18000 }] }], + artifacts: null, + processOrigins: null, + netShape: null + } + ]); + }); + + it("links the evidence rows to the detection the probe recorded", async () => { + const { probeDeployment, findDetections, findEvidence } = await setup({ verdict: "hard" }); + + await probeDeployment(); + + const [detection] = await findDetections(); + expect(await findEvidence()).toMatchObject([{ verdict: "hard", detectionId: detection.id }]); + }); + + it("records no evidence when the deployment has no live lease", async () => { + const { probeDeployment, findEvidence } = await setup({ verdict: "clean", probeStatus: "no_live_lease" }); + + await probeDeployment(); + + expect(await findEvidence()).toHaveLength(0); + }); + it("records nothing more for a deployment already judged abusive", async () => { const { probeDeployment, findDetections, probe, seedExistingDetection } = await setup({ verdict: "hard" }); await seedExistingDetection(); @@ -74,9 +113,16 @@ describe(ProbeTrialDeploymentHandler.name, () => { expect(await findDetections()).toHaveLength(1); }); - async function setup(input: { verdict: ProbeReport["verdict"]; attempt?: number; isTrialing?: boolean }) { + async function setup(input: { + verdict: ProbeReport["verdict"]; + attempt?: number; + isTrialing?: boolean; + probeStatus?: ProbeReport["probeStatus"]; + shellOutputs?: ProbeReport["shellOutputs"]; + }) { const { enqueue, startWorkers } = await jobWorkers(); const detectionRepository = container.resolve(WorkloadAbuseDetectionRepository); + const evidenceRepository = container.resolve(WorkloadProbeEvidenceRepository); const { user, wallet, address } = await seedUserWithWallet({ isTrialing: input.isTrialing ?? true }); const dseq = createDseq(); @@ -86,10 +132,11 @@ describe(ProbeTrialDeploymentHandler.name, () => { const probe = vi.spyOn(container.resolve(TrialWorkloadProbeService), "probe").mockResolvedValue({ verdict: input.verdict, - probeStatus: "probed", + probeStatus: input.probeStatus ?? "probed", signals: [], excerpt: "denied", - leases: [] + leases: [], + shellOutputs: input.shellOutputs ?? [{ service: "ssh", provider: "akash1provider", output: ACCELERATED_SHELL_OUTPUT }] }); const probeKey = probeTrialDeploymentKeyFor({ walletId: wallet.id, dseq }); @@ -98,6 +145,7 @@ describe(ProbeTrialDeploymentHandler.name, () => { dseq, probe, findDetections: () => detectionRepository.find({ walletId: wallet.id, dseq }), + findEvidence: () => evidenceRepository.find({ walletId: wallet.id, dseq }), seedExistingDetection: () => detectionRepository.create({ userId: user.id, diff --git a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts index 9a60666c63..10943b5c4f 100644 --- a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts +++ b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts @@ -8,6 +8,7 @@ import type { WorkloadAbuseDetectionRepository } from "@src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository"; import { EnforceTrialAbuse } from "@src/workload-abuse/services/enforce-trial-abuse/enforce-trial-abuse.handler"; +import type { ProbeEvidenceService } from "@src/workload-abuse/services/probe-evidence/probe-evidence.service"; import type { ProbeReport, TrialWorkloadProbeService } from "@src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service"; import type { TrialWorkloadProbeJobService } from "@src/workload-abuse/services/trial-workload-probe-job/trial-workload-probe-job.service"; import type { WorkloadAbuseConfigService } from "@src/workload-abuse/services/workload-abuse-config/workload-abuse-config.service"; @@ -50,7 +51,7 @@ describe(ProbeTrialDeploymentHandler.name, () => { expect(detectionRepository.create).not.toHaveBeenCalled(); }); - it("reschedules a clean probe and records nothing", async () => { + it("reschedules a clean probe and records no detection", async () => { const { handler, probeJobService, detectionRepository, instrumentation } = setup({ report: createReport({ verdict: "clean" }) }); await handler.handle(PAYLOAD); @@ -60,6 +61,46 @@ describe(ProbeTrialDeploymentHandler.name, () => { expect(instrumentation.recordProbe).toHaveBeenCalledWith(expect.objectContaining({ verdict: "clean", probeStatus: "probed" })); }); + it("records evidence on a clean probe with the raw shell outputs and no detection id", async () => { + const shellOutputs = [{ service: "web", provider: "akash1provider", output: "--loadavg\n0.10" }]; + const { handler, wallet, probeEvidenceService } = setup({ report: createReport({ verdict: "clean", shellOutputs }) }); + + await handler.handle(PAYLOAD); + + expect(probeEvidenceService.recordEvidence).toHaveBeenCalledWith({ + walletId: wallet.id, + dseq: PAYLOAD.dseq, + verdict: "clean", + probeStatus: "probed", + detectionId: undefined, + shellOutputs + }); + }); + + it("records evidence with the detection id on a probe that lands a detection", async () => { + const { handler, probeEvidenceService } = setup({ report: createReport({ verdict: "hard" }) }); + + await handler.handle(PAYLOAD); + + expect(probeEvidenceService.recordEvidence).toHaveBeenCalledWith(expect.objectContaining({ detectionId: "detection-1" })); + }); + + it("logs the recorded evidence services for the probe run", async () => { + const { handler, logger } = setup({ report: createReport({ verdict: "clean" }) }); + + await handler.handle(PAYLOAD); + + expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ event: "TRIAL_WORKLOAD_EVIDENCE_RECORDED", services: ["ssh"] })); + }); + + it("records no evidence when the deployment has no live lease", async () => { + const { handler, probeEvidenceService } = setup({ report: createReport({ probeStatus: "no_live_lease", shellOutputs: [] }) }); + + await handler.handle(PAYLOAD); + + expect(probeEvidenceService.recordEvidence).not.toHaveBeenCalled(); + }); + it("logs what the shell saw for every verdict, cut so the line survives log shipping", async () => { const { handler, logger } = setup({ report: createReport({ verdict: "clean", excerpt: "x".repeat(5_000) }) }); @@ -182,6 +223,7 @@ describe(ProbeTrialDeploymentHandler.name, () => { logStatus: "completed" } ], + shellOutputs: [{ service: "ssh", provider: "akash1provider", output: "--loadavg\n0.10" }], ...overrides }; } @@ -232,6 +274,7 @@ describe(ProbeTrialDeploymentHandler.name, () => { const detectionRepository = mock(); detectionRepository.create.mockResolvedValue(mock({ id: "detection-1" })); detectionRepository.findOneBy.mockResolvedValue(input.existingDetection ? mock({ id: "detection-0" }) : undefined); + const probeEvidenceService = mock(); const instrumentation = mock(); const config = mockConfigService({ WORKLOAD_ABUSE_PROBE_ENABLED: input.enabled ?? true, @@ -247,12 +290,24 @@ describe(ProbeTrialDeploymentHandler.name, () => { probeService, probeJobService, detectionRepository, + probeEvidenceService, instrumentation, config, jobQueueService, createLogger ); - return { handler, wallet: wallet!, userWalletRepository, probeService, probeJobService, detectionRepository, instrumentation, jobQueueService, logger }; + return { + handler, + wallet: wallet!, + userWalletRepository, + probeService, + probeJobService, + detectionRepository, + probeEvidenceService, + instrumentation, + jobQueueService, + logger + }; } }); diff --git a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts index 7de693700a..cb661e3e32 100644 --- a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts +++ b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts @@ -6,6 +6,7 @@ import { withoutFileContents } from "@src/workload-abuse/lib/evidence-scanner/ev import { truncateToUtf8Bytes } from "@src/workload-abuse/lib/utf8-text/utf8-text"; import { WorkloadAbuseDetectionRepository } from "@src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository"; import { EnforceTrialAbuse, enforceTrialAbuseKeyFor } from "@src/workload-abuse/services/enforce-trial-abuse/enforce-trial-abuse.handler"; +import { ProbeEvidenceService } from "@src/workload-abuse/services/probe-evidence/probe-evidence.service"; import { type ProbeReport, TrialWorkloadProbeService } from "@src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service"; import { ProbeTrialDeployment, TrialWorkloadProbeJobService } from "@src/workload-abuse/services/trial-workload-probe-job/trial-workload-probe-job.service"; import { WorkloadAbuseConfigService } from "@src/workload-abuse/services/workload-abuse-config/workload-abuse-config.service"; @@ -31,6 +32,7 @@ export class ProbeTrialDeploymentHandler implements JobHandler shellOutput.service) + }); + this.logger.info({ event: "TRIAL_WORKLOAD_PROBED", ...context, diff --git a/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.spec.ts b/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.spec.ts index 1d42c855e1..bf4f9bffca 100644 --- a/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.spec.ts +++ b/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.spec.ts @@ -1,5 +1,5 @@ import { spawnSync } from "node:child_process"; -import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, symlinkSync, truncateSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; @@ -184,6 +184,92 @@ describe(ProviderShellProbeService.name, () => { }); }); + describe("the --accel collector", () => { + it("reports the accelerator model, utilization and memory along with the processes holding it", () => { + const binDirectory = mkdtempSync(join(tmpdir(), "shell-probe-accel-")); + writeFileSync( + join(binDirectory, "nvidia-smi"), + [ + "#!/bin/sh", + 'case "$1" in', + " --query-gpu*) printf 'NVIDIA A100, 95, 20480, 24576\\n' ;;", + " --query-compute-apps*) printf '1234, python3, 18000\\n' ;;", + "esac" + ].join("\n"), + { mode: 0o755 } + ); + + const reported = runCollector("--accel", collector => collector, { ...process.env, PATH: `${binDirectory}:${process.env.PATH}` }); + + expect(reported).toEqual(["--accel", "NVIDIA A100, 95, 20480, 24576", "1234, python3, 18000"]); + }); + + it("reports the accelerator as unavailable when the container has no nvidia-smi", () => { + const binDirectory = mkdtempSync(join(tmpdir(), "shell-probe-accel-")); + symlinkSync("/bin/sh", join(binDirectory, "sh")); + + const reported = runCollector("--accel", collector => collector, { PATH: binDirectory }); + + expect(reported).toEqual(["--accel", "accel: unavailable"]); + }); + }); + + describe("the --netl collector", () => { + it("reports the local port and remote endpoint of each connected socket, counted per peer", () => { + const reported = runNetlCollector({ + tcp: [ + " 0: 0100007F:9C4E 140AB912:0D05 01 00000000:00000000 00:00000000 00000000 0 0 2 1 0 0 0", + " 1: 0100007F:9C4E 140AB912:0D05 01 00000000:00000000 00:00000000 00000000 0 0 2 1 0 0 0", + " 2: 00000000:1F90 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 3 1 0 0 0" + ], + tcp6: [ + " 0: 0000000000000000FFFF00000100007F:A0F0 0000000000000000FFFF000009030E34:0D05 01 00000000:00000000 00:00000000 00000000 0 0 4 1 0 0 0" + ] + }); + + expect(reported).toEqual(expect.arrayContaining(["2 9C4E 140AB912:0D05 01", "1 A0F0 0000000000000000FFFF000009030E34:0D05 01"])); + expect(reported).toHaveLength(3); + }); + + it("leaves out a socket that is neither established nor connecting", () => { + const reported = runNetlCollector({ + tcp: [" 0: 0100007F:9C4E 140AB912:0D05 06 00000000:00000000 00:00000000 00000000 0 0 7 1 0 0 0"] + }); + + expect(reported).toEqual(["--netl"]); + }); + }); + + describe("the --disk collector", () => { + it("reports the size and path of a file over 64 MiB and leaves smaller files out", () => { + const directory = mkdtempSync(join(tmpdir(), "shell-probe-disk-")); + writeFileSync(join(directory, "weights.bin"), ""); + truncateSync(join(directory, "weights.bin"), 70 * 1024 * 1024); + writeFileSync(join(directory, "notes.txt"), "notes"); + + const reported = runCollector("--disk", collector => collector.replace("find / -xdev", `find ${directory} -xdev`)).map(line => line.trim()); + + expect(reported).toEqual(["--disk", `73400320 ${join(directory, "weights.bin")}`]); + }); + }); + + describe("the --procorig collector", () => { + it("reports the boot time and the parent, start time and name of each process it finds", () => { + const reported = runProcorigCollector([{ pid: "9000001", comm: "node", ppid: 1, starttimeTicks: 100, cmdline: ["node", "app.js"] }]); + + expect(reported).toEqual(["--procorig", "btime=1740000000", "9000001 ppid=1 starttime=100 comm=node"]); + }); + + it("leaves out a process with no command line, so kernel threads do not crowd out the workload", () => { + const reported = runProcorigCollector([ + { pid: "9000001", comm: "kthreadd", ppid: 2, starttimeTicks: 5, cmdline: [] }, + { pid: "9000002", comm: "node", ppid: 1, starttimeTicks: 200, cmdline: ["node", "app.js"] } + ]); + + expect(reported).toEqual(["--procorig", "btime=1740000000", "9000002 ppid=1 starttime=200 comm=node"]); + }); + }); + describe("run", () => { it("joins stdout and stderr into the output and keeps the stream status", async () => { const { service } = setup({ @@ -242,9 +328,9 @@ describe(ProviderShellProbeService.name, () => { return { reported: runCollector("--files", collector => collector.replace(TMP_FILE_GLOBS, `${directory}/*.conf`)), path }; } - function runCollector(section: string, toRunnable: (collector: string) => string) { + function runCollector(section: string, toRunnable: (collector: string) => string, env?: NodeJS.ProcessEnv) { const collector = SHELL_PROBE_COLLECTORS.find(entry => entry.includes(section)) ?? ""; - const { stdout } = spawnSync("sh", ["-c", toRunnable(collector)], { encoding: "utf8" }); + const { stdout } = spawnSync("sh", ["-c", toRunnable(collector)], { encoding: "utf8", env }); return stdout .split("\n") @@ -263,6 +349,36 @@ describe(ProviderShellProbeService.name, () => { return runCollector("--net", collector => collector.replace("/proc/net/tcp /proc/net/tcp6", `${directory}/tcp ${directory}/tcp6`)).map(line => line.trim()); } + function runNetlCollector(procFiles: { tcp?: string[]; tcp6?: string[] }) { + const header = " sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode"; + const directory = mkdtempSync(join(tmpdir(), "shell-probe-netl-")); + + for (const name of ["tcp", "tcp6"] as const) { + writeFileSync(join(directory, name), [header, ...(procFiles[name] ?? [])].join("\n") + "\n"); + } + + return runCollector("--netl", collector => collector.replace("/proc/net/tcp /proc/net/tcp6", `${directory}/tcp ${directory}/tcp6`)).map(line => + line.trim() + ); + } + + function runProcorigCollector(processes: { pid: string; comm: string; ppid: number; starttimeTicks: number; cmdline: string[] }[]) { + const directory = mkdtempSync(join(tmpdir(), "shell-probe-procorig-")); + writeFileSync(join(directory, "stat"), "btime 1740000000\n"); + + for (const fixture of processes) { + const fieldsAfterComm: (string | number)[] = Array(22).fill(0); + fieldsAfterComm[0] = "S"; + fieldsAfterComm[1] = fixture.ppid; + fieldsAfterComm[19] = fixture.starttimeTicks; + mkdirSync(join(directory, fixture.pid)); + writeFileSync(join(directory, fixture.pid, "stat"), `${fixture.pid} (${fixture.comm}) ${fieldsAfterComm.join(" ")}\n`); + writeFileSync(join(directory, fixture.pid, "cmdline"), fixture.cmdline.map(argument => `${argument}\0`).join("")); + } + + return runCollector("--procorig", collector => collector.replaceAll("/proc/", `${directory}/`)); + } + function setup(result: ProviderStreamResult) { const providerStreamService = mock(); providerStreamService.collect.mockResolvedValue(result); diff --git a/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.ts b/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.ts index 4be465eece..1c2c0d091e 100644 --- a/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.ts +++ b/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.ts @@ -14,7 +14,11 @@ export const SHELL_PROBE_COLLECTORS = [ `echo '--files'; for f in /tmp/*.json /tmp/*.conf /tmp/*.txt /tmp/*/*.json /tmp/*/*.conf; do [ -f "$f" ] && [ "$(wc -c < "$f")" -lt 16384 ] && printf '%s\\n' "== $f" && while IFS= read -r l || [ -n "$l" ]; do printf '${FILE_BODY_PREFIX}%s\\n' "$l"; l=; done < "$f"; done 2>/dev/null | head -400`, "echo '--authorized-keys'; cat /root/.ssh/authorized_keys /home/*/.ssh/authorized_keys 2>/dev/null | sort -u | head -10", "echo '--recent-exec'; find / \\( -path /proc -o -path /sys -o -path /dev -o -name node_modules \\) -prune -o -type f -perm -100 -newer /proc/1 -print 2>/dev/null | head -60 | while read -r f; do ls -la \"$f\" 2>/dev/null; done", - `echo '--recent-conf'; find / \\( -path /proc -o -path /sys -o -path /dev -o -path /etc -o -path /tmp -o -name node_modules \\) -prune -o -type f -newer /proc/1 \\( -name '*.json' -o -name '*.conf' -o -name '*.ini' -o -name '*.txt' \\) -size -16k -print 2>/dev/null | head -20 | while read -r f; do printf '%s\\n' "== $f"; [ -r "$f" ] || continue; while IFS= read -r l || [ -n "$l" ]; do printf '${FILE_BODY_PREFIX}%s\\n' "$l"; l=; done < "$f"; done | head -400` + `echo '--recent-conf'; find / \\( -path /proc -o -path /sys -o -path /dev -o -path /etc -o -path /tmp -o -name node_modules \\) -prune -o -type f -newer /proc/1 \\( -name '*.json' -o -name '*.conf' -o -name '*.ini' -o -name '*.txt' \\) -size -16k -print 2>/dev/null | head -20 | while read -r f; do printf '%s\\n' "== $f"; [ -r "$f" ] || continue; while IFS= read -r l || [ -n "$l" ]; do printf '${FILE_BODY_PREFIX}%s\\n' "$l"; l=; done < "$f"; done | head -400`, + "echo '--accel'; T=$(command -v timeout >/dev/null 2>&1 && printf 'timeout 2'); command -v nvidia-smi >/dev/null 2>&1 && { $T nvidia-smi --query-gpu=name,utilization.gpu,memory.used,memory.total --format=csv,noheader,nounits 2>/dev/null; $T nvidia-smi --query-compute-apps=pid,process_name,used_gpu_memory --format=csv,noheader,nounits 2>/dev/null; } || printf 'accel: unavailable\\n'", + 'echo \'--netl\'; for f in /proc/net/tcp /proc/net/tcp6; do [ -r "$f" ] || continue; while read -r sl la ra st rest; do case $st in 01|02) ;; *) continue;; esac; printf \'%s %s %s\\n\' "${la#*:}" "$ra" "$st"; done < "$f"; done 2>/dev/null | sort | uniq -c | head -80', + 'echo \'--disk\'; find / -xdev -type f -size +64M 2>/dev/null | head -200 | while read -r f; do printf \'%s %s\\n\' "$(wc -c < "$f" 2>/dev/null)" "$f"; done 2>/dev/null | sort -rn | head -40', + 'echo \'--procorig\'; b=$(grep ^btime /proc/stat 2>/dev/null); printf \'btime=%s\\n\' "${b#btime }"; T=$(command -v timeout >/dev/null 2>&1 && printf \'timeout 2\'); n=0; for p in /proc/[0-9]*; do [ "${p#/proc/}" = "$$" ] && continue; { s=; while IFS= read -r x; do s="$s$x "; done < "$p/stat"; } 2>/dev/null; [ -n "$s" ] || continue; m=${s#*(}; m=${m%)*}; set -- ${s##*) }; [ "${2:-}" = "$$" ] && continue; c=$({ $T tr \'\\0\' \' \' < "$p/cmdline"; } 2>/dev/null); [ -n "$c" ] || continue; printf \'%s\\n\' "${p#/proc/} ppid=${2:-0} starttime=${20:-0} comm=$m"; n=$((n + 1)); [ "$n" -ge 150 ] && break; done' ]; export const SHELL_PROBE_SCRIPT = SHELL_PROBE_COLLECTORS.join("; "); diff --git a/apps/api/src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service.spec.ts b/apps/api/src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service.spec.ts index e4a381fedb..02beda39e2 100644 --- a/apps/api/src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service.spec.ts +++ b/apps/api/src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service.spec.ts @@ -31,7 +31,7 @@ describe(TrialWorkloadProbeService.name, () => { const report = await service.probe({ wallet, dseq: DSEQ }); - expect(report).toEqual({ verdict: "clean", signals: [], excerpt: "", probeStatus: "no_live_lease", leases: [] }); + expect(report).toEqual({ verdict: "clean", signals: [], excerpt: "", probeStatus: "no_live_lease", leases: [], shellOutputs: [] }); expect(providerService.toProviderAuth).not.toHaveBeenCalled(); }); @@ -177,6 +177,32 @@ describe(TrialWorkloadProbeService.name, () => { expect(shellProbeService.run).not.toHaveBeenCalled(); }); + it("keeps the raw shell output out of the scanned sources while recording it in shellOutputs", async () => { + const shellOutput = [ + "--loadavg", + "0.10 0.20 0.30", + "--accel", + "NVIDIA T4, 95, 14000, 15360", + "1234, stratum+tcp://pool.example:3333, 14000", + "--disk", + "1073741824 /tmp/stratum+tcp://pool.example:3333", + "--procorig", + "btime=1740000000", + "1234 ppid=1 starttime=650000 comm=stratum+tcp://pool.example:3333" + ].join("\n"); + const { service, wallet } = setup({ leases: [createRpcLease()], services: { web: 1 }, shell: { status: "completed", output: shellOutput } }); + + const report = await service.probe({ wallet, dseq: DSEQ }); + + expect(report.verdict).toBe("clean"); + expect(report.signals).toEqual([]); + expect(report.excerpt).not.toContain("--accel"); + expect(report.excerpt).not.toContain("--disk"); + expect(report.excerpt).not.toContain("--procorig"); + expect(report.excerpt).not.toContain("stratum+tcp"); + expect(report.shellOutputs).toEqual([{ service: "web", provider: PROVIDER, output: shellOutput }]); + }); + function createRpcLease(overrides: Partial = {}): RpcLease { return mock({ lease: { id: { owner: "akash1owner", dseq: DSEQ, gseq: 1, oseq: 1, provider: PROVIDER, bseq: 1, ...overrides }, state: "active" } diff --git a/apps/api/src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service.ts b/apps/api/src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service.ts index 3586eec2c7..8c63c4eb92 100644 --- a/apps/api/src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service.ts +++ b/apps/api/src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service.ts @@ -15,6 +15,7 @@ import { toVerdict, type WorkloadVerdict } from "@src/workload-abuse/lib/evidence-scanner/evidence-scanner"; +import { withoutEvidenceSections } from "@src/workload-abuse/lib/probe-evidence/parse-probe-evidence"; import { ProviderLogTailService } from "@src/workload-abuse/services/provider-log-tail/provider-log-tail.service"; import { ProviderShellProbeService, type ShellProbeStatus } from "@src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service"; import { WorkloadAbuseConfigService } from "@src/workload-abuse/services/workload-abuse-config/workload-abuse-config.service"; @@ -37,6 +38,7 @@ export type ProbeReport = { excerpt: string; probeStatus: ProbeStatus; leases: ProbedLease[]; + shellOutputs: Array<{ service: string; provider: string; output: string }>; }; const PROVIDER_SCOPES = ["status", "logs", "shell"] as const; @@ -69,10 +71,11 @@ export class TrialWorkloadProbeService { const leases = await this.#findLiveLeases(input.wallet.address, input.dseq); if (leases.length === 0) { - return { verdict: "clean", signals: [], excerpt: "", probeStatus: "no_live_lease", leases: [] }; + return { verdict: "clean", signals: [], excerpt: "", probeStatus: "no_live_lease", leases: [], shellOutputs: [] }; } const sources: EvidenceSource[] = []; + const shellOutputs: ProbeReport["shellOutputs"] = []; const setting = await this.deploymentSettingRepository.findOneBy({ userId: input.wallet.userId, dseq: input.dseq }); if (setting?.sdl) sources.push({ kind: "sdl", text: setting.sdl }); @@ -91,7 +94,7 @@ export class TrialWorkloadProbeService { } for (const lease of leasesToProbe) { - const probed = await this.#probeLease(input.wallet, lease, sources); + const probed = await this.#probeLease(input.wallet, lease, sources, shellOutputs); statuses.push(probed.status); if (probed.lease) probedLeases.push(probed.lease); } @@ -103,7 +106,8 @@ export class TrialWorkloadProbeService { signals, excerpt: buildExcerpt(sources, signals), probeStatus: statuses.includes("probed") ? "probed" : statuses[0] ?? "stream_failed", - leases: probedLeases + leases: probedLeases, + shellOutputs }; } @@ -113,7 +117,12 @@ export class TrialWorkloadProbeService { return responses.flatMap(response => response.leases); } - async #probeLease(wallet: WalletInitialized, lease: RpcLease, sources: EvidenceSource[]): Promise<{ status: ProbeStatus; lease?: ProbedLease }> { + async #probeLease( + wallet: WalletInitialized, + lease: RpcLease, + sources: EvidenceSource[], + shellOutputs: ProbeReport["shellOutputs"] + ): Promise<{ status: ProbeStatus; lease?: ProbedLease }> { const { provider: providerAddress, dseq, gseq, oseq } = lease.lease.id; const provider = await this.providerRepository.findActiveByAddress(providerAddress); @@ -147,7 +156,10 @@ export class TrialWorkloadProbeService { for (const service of probedServices) { const shell = await this.shellProbeService.run({ ...target, service }); probedLease.shellStatuses.push(shell.status); - if (shell.output) sources.push({ kind: "shell", service, text: shell.output }); + if (shell.output) { + shellOutputs.push({ service, provider: providerAddress, output: shell.output }); + sources.push({ kind: "shell", service, text: withoutEvidenceSections(shell.output) }); + } } const logs = await this.logTailService.collect({ ...target, services: probedServices }); diff --git a/apps/api/src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service.ts b/apps/api/src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service.ts index 6307f13c87..d6943ee530 100644 --- a/apps/api/src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service.ts +++ b/apps/api/src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service.ts @@ -14,6 +14,7 @@ export class WorkloadAbuseInstrumentationService { private readonly enforcements: Counter; private readonly blockedDomainLookupFailures: Counter; private readonly domainBlocks: Counter; + private readonly evidenceWriteFailures: Counter; constructor(metricsService: MetricsService) { this.meter = metricsService.getMeter("workload-abuse", "1.0.0"); @@ -32,6 +33,9 @@ export class WorkloadAbuseInstrumentationService { this.domainBlocks = metricsService.createCounter(this.meter, "workload_abuse_domain_blocks_total", { description: "Email domain auto-block outcomes, by result and (on a skip) reason" }); + this.evidenceWriteFailures = metricsService.createCounter(this.meter, "workload_abuse_evidence_write_failures_total", { + description: "Probe evidence writes that failed to persist" + }); } recordProbe(input: { verdict: WorkloadVerdict; probeStatus: string }): void { @@ -53,4 +57,8 @@ export class WorkloadAbuseInstrumentationService { recordDomainBlock(result: DomainBlockResult, reason?: string): void { this.domainBlocks.add(1, reason ? { result, reason } : { result }); } + + recordEvidenceWriteFailure(): void { + this.evidenceWriteFailures.add(1); + } } From d3670fb6796154806f1936e4551df7100ebe8d79 Mon Sep 17 00:00:00 2001 From: Maxime Beauchamp <15185355+baktun14@users.noreply.github.com> Date: Sun, 20 Sep 2026 15:57:02 +0400 Subject: [PATCH 2/5] fix(deployment): split probe evidence from collected output at a per-run boundary The probe closes the collected sections with a line the workload cannot predict, and the shell probe splits the stream there. The scanner reads only what came before that line and the evidence parser only what came after, so neither side can be steered by what a container prints. Evidence rows now carry the status of the shell session that produced them instead of the deployment level status, so a collection that was cut short is visible rather than reading as an empty one. The retention purge no longer waits on a healthy sweep and no longer runs on a dry run, accelerator processes land on the card they ran on, and the evidence failure counter says which statement failed. --- .../drizzle/0058_workload_probe_evidence.sql | 2 +- apps/api/drizzle/meta/0058_snapshot.json | 6 +- .../workload-abuse.controller.spec.ts | 12 +- .../controllers/workload-abuse.controller.ts | 7 +- .../parse-probe-evidence.spec.ts | 69 ++++------- .../probe-evidence/parse-probe-evidence.ts | 74 ++++++------ .../workload-probe-evidence.schema.ts | 2 +- .../probe-evidence.service.spec.ts | 28 +++-- .../probe-evidence/probe-evidence.service.ts | 28 ++--- ...be-trial-deployment.handler.integration.ts | 7 +- .../probe-trial-deployment.handler.spec.ts | 13 +-- .../probe-trial-deployment.handler.ts | 5 +- .../provider-shell-probe.service.spec.ts | 108 +++++++++++++----- .../provider-shell-probe.service.ts | 43 +++++-- .../trial-workload-probe.service.spec.ts | 26 ++--- .../trial-workload-probe.service.ts | 21 ++-- .../workload-abuse-instrumentation.service.ts | 8 +- 17 files changed, 248 insertions(+), 211 deletions(-) diff --git a/apps/api/drizzle/0058_workload_probe_evidence.sql b/apps/api/drizzle/0058_workload_probe_evidence.sql index 1da7cede41..c2ba83b868 100644 --- a/apps/api/drizzle/0058_workload_probe_evidence.sql +++ b/apps/api/drizzle/0058_workload_probe_evidence.sql @@ -4,7 +4,7 @@ CREATE TABLE "workload_probe_evidence" ( "dseq" varchar NOT NULL, "provider" text NOT NULL, "service" varchar(255) NOT NULL, - "probe_status" varchar(64) NOT NULL, + "shell_status" varchar(64) NOT NULL, "verdict" varchar(16) NOT NULL, "detection_id" uuid, "accelerator" jsonb, diff --git a/apps/api/drizzle/meta/0058_snapshot.json b/apps/api/drizzle/meta/0058_snapshot.json index 2695cc76b6..db5f655c94 100644 --- a/apps/api/drizzle/meta/0058_snapshot.json +++ b/apps/api/drizzle/meta/0058_snapshot.json @@ -1865,8 +1865,8 @@ "primaryKey": false, "notNull": true }, - "probe_status": { - "name": "probe_status", + "shell_status": { + "name": "shell_status", "type": "varchar(64)", "primaryKey": false, "notNull": true @@ -2050,4 +2050,4 @@ "schemas": {}, "tables": {} } -} \ No newline at end of file +} diff --git a/apps/api/src/workload-abuse/controllers/workload-abuse.controller.spec.ts b/apps/api/src/workload-abuse/controllers/workload-abuse.controller.spec.ts index 45ad2b1973..7a79090175 100644 --- a/apps/api/src/workload-abuse/controllers/workload-abuse.controller.spec.ts +++ b/apps/api/src/workload-abuse/controllers/workload-abuse.controller.spec.ts @@ -39,17 +39,25 @@ describe(WorkloadAbuseController.name, () => { it("purges expired evidence once the sweeps settle", async () => { const { controller, probeEvidenceService } = setup(); - await controller.probeTrialDeployments({ dryRun: true }); + await controller.probeTrialDeployments({ dryRun: false }); expect(probeEvidenceService.purgeExpired).toHaveBeenCalledTimes(1); }); - it("leaves the purge to the next run when a sweep fails", async () => { + it("purges expired evidence even when a sweep fails", async () => { const { controller, probeJobService, probeEvidenceService } = setup(); probeJobService.reconcile.mockRejectedValue(new Error("db down")); await expect(controller.probeTrialDeployments({ dryRun: false })).rejects.toThrow("db down"); + expect(probeEvidenceService.purgeExpired).toHaveBeenCalledTimes(1); + }); + + it("deletes nothing on a dry run", async () => { + const { controller, probeEvidenceService } = setup(); + + await controller.probeTrialDeployments({ dryRun: true }); + expect(probeEvidenceService.purgeExpired).not.toHaveBeenCalled(); }); diff --git a/apps/api/src/workload-abuse/controllers/workload-abuse.controller.ts b/apps/api/src/workload-abuse/controllers/workload-abuse.controller.ts index 98467ac97f..b949c7651c 100644 --- a/apps/api/src/workload-abuse/controllers/workload-abuse.controller.ts +++ b/apps/api/src/workload-abuse/controllers/workload-abuse.controller.ts @@ -13,14 +13,15 @@ export class WorkloadAbuseController { private readonly probeEvidenceService: ProbeEvidenceService ) {} - /** Each sweep runs whether or not the other fails, so a probe outage does not leave stuck wipes waiting another run. */ + /** Each sweep runs whether or not the other fails, and retention runs whether or not the sweeps do, so nothing waits another run. */ async probeTrialDeployments(options: DryRunOptions): Promise { const sweeps = await Promise.allSettled([this.probeJobService.reconcile(options), this.enforcementJobService.reconcile(options)]); + + if (!options.dryRun) await this.probeEvidenceService.purgeExpired(); + const failures = sweeps.filter((sweep): sweep is PromiseRejectedResult => sweep.status === "rejected").map(sweep => sweep.reason); if (failures.length === 1) throw failures[0]; if (failures.length > 1) throw new AggregateError(failures, "Both the probe sweep and the enforcement sweep failed"); - - await this.probeEvidenceService.purgeExpired(); } } diff --git a/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.spec.ts b/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.spec.ts index 54db21df06..0f15bb2df0 100644 --- a/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.spec.ts +++ b/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.spec.ts @@ -1,33 +1,15 @@ import { describe, expect, it } from "vitest"; -import { parseProbeEvidence, withoutEvidenceSections } from "./parse-probe-evidence"; +import { parseProbeEvidence } from "./parse-probe-evidence"; describe("parseProbeEvidence", () => { - it("parses accelerator, processes, artifacts, process origins, and net shape from full probe output", () => { + it("parses accelerator, processes, artifacts, process origins, and net shape from one evidence block", () => { const raw = [ - "--loadavg", - "0.52 0.58 0.59 1/902 12345", - "--nproc", - "4", - "--procs", - "999 cpu_s=12 rss_mb=512 comm=python3 exe=/usr/bin/python3 cwd=/ app.py", - "--net", - " 2 listen=8080", - " 3 st=01 remote=18.185.10.20:443", - "--tmp", - "total 8", - "--files", - "== /tmp/app.conf", - "| key=value", - "--authorized-keys", - "--recent-exec", - "/usr/bin/python3", - "--recent-conf", - "/home/user/app.ini", "--accel", - "NVIDIA T4, 95, 15130, 15360", - "1234, python3, 15100", + "GPU-0001, NVIDIA T4, 95, 15130, 15360", + "GPU-0001, 1234, python3, 15100", "--netl", + " 2 listen=8080", " 3 0D05 140AB912:01BB 01", "--disk", "73014444032 /root/.cache/model.safetensors", @@ -50,20 +32,31 @@ describe("parseProbeEvidence", () => { }); }); + it("keeps each process on the card it ran on when the host has more than one", () => { + const features = parseProbeEvidence( + ["--accel", "GPU-0001, NVIDIA A100, 99, 20480, 24576", "GPU-0002, NVIDIA A100, 0, 4, 24576", "GPU-0001, 1234, trainer, 18000", ""].join("\n") + ); + + expect(features.accelerator).toEqual([ + { name: "NVIDIA A100", utilPct: 99, memUsedMb: 20480, memTotalMb: 24576, processes: [{ pid: 1234, name: "trainer", vramMb: 18000 }] }, + { name: "NVIDIA A100", utilPct: 0, memUsedMb: 4, memTotalMb: 24576, processes: [] } + ]); + }); + it("returns null accelerator when the tooling is absent", () => { const features = parseProbeEvidence("--accel\naccel: unavailable\n"); expect(features.accelerator).toBeNull(); }); - it("returns nulls when no evidence sections are present", () => { - const features = parseProbeEvidence("--loadavg\n0.10 0.20 0.30 1/900 1\n--nproc\n2\n"); + it("returns nulls when the evidence block is empty", () => { + const features = parseProbeEvidence(""); expect(features).toEqual({ accelerator: null, artifacts: null, processOrigins: null, netShape: null }); }); it("tolerates non-numeric vram placeholders in compute app lines", () => { - const features = parseProbeEvidence("--accel\nNVIDIA T4, 0, 0, 15360\n1234, python3, [N/A]\n"); + const features = parseProbeEvidence("--accel\nGPU-0001, NVIDIA T4, 0, 0, 15360\nGPU-0001, 1234, python3, [N/A]\n"); expect(features.accelerator).toEqual([ { name: "NVIDIA T4", utilPct: 0, memUsedMb: 0, memTotalMb: 15360, processes: [{ pid: 1234, name: "python3", vramMb: 0 }] } @@ -71,7 +64,7 @@ describe("parseProbeEvidence", () => { }); it("keeps commas in process names within one process entry", () => { - const features = parseProbeEvidence("--accel\nNVIDIA T4, 5, 100, 15360\n42, trainer, extra, part, 96\n"); + const features = parseProbeEvidence("--accel\nGPU-0001, NVIDIA T4, 5, 100, 15360\nGPU-0001, 42, trainer, extra, part, 96\n"); expect(features.accelerator).toEqual([ { name: "NVIDIA T4", utilPct: 5, memUsedMb: 100, memTotalMb: 15360, processes: [{ pid: 42, name: "trainer, extra, part", vramMb: 96 }] } @@ -89,9 +82,8 @@ describe("parseProbeEvidence", () => { it("decodes v4-mapped and native ipv6 remotes from netl lines", () => { const raw = [ - "--net", - " 1 listen=22", "--netl", + " 1 listen=22", " 1 0016 0000000000000000FFFF00000100007F:01BB 01", " 2 0EA7 F804012A8F0B170C0000000002000000:0050 01", "" @@ -131,22 +123,3 @@ describe("parseProbeEvidence", () => { expect(features.processOrigins).toEqual([{ pid: 999, ppid: 1, comm: "sh", startedAtEpochMs: 0 }]); }); }); - -describe("withoutEvidenceSections", () => { - it("strips everything from the first --accel marker onward", () => { - const legacy = ["--loadavg", "0.10", "--nproc", "2"].join("\n") + "\n"; - const evidence = ["--accel", "NVIDIA T4, 1, 2, 3", "--netl", " 1 0D05 140AB912:01BB 01", "--disk", "1 /x", "--procorig", "btime=1"].join("\n") + "\n"; - - expect(withoutEvidenceSections(legacy + evidence)).toBe(legacy); - }); - - it("returns empty output when --accel opens the output", () => { - expect(withoutEvidenceSections("--accel\nNVIDIA T4, 1, 2, 3\n")).toBe(""); - }); - - it("returns the output unchanged when no evidence markers exist", () => { - const legacyOnly = "--loadavg\n0.10\n--nproc\n2\n"; - - expect(withoutEvidenceSections(legacyOnly)).toBe(legacyOnly); - }); -}); diff --git a/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.ts b/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.ts index d1c87946bb..24fc73fdfa 100644 --- a/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.ts +++ b/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.ts @@ -7,39 +7,19 @@ export type ProbeEvidenceFeatures = { netShape: ProbeEvidenceNetShape | null; }; -const SECTION_MARKERS = [ - "--loadavg", - "--nproc", - "--procs", - "--net", - "--tmp", - "--files", - "--authorized-keys", - "--recent-exec", - "--recent-conf", - "--accel", - "--netl", - "--disk", - "--procorig" -] as const; - -export function parseProbeEvidence(rawShellOutput: string): ProbeEvidenceFeatures { - const sections = splitSections(rawShellOutput); +const SECTION_MARKERS = ["--accel", "--netl", "--disk", "--procorig"] as const; + +export function parseProbeEvidence(evidenceOutput: string): ProbeEvidenceFeatures { + const sections = splitSections(evidenceOutput); return { accelerator: parseAccelerator(sections.get("--accel")), artifacts: parseArtifacts(sections.get("--disk")), processOrigins: parseProcessOrigins(sections.get("--procorig")), - netShape: parseNetShape(sections.get("--net"), sections.get("--netl")) + netShape: parseNetShape(sections.get("--netl")) }; } -export function withoutEvidenceSections(rawShellOutput: string): string { - const boundary = rawShellOutput.indexOf("\n--accel\n"); - if (boundary !== -1) return rawShellOutput.slice(0, boundary + 1); - return rawShellOutput.startsWith("--accel\n") ? "" : rawShellOutput; -} - function splitSections(output: string): Map { const sections = new Map(); let currentLines: string[] | null = null; @@ -56,32 +36,44 @@ function splitSections(output: string): Map { return sections; } +type AcceleratorOnGpu = Omit & { gpuUuid: string }; + +/** Both accelerator queries report the gpu uuid, so a process lands on the card it actually ran on rather than on every card in the host. */ function parseAccelerator(lines: string[] | undefined): ProbeEvidenceAccelerator[] | null { if (!lines) return null; if (lines.some(line => line.trim() === "accel: unavailable")) return null; - const accelerators: Array> = []; - const processes: ProbeEvidenceAccelerator["processes"] = []; + const accelerators: AcceleratorOnGpu[] = []; + const processesByGpu = new Map(); for (const line of lines) { const fields = line.split(",").map(field => field.trim()); - if (fields.length >= 4 && !isNumeric(fields[0]) && isNumeric(fields[1]) && isNumeric(fields[2]) && isNumeric(fields[3])) { + if (fields.length >= 5 && !isNumeric(fields[1]) && isNumeric(fields[2]) && isNumeric(fields[3]) && isNumeric(fields[4])) { accelerators.push({ - name: fields[0], - utilPct: Number(fields[1]), - memUsedMb: Number(fields[2]), - memTotalMb: Number(fields[3]) + gpuUuid: fields[0], + name: fields[1], + utilPct: Number(fields[2]), + memUsedMb: Number(fields[3]), + memTotalMb: Number(fields[4]) }); - } else if (fields.length >= 3 && isNumeric(fields[0])) { + } else if (fields.length >= 4 && isNumeric(fields[1])) { + const processes = processesByGpu.get(fields[0]) ?? []; processes.push({ - pid: Number(fields[0]), - name: fields.slice(1, -1).join(", "), + pid: Number(fields[1]), + name: fields.slice(2, -1).join(", "), vramMb: toNumberOrZero(fields[fields.length - 1]) }); + processesByGpu.set(fields[0], processes); } } - return accelerators.map(accelerator => ({ ...accelerator, processes })); + return accelerators.map(accelerator => ({ + name: accelerator.name, + utilPct: accelerator.utilPct, + memUsedMb: accelerator.memUsedMb, + memTotalMb: accelerator.memTotalMb, + processes: processesByGpu.get(accelerator.gpuUuid) ?? [] + })); } function parseArtifacts(lines: string[] | undefined): ProbeEvidenceArtifact[] | null { @@ -122,15 +114,13 @@ function parseProcessOrigins(lines: string[] | undefined): ProbeEvidenceProcessO return origins; } -function parseNetShape(netLines: string[] | undefined, netlLines: string[] | undefined): ProbeEvidenceNetShape | null { +function parseNetShape(netlLines: string[] | undefined): ProbeEvidenceNetShape | null { if (!netlLines) return null; const listenPorts = new Set(); - if (netLines) { - for (const line of netLines) { - const match = line.trim().match(/^\d+\s+listen=(\d+)$/); - if (match) listenPorts.add(Number(match[1])); - } + for (const line of netlLines) { + const match = line.trim().match(/^\d+\s+listen=(\d+)$/); + if (match) listenPorts.add(Number(match[1])); } const establishedByKey = new Map(); diff --git a/apps/api/src/workload-abuse/model-schemas/workload-probe-evidence/workload-probe-evidence.schema.ts b/apps/api/src/workload-abuse/model-schemas/workload-probe-evidence/workload-probe-evidence.schema.ts index fa03581ae1..d92e57f2d3 100644 --- a/apps/api/src/workload-abuse/model-schemas/workload-probe-evidence/workload-probe-evidence.schema.ts +++ b/apps/api/src/workload-abuse/model-schemas/workload-probe-evidence/workload-probe-evidence.schema.ts @@ -31,7 +31,7 @@ export const WorkloadProbeEvidence = pgTable( dseq: varchar("dseq").notNull(), provider: text("provider").notNull(), service: varchar("service", { length: 255 }).notNull(), - probeStatus: varchar("probe_status", { length: 64 }).notNull(), + shellStatus: varchar("shell_status", { length: 64 }).notNull(), verdict: varchar("verdict", { length: 16 }).notNull(), detectionId: uuid("detection_id"), accelerator: jsonb("accelerator").$type(), diff --git a/apps/api/src/workload-abuse/services/probe-evidence/probe-evidence.service.spec.ts b/apps/api/src/workload-abuse/services/probe-evidence/probe-evidence.service.spec.ts index 75120bbd8d..2a6260cb67 100644 --- a/apps/api/src/workload-abuse/services/probe-evidence/probe-evidence.service.spec.ts +++ b/apps/api/src/workload-abuse/services/probe-evidence/probe-evidence.service.spec.ts @@ -10,7 +10,7 @@ import { ProbeEvidenceService } from "./probe-evidence.service"; import { mockConfigService } from "@test/mocks/config-service.mock"; describe(ProbeEvidenceService.name, () => { - it("records one row per shell output with parsed features", async () => { + it("records one row per probed service with parsed features", async () => { const { service, evidenceRepository } = setup(); evidenceRepository.insertMany.mockResolvedValue([]); @@ -18,14 +18,14 @@ describe(ProbeEvidenceService.name, () => { walletId: 42, dseq: "1000001", verdict: "clean", - probeStatus: "completed", - shellOutputs: [ + shellEvidence: [ { service: "web", provider: "akash1provider", - output: "--accel\nNVIDIA T4, 90, 14000, 15360\n1234, python3, 14000\n--disk\n1073741824 /root/model.bin\n" + status: "completed", + evidence: "--accel\nGPU-0001, NVIDIA T4, 90, 14000, 15360\nGPU-0001, 1234, python3, 14000\n--disk\n1073741824 /root/model.bin\n" }, - { service: "sidecar", provider: "akash1provider", output: "--loadavg\n0.10\n" } + { service: "sidecar", provider: "akash1provider", status: "output_capped", evidence: "" } ] }); @@ -35,19 +35,19 @@ describe(ProbeEvidenceService.name, () => { dseq: "1000001", provider: "akash1provider", service: "web", - probeStatus: "completed", + shellStatus: "completed", verdict: "clean", accelerator: [expect.objectContaining({ name: "NVIDIA T4", utilPct: 90 })], artifacts: [{ path: "/root/model.bin", sizeBytes: 1073741824 }] }), - expect.objectContaining({ service: "sidecar", accelerator: null, artifacts: null }) + expect.objectContaining({ service: "sidecar", shellStatus: "output_capped", accelerator: null, artifacts: null }) ]); }); - it("skips the write when there are no shell outputs", async () => { + it("skips the write when no service was reached", async () => { const { service, evidenceRepository } = setup(); - await service.recordEvidence({ walletId: 42, dseq: "1000001", verdict: "clean", probeStatus: "completed", shellOutputs: [] }); + await service.recordEvidence({ walletId: 42, dseq: "1000001", verdict: "clean", shellEvidence: [] }); expect(evidenceRepository.insertMany).not.toHaveBeenCalled(); }); @@ -60,9 +60,8 @@ describe(ProbeEvidenceService.name, () => { walletId: 42, dseq: "1000001", verdict: "hard", - probeStatus: "completed", detectionId: "detection-uuid", - shellOutputs: [{ service: "web", provider: "akash1provider", output: "--accel\nNVIDIA T4, 1, 2, 3\n" }] + shellEvidence: [{ service: "web", provider: "akash1provider", status: "completed", evidence: "--accel\nGPU-0001, NVIDIA T4, 1, 2, 3\n" }] }); expect(evidenceRepository.insertMany).toHaveBeenCalledWith([expect.objectContaining({ detectionId: "detection-uuid" })]); @@ -77,12 +76,11 @@ describe(ProbeEvidenceService.name, () => { walletId: 42, dseq: "1000001", verdict: "clean", - probeStatus: "completed", - shellOutputs: [{ service: "web", provider: "akash1provider", output: "--disk\n1 /root/x\n" }] + shellEvidence: [{ service: "web", provider: "akash1provider", status: "completed", evidence: "--disk\n1 /root/x\n" }] }) ).resolves.toBeUndefined(); - expect(instrumentation.recordEvidenceWriteFailure).toHaveBeenCalledTimes(1); + expect(instrumentation.recordEvidenceWriteFailure).toHaveBeenCalledWith("insert"); expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ event: "WORKLOAD_EVIDENCE_WRITE_FAILED", walletId: 42, dseq: "1000001" })); }); @@ -100,7 +98,7 @@ describe(ProbeEvidenceService.name, () => { await expect(service.purgeExpired()).resolves.toBeUndefined(); - expect(instrumentation.recordEvidenceWriteFailure).toHaveBeenCalledTimes(1); + expect(instrumentation.recordEvidenceWriteFailure).toHaveBeenCalledWith("purge"); expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ event: "WORKLOAD_EVIDENCE_PURGE_FAILED" })); }); diff --git a/apps/api/src/workload-abuse/services/probe-evidence/probe-evidence.service.ts b/apps/api/src/workload-abuse/services/probe-evidence/probe-evidence.service.ts index dc54a831f0..5da42b8eb4 100644 --- a/apps/api/src/workload-abuse/services/probe-evidence/probe-evidence.service.ts +++ b/apps/api/src/workload-abuse/services/probe-evidence/probe-evidence.service.ts @@ -3,11 +3,10 @@ import { inject, singleton } from "tsyringe"; import { type CreateLogger, LOGGER_FACTORY } from "@src/core"; import { parseProbeEvidence } from "@src/workload-abuse/lib/probe-evidence/parse-probe-evidence"; import { WorkloadProbeEvidenceRepository } from "@src/workload-abuse/repositories/workload-probe-evidence/workload-probe-evidence.repository"; +import type { ShellEvidence } from "@src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service"; import { WorkloadAbuseConfigService } from "@src/workload-abuse/services/workload-abuse-config/workload-abuse-config.service"; import { WorkloadAbuseInstrumentationService } from "@src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service"; -type ProbeEvidenceShellOutput = { service: string; provider: string; output: string }; - @singleton() export class ProbeEvidenceService { private readonly logger: ReturnType; @@ -21,26 +20,19 @@ export class ProbeEvidenceService { this.logger = createLogger({ context: ProbeEvidenceService.name }); } - async recordEvidence(input: { - walletId: number; - dseq: string; - verdict: string; - probeStatus: string; - detectionId?: string; - shellOutputs: ProbeEvidenceShellOutput[]; - }): Promise { - if (!input.shellOutputs.length) return; + async recordEvidence(input: { walletId: number; dseq: string; verdict: string; detectionId?: string; shellEvidence: ShellEvidence[] }): Promise { + if (!input.shellEvidence.length) return; try { await this.evidenceRepository.insertMany( - input.shellOutputs.map(shellOutput => { - const features = parseProbeEvidence(shellOutput.output); + input.shellEvidence.map(shellEvidence => { + const features = parseProbeEvidence(shellEvidence.evidence); return { walletId: input.walletId, dseq: input.dseq, - provider: shellOutput.provider, - service: shellOutput.service, - probeStatus: input.probeStatus, + provider: shellEvidence.provider, + service: shellEvidence.service, + shellStatus: shellEvidence.status, verdict: input.verdict, detectionId: input.detectionId, accelerator: features.accelerator, @@ -51,7 +43,7 @@ export class ProbeEvidenceService { }) ); } catch (error) { - this.instrumentation.recordEvidenceWriteFailure(); + this.instrumentation.recordEvidenceWriteFailure("insert"); this.logger.warn({ event: "WORKLOAD_EVIDENCE_WRITE_FAILED", error, walletId: input.walletId, dseq: input.dseq }); } } @@ -63,7 +55,7 @@ export class ProbeEvidenceService { try { await this.evidenceRepository.deleteOlderThan({ before }); } catch (error) { - this.instrumentation.recordEvidenceWriteFailure(); + this.instrumentation.recordEvidenceWriteFailure("purge"); this.logger.warn({ event: "WORKLOAD_EVIDENCE_PURGE_FAILED", error, before }); } } diff --git a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.integration.ts b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.integration.ts index 45224b9aae..0a1164ada9 100644 --- a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.integration.ts +++ b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.integration.ts @@ -17,7 +17,7 @@ import { expectJobCompleted, findJobRows, useJobWorkers } from "@test/services/j const MAX_ATTEMPTS = 30; -const ACCELERATED_SHELL_OUTPUT = "--loadavg\n0.10\n--accel\nNVIDIA A100, 95, 20480, 24576\n1234, python3, 18000"; +const ACCELERATED_EVIDENCE = "--accel\nGPU-0001, NVIDIA A100, 95, 20480, 24576\nGPU-0001, 1234, python3, 18000"; const jobWorkers = useJobWorkers(() => [container.resolve(ProbeTrialDeploymentHandler)]); @@ -75,6 +75,7 @@ describe(ProbeTrialDeploymentHandler.name, () => { expect(await findEvidence()).toMatchObject([ { verdict: "clean", + shellStatus: "completed", detectionId: null, provider: "akash1provider", service: "ssh", @@ -118,7 +119,7 @@ describe(ProbeTrialDeploymentHandler.name, () => { attempt?: number; isTrialing?: boolean; probeStatus?: ProbeReport["probeStatus"]; - shellOutputs?: ProbeReport["shellOutputs"]; + shellEvidence?: ProbeReport["shellEvidence"]; }) { const { enqueue, startWorkers } = await jobWorkers(); const detectionRepository = container.resolve(WorkloadAbuseDetectionRepository); @@ -136,7 +137,7 @@ describe(ProbeTrialDeploymentHandler.name, () => { signals: [], excerpt: "denied", leases: [], - shellOutputs: input.shellOutputs ?? [{ service: "ssh", provider: "akash1provider", output: ACCELERATED_SHELL_OUTPUT }] + shellEvidence: input.shellEvidence ?? [{ service: "ssh", provider: "akash1provider", status: "completed", evidence: ACCELERATED_EVIDENCE }] }); const probeKey = probeTrialDeploymentKeyFor({ walletId: wallet.id, dseq }); diff --git a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts index 10943b5c4f..1007b51fb0 100644 --- a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts +++ b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts @@ -61,9 +61,9 @@ describe(ProbeTrialDeploymentHandler.name, () => { expect(instrumentation.recordProbe).toHaveBeenCalledWith(expect.objectContaining({ verdict: "clean", probeStatus: "probed" })); }); - it("records evidence on a clean probe with the raw shell outputs and no detection id", async () => { - const shellOutputs = [{ service: "web", provider: "akash1provider", output: "--loadavg\n0.10" }]; - const { handler, wallet, probeEvidenceService } = setup({ report: createReport({ verdict: "clean", shellOutputs }) }); + it("records evidence on a clean probe with the collected evidence and no detection id", async () => { + const shellEvidence = [{ service: "web", provider: "akash1provider", status: "completed" as const, evidence: "--accel\naccel: unavailable" }]; + const { handler, wallet, probeEvidenceService } = setup({ report: createReport({ verdict: "clean", shellEvidence }) }); await handler.handle(PAYLOAD); @@ -71,9 +71,8 @@ describe(ProbeTrialDeploymentHandler.name, () => { walletId: wallet.id, dseq: PAYLOAD.dseq, verdict: "clean", - probeStatus: "probed", detectionId: undefined, - shellOutputs + shellEvidence }); }); @@ -94,7 +93,7 @@ describe(ProbeTrialDeploymentHandler.name, () => { }); it("records no evidence when the deployment has no live lease", async () => { - const { handler, probeEvidenceService } = setup({ report: createReport({ probeStatus: "no_live_lease", shellOutputs: [] }) }); + const { handler, probeEvidenceService } = setup({ report: createReport({ probeStatus: "no_live_lease", shellEvidence: [] }) }); await handler.handle(PAYLOAD); @@ -223,7 +222,7 @@ describe(ProbeTrialDeploymentHandler.name, () => { logStatus: "completed" } ], - shellOutputs: [{ service: "ssh", provider: "akash1provider", output: "--loadavg\n0.10" }], + shellEvidence: [{ service: "ssh", provider: "akash1provider", status: "completed", evidence: "--accel\naccel: unavailable" }], ...overrides }; } diff --git a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts index cb661e3e32..306ffd98d0 100644 --- a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts +++ b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts @@ -99,15 +99,14 @@ export class ProbeTrialDeploymentHandler implements JobHandler shellOutput.service) + services: report.shellEvidence.map(shellEvidence => shellEvidence.service) }); this.logger.info({ diff --git a/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.spec.ts b/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.spec.ts index bf4f9bffca..bb6c416f14 100644 --- a/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.spec.ts +++ b/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.spec.ts @@ -8,17 +8,19 @@ import { mock } from "vitest-mock-extended"; import { withoutFileContents } from "@src/workload-abuse/lib/evidence-scanner/evidence-scanner"; import type { ProviderStreamResult, ProviderStreamService } from "@src/workload-abuse/services/provider-stream/provider-stream.service"; import type { WorkloadAbuseConfigService } from "@src/workload-abuse/services/workload-abuse-config/workload-abuse-config.service"; -import { buildShellProbeUrl, ProviderShellProbeService, SHELL_PROBE_COLLECTORS, SHELL_PROBE_SCRIPT } from "./provider-shell-probe.service"; +import { buildShellProbeScript, buildShellProbeUrl, ProviderShellProbeService, SHELL_PROBE_COLLECTORS } from "./provider-shell-probe.service"; import { mockConfigService } from "@test/mocks/config-service.mock"; const TMP_FILE_GLOBS = "/tmp/*.json /tmp/*.conf /tmp/*.txt /tmp/*/*.json /tmp/*/*.conf"; const TARGET = { hostUri: "https://provider.example:8443", providerAddress: "akash1provider", token: "jwt", dseq: "123", gseq: 1, oseq: 2, service: "ssh box" }; +const BOUNDARY = "0f1e2d3c4b5a69788796a5b4c3d2e1f0"; +const SCRIPT = buildShellProbeScript(BOUNDARY); describe(ProviderShellProbeService.name, () => { describe("buildShellProbeUrl", () => { it("runs the collector script through sh without stdin or a tty on the first pod of the service", () => { - const url = new URL(buildShellProbeUrl(TARGET)); + const url = new URL(buildShellProbeUrl(TARGET, BOUNDARY)); expect(url.origin + url.pathname).toBe("https://provider.example:8443/lease/123/1/2/shell"); expect(url.searchParams.get("stdin")).toBe("0"); @@ -26,51 +28,57 @@ describe(ProviderShellProbeService.name, () => { expect(url.searchParams.get("podIndex")).toBe("0"); expect(url.searchParams.get("cmd0")).toBe("sh"); expect(url.searchParams.get("cmd1")).toBe("-c"); - expect(url.searchParams.get("cmd2")).toBe(SHELL_PROBE_SCRIPT); + expect(url.searchParams.get("cmd2")).toBe(SCRIPT); expect(url.searchParams.get("service")).toBe("ssh box"); }); it("prints expanded lines with printf so a dash echo cannot turn the script's own cmdline into NUL bytes", () => { - expect(SHELL_PROBE_SCRIPT).toContain("printf '%s\\n' \"${p#/proc/} cpu_s="); - expect(SHELL_PROBE_SCRIPT).toContain("printf '%s\\n' \"== $f\""); - expect(SHELL_PROBE_SCRIPT).not.toMatch(/echo "/); + expect(SCRIPT).toContain("printf '%s\\n' \"${p#/proc/} cpu_s="); + expect(SCRIPT).toContain("printf '%s\\n' \"== $f\""); + expect(SCRIPT).not.toMatch(/echo "/); }); it("marks every file body line as it reads it, so no body line can pass for a section or a file header", () => { - expect(SHELL_PROBE_SCRIPT).not.toMatch(/cat "\$f"/); - expect(SHELL_PROBE_SCRIPT.match(/printf '\| %s\\n' "\$l"/g)).toHaveLength(2); + expect(SCRIPT).not.toMatch(/cat "\$f"/); + expect(SCRIPT.match(/printf '\| %s\\n' "\$l"/g)).toHaveLength(2); }); it("reads a whole stat file before parsing it, since a workload can put a newline in its own process name", () => { - expect(SHELL_PROBE_SCRIPT).toContain('{ s=; while IFS= read -r x; do s="$s$x "; done < "$p/stat"; } 2>/dev/null; [ -n "$s" ] || continue'); + expect(SCRIPT).toContain('{ s=; while IFS= read -r x; do s="$s$x "; done < "$p/stat"; } 2>/dev/null; [ -n "$s" ] || continue'); }); it("streams the process listing line by line and bounds each cmdline read, so a hung or starved process still leaves the rest visible", () => { - expect(SHELL_PROBE_SCRIPT).toContain("T=$(command -v timeout >/dev/null 2>&1 && printf 'timeout 2')"); - expect(SHELL_PROBE_SCRIPT).toContain("c=$({ $T tr '\\0' ' ' < \"$p/cmdline\"; } 2>/dev/null)"); - expect(SHELL_PROBE_SCRIPT).toContain('n=$((n + 1)); [ "$n" -ge 150 ] && break; done'); - expect(SHELL_PROBE_SCRIPT).not.toContain("head -150"); + expect(SCRIPT).toContain("T=$(command -v timeout >/dev/null 2>&1 && printf 'timeout 2')"); + expect(SCRIPT).toContain("c=$({ $T tr '\\0' ' ' < \"$p/cmdline\"; } 2>/dev/null)"); + expect(SCRIPT).toContain('n=$((n + 1)); [ "$n" -ge 150 ] && break; done'); + expect(SCRIPT).not.toContain("head -150"); }); it("leaves its own shell and that shell's children out of the process listing by pid, not by what they run", () => { - expect(SHELL_PROBE_SCRIPT).toContain('[ "${p#/proc/}" = "$$" ] && continue'); - expect(SHELL_PROBE_SCRIPT).toContain('[ "${2:-}" = "$$" ] && continue'); + expect(SCRIPT).toContain('[ "${p#/proc/}" = "$$" ] && continue'); + expect(SCRIPT).toContain('[ "${2:-}" = "$$" ] && continue'); }); it("records cpu time and memory per process, listening ports, and files written since the container started", () => { - expect(SHELL_PROBE_SCRIPT).toContain("cpu_s=$(( (${12:-0} + ${13:-0}) / 100 )) rss_mb=$(( ${22:-0} * 4 / 1024 ))"); - expect(SHELL_PROBE_SCRIPT).toContain("printf 'listen=%d\\n' \"0x${la#*:}\""); - expect(SHELL_PROBE_SCRIPT).toContain( + expect(SCRIPT).toContain("cpu_s=$(( (${12:-0} + ${13:-0}) / 100 )) rss_mb=$(( ${22:-0} * 4 / 1024 ))"); + expect(SCRIPT).toContain("printf 'listen=%d\\n' \"0x${la#*:}\""); + expect(SCRIPT).toContain( "echo '--recent-exec'; find / \\( -path /proc -o -path /sys -o -path /dev -o -name node_modules \\) -prune -o -type f -perm -100 -newer /proc/1 -print" ); - expect(SHELL_PROBE_SCRIPT).toContain( + expect(SCRIPT).toContain( "echo '--recent-conf'; find / \\( -path /proc -o -path /sys -o -path /dev -o -path /etc -o -path /tmp -o -name node_modules \\) -prune" ); }); + it("closes the collected sections with the boundary it was given, before the sections only the evidence table reads", () => { + expect(SCRIPT).toContain(`echo '--evidence ${BOUNDARY}'`); + expect(SCRIPT.indexOf(`--evidence ${BOUNDARY}`)).toBeLessThan(SCRIPT.indexOf("--accel")); + expect(SCRIPT.indexOf("--recent-conf")).toBeLessThan(SCRIPT.indexOf(`--evidence ${BOUNDARY}`)); + }); + it("only reads the container and never changes, fetches or runs anything in it", () => { - expect(SHELL_PROBE_SCRIPT).not.toMatch(/\b(curl|wget|chmod|chown|kill|rm|mv|cp|apt|apk|pip|npm)\b/); - expect(SHELL_PROBE_SCRIPT).not.toMatch(/[^2<]>\s*\/(?!dev\/null)/); + expect(SCRIPT).not.toMatch(/\b(curl|wget|chmod|chown|kill|rm|mv|cp|apt|apk|pip|npm)\b/); + expect(SCRIPT).not.toMatch(/[^2<]>\s*\/(?!dev\/null)/); }); }); @@ -192,8 +200,8 @@ describe(ProviderShellProbeService.name, () => { [ "#!/bin/sh", 'case "$1" in', - " --query-gpu*) printf 'NVIDIA A100, 95, 20480, 24576\\n' ;;", - " --query-compute-apps*) printf '1234, python3, 18000\\n' ;;", + " --query-gpu*) printf 'GPU-0001, NVIDIA A100, 95, 20480, 24576\\n' ;;", + " --query-compute-apps*) printf 'GPU-0001, 1234, python3, 18000\\n' ;;", "esac" ].join("\n"), { mode: 0o755 } @@ -201,7 +209,7 @@ describe(ProviderShellProbeService.name, () => { const reported = runCollector("--accel", collector => collector, { ...process.env, PATH: `${binDirectory}:${process.env.PATH}` }); - expect(reported).toEqual(["--accel", "NVIDIA A100, 95, 20480, 24576", "1234, python3, 18000"]); + expect(reported).toEqual(["--accel", "GPU-0001, NVIDIA A100, 95, 20480, 24576", "GPU-0001, 1234, python3, 18000"]); }); it("reports the accelerator as unavailable when the container has no nvidia-smi", () => { @@ -227,8 +235,8 @@ describe(ProviderShellProbeService.name, () => { ] }); - expect(reported).toEqual(expect.arrayContaining(["2 9C4E 140AB912:0D05 01", "1 A0F0 0000000000000000FFFF000009030E34:0D05 01"])); - expect(reported).toHaveLength(3); + expect(reported).toEqual(expect.arrayContaining(["2 9C4E 140AB912:0D05 01", "1 A0F0 0000000000000000FFFF000009030E34:0D05 01", "1 listen=8080"])); + expect(reported).toHaveLength(4); }); it("leaves out a socket that is neither established nor connecting", () => { @@ -284,7 +292,39 @@ describe(ProviderShellProbeService.name, () => { const result = await service.run(TARGET); - expect(result).toEqual({ status: "completed", output: "--loadavg\n1.00\nwarn", exitCode: 0 }); + expect(result).toEqual({ status: "completed", output: "--loadavg\n1.00\nwarn", evidence: "", exitCode: 0 }); + }); + + it("splits the collected output from the evidence block at the boundary the run issued", async () => { + const { service } = setup(boundary => streamOf(`--loadavg\n1.00\n--evidence ${boundary}\n--accel\naccel: unavailable`)); + + expect(await service.run(TARGET)).toEqual({ + status: "completed", + output: "--loadavg\n1.00\n", + evidence: "--accel\naccel: unavailable", + exitCode: 0 + }); + }); + + it("keeps a collected line that reads like a boundary with the collected output", async () => { + const { service } = setup(streamOf("--tmp\n--evidence 0f1e2d3c4b5a69788796a5b4c3d2e1f0\n--accel\nGPU-0001, NVIDIA A100, 95, 20480, 24576")); + + const result = await service.run(TARGET); + + expect(result.evidence).toBe(""); + expect(result.output).toContain("--accel"); + }); + + it("issues a boundary the workload has not seen before on every run", async () => { + const { service, providerStreamService } = setup(streamOf("--loadavg\n1.00")); + + await service.run(TARGET); + await service.run(TARGET); + + const [first, second] = providerStreamService.collect.mock.calls.map(([input]) => boundaryOf(input.url)); + + expect(first).toMatch(/^[0-9a-f]{32}$/); + expect(second).not.toBe(first); }); it("reports the shell as unavailable when the provider fails to exec or nothing comes back", async () => { @@ -379,9 +419,19 @@ describe(ProviderShellProbeService.name, () => { return runCollector("--procorig", collector => collector.replaceAll("/proc/", `${directory}/`)); } - function setup(result: ProviderStreamResult) { + function streamOf(payload: string): ProviderStreamResult { + return { status: "completed", exitCode: 0, frames: [{ kind: "shell", stream: "stdout", payload }] }; + } + + function boundaryOf(url: string): string { + const script = new URL(url).searchParams.get("cmd2") ?? ""; + + return script.match(/--evidence ([0-9a-f]+)/)?.[1] ?? ""; + } + + function setup(result: ProviderStreamResult | ((boundary: string) => ProviderStreamResult)) { const providerStreamService = mock(); - providerStreamService.collect.mockResolvedValue(result); + providerStreamService.collect.mockImplementation(async input => (typeof result === "function" ? result(boundaryOf(input.url)) : result)); const config = mockConfigService({ WORKLOAD_ABUSE_PROBE_IDLE_TIMEOUT_MS: 5_000, WORKLOAD_ABUSE_PROBE_HARD_TIMEOUT_MS: 30_000, diff --git a/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.ts b/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.ts index 1c2c0d091e..567406ada1 100644 --- a/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.ts +++ b/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.ts @@ -1,3 +1,4 @@ +import { randomBytes } from "node:crypto"; import { singleton } from "tsyringe"; import { FILE_BODY_PREFIX } from "@src/workload-abuse/lib/evidence-scanner/evidence-scanner"; @@ -5,7 +6,7 @@ import { ProviderStreamService, type ProviderStreamStatus } from "@src/workload- import { WorkloadAbuseConfigService } from "@src/workload-abuse/services/workload-abuse-config/workload-abuse-config.service"; /** Collect-only and printf-only: argv is visible to the workload through /proc, and dash's echo expands the `\0` in this script's own cmdline into a NUL byte. */ -export const SHELL_PROBE_COLLECTORS = [ +const WORKLOAD_COLLECTORS = [ "echo '--loadavg'; cat /proc/loadavg 2>/dev/null", "echo '--nproc'; nproc 2>/dev/null || grep -c ^processor /proc/cpuinfo 2>/dev/null", 'echo \'--procs\'; T=$(command -v timeout >/dev/null 2>&1 && printf \'timeout 2\'); n=0; for p in /proc/[0-9]*; do [ "${p#/proc/}" = "$$" ] && continue; { s=; while IFS= read -r x; do s="$s$x "; done < "$p/stat"; } 2>/dev/null; [ -n "$s" ] || continue; m=${s#*(}; m=${m%)*}; set -- ${s##*) }; [ "${2:-}" = "$$" ] && continue; c=$({ $T tr \'\\0\' \' \' < "$p/cmdline"; } 2>/dev/null); [ -n "$c" ] || continue; printf \'%s\\n\' "${p#/proc/} cpu_s=$(( (${12:-0} + ${13:-0}) / 100 )) rss_mb=$(( ${22:-0} * 4 / 1024 )) comm=$m exe=$(readlink "$p/exe" 2>/dev/null) cwd=$(readlink "$p/cwd" 2>/dev/null) cmd=$c"; n=$((n + 1)); [ "$n" -ge 150 ] && break; done', @@ -14,18 +15,28 @@ export const SHELL_PROBE_COLLECTORS = [ `echo '--files'; for f in /tmp/*.json /tmp/*.conf /tmp/*.txt /tmp/*/*.json /tmp/*/*.conf; do [ -f "$f" ] && [ "$(wc -c < "$f")" -lt 16384 ] && printf '%s\\n' "== $f" && while IFS= read -r l || [ -n "$l" ]; do printf '${FILE_BODY_PREFIX}%s\\n' "$l"; l=; done < "$f"; done 2>/dev/null | head -400`, "echo '--authorized-keys'; cat /root/.ssh/authorized_keys /home/*/.ssh/authorized_keys 2>/dev/null | sort -u | head -10", "echo '--recent-exec'; find / \\( -path /proc -o -path /sys -o -path /dev -o -name node_modules \\) -prune -o -type f -perm -100 -newer /proc/1 -print 2>/dev/null | head -60 | while read -r f; do ls -la \"$f\" 2>/dev/null; done", - `echo '--recent-conf'; find / \\( -path /proc -o -path /sys -o -path /dev -o -path /etc -o -path /tmp -o -name node_modules \\) -prune -o -type f -newer /proc/1 \\( -name '*.json' -o -name '*.conf' -o -name '*.ini' -o -name '*.txt' \\) -size -16k -print 2>/dev/null | head -20 | while read -r f; do printf '%s\\n' "== $f"; [ -r "$f" ] || continue; while IFS= read -r l || [ -n "$l" ]; do printf '${FILE_BODY_PREFIX}%s\\n' "$l"; l=; done < "$f"; done | head -400`, - "echo '--accel'; T=$(command -v timeout >/dev/null 2>&1 && printf 'timeout 2'); command -v nvidia-smi >/dev/null 2>&1 && { $T nvidia-smi --query-gpu=name,utilization.gpu,memory.used,memory.total --format=csv,noheader,nounits 2>/dev/null; $T nvidia-smi --query-compute-apps=pid,process_name,used_gpu_memory --format=csv,noheader,nounits 2>/dev/null; } || printf 'accel: unavailable\\n'", - 'echo \'--netl\'; for f in /proc/net/tcp /proc/net/tcp6; do [ -r "$f" ] || continue; while read -r sl la ra st rest; do case $st in 01|02) ;; *) continue;; esac; printf \'%s %s %s\\n\' "${la#*:}" "$ra" "$st"; done < "$f"; done 2>/dev/null | sort | uniq -c | head -80', + `echo '--recent-conf'; find / \\( -path /proc -o -path /sys -o -path /dev -o -path /etc -o -path /tmp -o -name node_modules \\) -prune -o -type f -newer /proc/1 \\( -name '*.json' -o -name '*.conf' -o -name '*.ini' -o -name '*.txt' \\) -size -16k -print 2>/dev/null | head -20 | while read -r f; do printf '%s\\n' "== $f"; [ -r "$f" ] || continue; while IFS= read -r l || [ -n "$l" ]; do printf '${FILE_BODY_PREFIX}%s\\n' "$l"; l=; done < "$f"; done | head -400` +]; + +const EVIDENCE_COLLECTORS = [ + "echo '--accel'; T=$(command -v timeout >/dev/null 2>&1 && printf 'timeout 2'); command -v nvidia-smi >/dev/null 2>&1 && { $T nvidia-smi --query-gpu=uuid,name,utilization.gpu,memory.used,memory.total --format=csv,noheader,nounits 2>/dev/null; $T nvidia-smi --query-compute-apps=gpu_uuid,pid,process_name,used_gpu_memory --format=csv,noheader,nounits 2>/dev/null; } || printf 'accel: unavailable\\n'", + 'echo \'--netl\'; for f in /proc/net/tcp /proc/net/tcp6; do [ -r "$f" ] || continue; while read -r sl la ra st rest; do case $st in 0A) printf \'listen=%d\\n\' "0x${la#*:}"; continue;; 01|02) ;; *) continue;; esac; printf \'%s %s %s\\n\' "${la#*:}" "$ra" "$st"; done < "$f"; done 2>/dev/null | sort | uniq -c | head -80', 'echo \'--disk\'; find / -xdev -type f -size +64M 2>/dev/null | head -200 | while read -r f; do printf \'%s %s\\n\' "$(wc -c < "$f" 2>/dev/null)" "$f"; done 2>/dev/null | sort -rn | head -40', 'echo \'--procorig\'; b=$(grep ^btime /proc/stat 2>/dev/null); printf \'btime=%s\\n\' "${b#btime }"; T=$(command -v timeout >/dev/null 2>&1 && printf \'timeout 2\'); n=0; for p in /proc/[0-9]*; do [ "${p#/proc/}" = "$$" ] && continue; { s=; while IFS= read -r x; do s="$s$x "; done < "$p/stat"; } 2>/dev/null; [ -n "$s" ] || continue; m=${s#*(}; m=${m%)*}; set -- ${s##*) }; [ "${2:-}" = "$$" ] && continue; c=$({ $T tr \'\\0\' \' \' < "$p/cmdline"; } 2>/dev/null); [ -n "$c" ] || continue; printf \'%s\\n\' "${p#/proc/} ppid=${2:-0} starttime=${20:-0} comm=$m"; n=$((n + 1)); [ "$n" -ge 150 ] && break; done' ]; -export const SHELL_PROBE_SCRIPT = SHELL_PROBE_COLLECTORS.join("; "); +export const SHELL_PROBE_COLLECTORS = [...WORKLOAD_COLLECTORS, ...EVIDENCE_COLLECTORS]; + +const EVIDENCE_BOUNDARY_PREFIX = "--evidence "; + +/** The workload writes part of what the collectors print, so the line that closes their output carries a token it cannot predict. */ +export function buildShellProbeScript(boundary: string): string { + return [...WORKLOAD_COLLECTORS, `echo '${EVIDENCE_BOUNDARY_PREFIX}${boundary}'`, ...EVIDENCE_COLLECTORS].join("; "); +} export type ShellProbeStatus = ProviderStreamStatus | "shell_unavailable"; -export type ShellProbeResult = { status: ShellProbeStatus; output: string; exitCode?: number }; +export type ShellProbeResult = { status: ShellProbeStatus; output: string; evidence: string; exitCode?: number }; export type ShellProbeTarget = { hostUri: string; @@ -37,8 +48,8 @@ export type ShellProbeTarget = { service: string; }; -export function buildShellProbeUrl(target: ShellProbeTarget): string { - const command = ["sh", "-c", SHELL_PROBE_SCRIPT].map((part, index) => `cmd${index}=${encodeURIComponent(part)}`).join("&"); +export function buildShellProbeUrl(target: ShellProbeTarget, boundary: string): string { + const command = ["sh", "-c", buildShellProbeScript(boundary)].map((part, index) => `cmd${index}=${encodeURIComponent(part)}`).join("&"); return `${target.hostUri}/lease/${target.dseq}/${target.gseq}/${target.oseq}/shell?stdin=0&tty=0&podIndex=0&${command}&service=${encodeURIComponent(target.service)}`; } @@ -51,8 +62,9 @@ export class ProviderShellProbeService { ) {} async run(target: ShellProbeTarget): Promise { + const boundary = randomBytes(16).toString("hex"); const result = await this.providerStreamService.collect({ - url: buildShellProbeUrl(target), + url: buildShellProbeUrl(target, boundary), providerAddress: target.providerAddress, token: target.token, idleTimeoutMs: this.config.get("WORKLOAD_ABUSE_PROBE_IDLE_TIMEOUT_MS"), @@ -67,6 +79,17 @@ export class ProviderShellProbeService { const failed = result.frames.some(frame => frame.kind === "shell" && frame.stream === "failure"); const status: ShellProbeStatus = failed || (result.status === "completed" && output.length === 0) ? "shell_unavailable" : result.status; - return { status, output, exitCode: result.exitCode }; + return { status, ...splitAtEvidenceBoundary(output, boundary), exitCode: result.exitCode }; } } + +function splitAtEvidenceBoundary(output: string, boundary: string): { output: string; evidence: string } { + const lines = output.split("\n"); + const boundaryIndex = lines.indexOf(`${EVIDENCE_BOUNDARY_PREFIX}${boundary}`); + + if (boundaryIndex === -1) return { output, evidence: "" }; + + const collected = lines.slice(0, boundaryIndex); + + return { output: collected.length ? `${collected.join("\n")}\n` : "", evidence: lines.slice(boundaryIndex + 1).join("\n") }; +} diff --git a/apps/api/src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service.spec.ts b/apps/api/src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service.spec.ts index 02beda39e2..82b6eb5309 100644 --- a/apps/api/src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service.spec.ts +++ b/apps/api/src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service.spec.ts @@ -31,7 +31,7 @@ describe(TrialWorkloadProbeService.name, () => { const report = await service.probe({ wallet, dseq: DSEQ }); - expect(report).toEqual({ verdict: "clean", signals: [], excerpt: "", probeStatus: "no_live_lease", leases: [], shellOutputs: [] }); + expect(report).toEqual({ verdict: "clean", signals: [], excerpt: "", probeStatus: "no_live_lease", leases: [], shellEvidence: [] }); expect(providerService.toProviderAuth).not.toHaveBeenCalled(); }); @@ -177,30 +177,30 @@ describe(TrialWorkloadProbeService.name, () => { expect(shellProbeService.run).not.toHaveBeenCalled(); }); - it("keeps the raw shell output out of the scanned sources while recording it in shellOutputs", async () => { - const shellOutput = [ - "--loadavg", - "0.10 0.20 0.30", + it("records the evidence block the probe split off and scans only what came before it", async () => { + const evidence = [ "--accel", - "NVIDIA T4, 95, 14000, 15360", - "1234, stratum+tcp://pool.example:3333, 14000", + "GPU-0001, NVIDIA T4, 95, 14000, 15360", + "GPU-0001, 1234, stratum+tcp://pool.example:3333, 14000", "--disk", "1073741824 /tmp/stratum+tcp://pool.example:3333", "--procorig", "btime=1740000000", "1234 ppid=1 starttime=650000 comm=stratum+tcp://pool.example:3333" ].join("\n"); - const { service, wallet } = setup({ leases: [createRpcLease()], services: { web: 1 }, shell: { status: "completed", output: shellOutput } }); + const { service, wallet } = setup({ + leases: [createRpcLease()], + services: { web: 1 }, + shell: { status: "completed", output: "--loadavg\n0.10 0.20 0.30\n", evidence } + }); const report = await service.probe({ wallet, dseq: DSEQ }); expect(report.verdict).toBe("clean"); expect(report.signals).toEqual([]); expect(report.excerpt).not.toContain("--accel"); - expect(report.excerpt).not.toContain("--disk"); - expect(report.excerpt).not.toContain("--procorig"); expect(report.excerpt).not.toContain("stratum+tcp"); - expect(report.shellOutputs).toEqual([{ service: "web", provider: PROVIDER, output: shellOutput }]); + expect(report.shellEvidence).toEqual([{ service: "web", provider: PROVIDER, status: "completed", evidence }]); }); function createRpcLease(overrides: Partial = {}): RpcLease { @@ -214,7 +214,7 @@ describe(TrialWorkloadProbeService.name, () => { provider?: Provider | null; services?: Record; sdl?: string | null; - shell?: ShellProbeResult; + shell?: { status: ShellProbeResult["status"]; output: string; evidence?: string }; logs?: string[]; }) { const wallet = { ...createUserWallet({ isTrialing: true }), address: "akash1owner" }; @@ -237,7 +237,7 @@ describe(TrialWorkloadProbeService.name, () => { const deploymentSettingRepository = mock(); deploymentSettingRepository.findOneBy.mockResolvedValue(mock({ sdl: input.sdl ?? null })); const shellProbeService = mock(); - shellProbeService.run.mockResolvedValue(input.shell ?? { status: "completed", output: "--loadavg\n0.10" }); + shellProbeService.run.mockResolvedValue({ evidence: "", ...(input.shell ?? { status: "completed", output: "--loadavg\n0.10" }) }); const logTailService = mock(); logTailService.collect.mockResolvedValue({ status: "completed", lines: input.logs ?? [] }); const config = mockConfigService({ diff --git a/apps/api/src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service.ts b/apps/api/src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service.ts index 8c63c4eb92..31afb1c732 100644 --- a/apps/api/src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service.ts +++ b/apps/api/src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service.ts @@ -15,7 +15,6 @@ import { toVerdict, type WorkloadVerdict } from "@src/workload-abuse/lib/evidence-scanner/evidence-scanner"; -import { withoutEvidenceSections } from "@src/workload-abuse/lib/probe-evidence/parse-probe-evidence"; import { ProviderLogTailService } from "@src/workload-abuse/services/provider-log-tail/provider-log-tail.service"; import { ProviderShellProbeService, type ShellProbeStatus } from "@src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service"; import { WorkloadAbuseConfigService } from "@src/workload-abuse/services/workload-abuse-config/workload-abuse-config.service"; @@ -38,9 +37,11 @@ export type ProbeReport = { excerpt: string; probeStatus: ProbeStatus; leases: ProbedLease[]; - shellOutputs: Array<{ service: string; provider: string; output: string }>; + shellEvidence: ShellEvidence[]; }; +export type ShellEvidence = { service: string; provider: string; status: ShellProbeStatus; evidence: string }; + const PROVIDER_SCOPES = ["status", "logs", "shell"] as const; /** The probed provider reports its own service list, so a hostile one must not be able to stretch a run past this many shell sessions. */ const MAX_PROBED_SERVICES_PER_LEASE = 8; @@ -71,11 +72,11 @@ export class TrialWorkloadProbeService { const leases = await this.#findLiveLeases(input.wallet.address, input.dseq); if (leases.length === 0) { - return { verdict: "clean", signals: [], excerpt: "", probeStatus: "no_live_lease", leases: [], shellOutputs: [] }; + return { verdict: "clean", signals: [], excerpt: "", probeStatus: "no_live_lease", leases: [], shellEvidence: [] }; } const sources: EvidenceSource[] = []; - const shellOutputs: ProbeReport["shellOutputs"] = []; + const shellEvidence: ShellEvidence[] = []; const setting = await this.deploymentSettingRepository.findOneBy({ userId: input.wallet.userId, dseq: input.dseq }); if (setting?.sdl) sources.push({ kind: "sdl", text: setting.sdl }); @@ -94,7 +95,7 @@ export class TrialWorkloadProbeService { } for (const lease of leasesToProbe) { - const probed = await this.#probeLease(input.wallet, lease, sources, shellOutputs); + const probed = await this.#probeLease(input.wallet, lease, sources, shellEvidence); statuses.push(probed.status); if (probed.lease) probedLeases.push(probed.lease); } @@ -107,7 +108,7 @@ export class TrialWorkloadProbeService { excerpt: buildExcerpt(sources, signals), probeStatus: statuses.includes("probed") ? "probed" : statuses[0] ?? "stream_failed", leases: probedLeases, - shellOutputs + shellEvidence }; } @@ -121,7 +122,7 @@ export class TrialWorkloadProbeService { wallet: WalletInitialized, lease: RpcLease, sources: EvidenceSource[], - shellOutputs: ProbeReport["shellOutputs"] + shellEvidence: ShellEvidence[] ): Promise<{ status: ProbeStatus; lease?: ProbedLease }> { const { provider: providerAddress, dseq, gseq, oseq } = lease.lease.id; const provider = await this.providerRepository.findActiveByAddress(providerAddress); @@ -156,9 +157,9 @@ export class TrialWorkloadProbeService { for (const service of probedServices) { const shell = await this.shellProbeService.run({ ...target, service }); probedLease.shellStatuses.push(shell.status); - if (shell.output) { - shellOutputs.push({ service, provider: providerAddress, output: shell.output }); - sources.push({ kind: "shell", service, text: withoutEvidenceSections(shell.output) }); + if (shell.output) sources.push({ kind: "shell", service, text: shell.output }); + if (shell.output || shell.evidence) { + shellEvidence.push({ service, provider: providerAddress, status: shell.status, evidence: shell.evidence }); } } diff --git a/apps/api/src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service.ts b/apps/api/src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service.ts index d6943ee530..8de0d3b7d6 100644 --- a/apps/api/src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service.ts +++ b/apps/api/src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service.ts @@ -6,6 +6,8 @@ import type { WorkloadVerdict } from "@src/workload-abuse/lib/evidence-scanner/e export type DomainBlockResult = "blocked" | "raced" | "skipped" | "dry_run" | "failed" | "sibling_limit_reached"; +export type EvidenceWriteOperation = "insert" | "purge"; + @singleton() export class WorkloadAbuseInstrumentationService { private readonly meter: Meter; @@ -34,7 +36,7 @@ export class WorkloadAbuseInstrumentationService { description: "Email domain auto-block outcomes, by result and (on a skip) reason" }); this.evidenceWriteFailures = metricsService.createCounter(this.meter, "workload_abuse_evidence_write_failures_total", { - description: "Probe evidence writes that failed to persist" + description: "Probe evidence statements that failed to persist, by operation" }); } @@ -58,7 +60,7 @@ export class WorkloadAbuseInstrumentationService { this.domainBlocks.add(1, reason ? { result, reason } : { result }); } - recordEvidenceWriteFailure(): void { - this.evidenceWriteFailures.add(1); + recordEvidenceWriteFailure(operation: EvidenceWriteOperation): void { + this.evidenceWriteFailures.add(1, { operation }); } } From 72cdafe2f6b68063147711fbc81b18cd2af760cb Mon Sep 17 00:00:00 2001 From: Maxime Beauchamp <15185355+baktun14@users.noreply.github.com> Date: Sun, 20 Sep 2026 16:18:35 +0400 Subject: [PATCH 3/5] fix(deployment): keep the socket state on recorded connections The collector reports live sockets and half open ones, and the parser was storing both under a name that claimed they were live. Each connection now carries the state the kernel reported, so a connection attempt in progress reads as one. The split between collected output and evidence takes the last boundary rather than the first, so a repeat of it can only widen what the scanner reads. --- .../parse-probe-evidence.spec.ts | 23 ++++++++++++++----- .../probe-evidence/parse-probe-evidence.ts | 16 ++++++++----- .../workload-probe-evidence.schema.ts | 4 +++- .../provider-shell-probe.service.spec.ts | 9 ++++++++ .../provider-shell-probe.service.ts | 3 ++- 5 files changed, 41 insertions(+), 14 deletions(-) diff --git a/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.spec.ts b/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.spec.ts index 0f15bb2df0..2626634080 100644 --- a/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.spec.ts +++ b/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.spec.ts @@ -28,7 +28,7 @@ describe("parseProbeEvidence", () => { expect(features.processOrigins).toEqual([{ pid: 1234, ppid: 1, comm: "python3", startedAtEpochMs: 1740006500000 }]); expect(features.netShape).toEqual({ listenPorts: [8080], - established: [{ localPort: 3333, remoteIp: "18.185.10.20", remotePort: 443, count: 3 }] + connections: [{ localPort: 3333, remoteIp: "18.185.10.20", remotePort: 443, count: 3, state: "established" }] }); }); @@ -92,9 +92,9 @@ describe("parseProbeEvidence", () => { const features = parseProbeEvidence(raw); expect(features.netShape?.listenPorts).toEqual([22]); - expect(features.netShape?.established).toEqual([ - { localPort: 22, remoteIp: "127.0.0.1", remotePort: 443, count: 1 }, - { localPort: 3751, remoteIp: "2a01:04f8:0c17:0b8f:0000:0000:0000:0002", remotePort: 80, count: 2 } + expect(features.netShape?.connections).toEqual([ + { localPort: 22, remoteIp: "127.0.0.1", remotePort: 443, count: 1, state: "established" }, + { localPort: 3751, remoteIp: "2a01:04f8:0c17:0b8f:0000:0000:0000:0002", remotePort: 80, count: 2, state: "established" } ]); }); @@ -103,7 +103,18 @@ describe("parseProbeEvidence", () => { const features = parseProbeEvidence(raw); - expect(features.netShape?.established).toEqual([{ localPort: 3333, remoteIp: "18.185.10.20", remotePort: 443, count: 5 }]); + expect(features.netShape?.connections).toEqual([{ localPort: 3333, remoteIp: "18.185.10.20", remotePort: 443, count: 5, state: "established" }]); + }); + + it("keeps a half open socket apart from a live one", () => { + const raw = ["--netl", " 1 0D05 140AB912:01BB 01", " 1 0D06 140AB912:01BB 02", ""].join("\n"); + + const features = parseProbeEvidence(raw); + + expect(features.netShape?.connections).toEqual([ + { localPort: 3333, remoteIp: "18.185.10.20", remotePort: 443, count: 1, state: "established" }, + { localPort: 3334, remoteIp: "18.185.10.20", remotePort: 443, count: 1, state: "connecting" } + ]); }); it("ignores malformed section lines instead of throwing", () => { @@ -114,7 +125,7 @@ describe("parseProbeEvidence", () => { expect(features.accelerator).toEqual([]); expect(features.artifacts).toEqual([]); expect(features.processOrigins).toEqual([]); - expect(features.netShape).toEqual({ listenPorts: [], established: [] }); + expect(features.netShape).toEqual({ listenPorts: [], connections: [] }); }); it("reports zero startedAtEpochMs when the btime header is missing", () => { diff --git a/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.ts b/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.ts index 24fc73fdfa..040e5f45cb 100644 --- a/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.ts +++ b/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.ts @@ -7,6 +7,9 @@ export type ProbeEvidenceFeatures = { netShape: ProbeEvidenceNetShape | null; }; +/** The kernel numbers a live socket 01 and a half open one 02, and the collector reports both. */ +const ESTABLISHED_SOCKET_STATE = "01"; + const SECTION_MARKERS = ["--accel", "--netl", "--disk", "--procorig"] as const; export function parseProbeEvidence(evidenceOutput: string): ProbeEvidenceFeatures { @@ -123,23 +126,24 @@ function parseNetShape(netlLines: string[] | undefined): ProbeEvidenceNetShape | if (match) listenPorts.add(Number(match[1])); } - const establishedByKey = new Map(); + const connectionsByKey = new Map(); for (const line of netlLines) { - const match = line.trim().match(/^(\d+)\s+([0-9A-Fa-f]+)\s+([0-9A-Fa-f]+):([0-9A-Fa-f]+)\s+\d+$/); + const match = line.trim().match(/^(\d+)\s+([0-9A-Fa-f]+)\s+([0-9A-Fa-f]+):([0-9A-Fa-f]+)\s+(\d+)$/); if (!match) continue; const localPort = parseInt(match[2], 16); const remoteIp = decodeHexIp(match[3]); const remotePort = parseInt(match[4], 16); if (!remoteIp) continue; - const key = `${localPort}|${remoteIp}|${remotePort}`; - const existing = establishedByKey.get(key); + const state = match[5] === ESTABLISHED_SOCKET_STATE ? "established" : "connecting"; + const key = `${localPort}|${remoteIp}|${remotePort}|${state}`; + const existing = connectionsByKey.get(key); if (existing) existing.count += Number(match[1]); - else establishedByKey.set(key, { localPort, remoteIp, remotePort, count: Number(match[1]) }); + else connectionsByKey.set(key, { localPort, remoteIp, remotePort, count: Number(match[1]), state }); } return { listenPorts: [...listenPorts].sort((a, b) => a - b), - established: [...establishedByKey.values()] + connections: [...connectionsByKey.values()] }; } diff --git a/apps/api/src/workload-abuse/model-schemas/workload-probe-evidence/workload-probe-evidence.schema.ts b/apps/api/src/workload-abuse/model-schemas/workload-probe-evidence/workload-probe-evidence.schema.ts index d92e57f2d3..f6f83734cc 100644 --- a/apps/api/src/workload-abuse/model-schemas/workload-probe-evidence/workload-probe-evidence.schema.ts +++ b/apps/api/src/workload-abuse/model-schemas/workload-probe-evidence/workload-probe-evidence.schema.ts @@ -13,9 +13,11 @@ export type ProbeEvidenceArtifact = { path: string; sizeBytes: number }; export type ProbeEvidenceProcessOrigin = { pid: number; ppid: number; comm: string; startedAtEpochMs: number }; +export type ProbeEvidenceConnectionState = "established" | "connecting"; + export type ProbeEvidenceNetShape = { listenPorts: number[]; - established: Array<{ localPort: number; remoteIp: string; remotePort: number; count: number }>; + connections: Array<{ localPort: number; remoteIp: string; remotePort: number; count: number; state: ProbeEvidenceConnectionState }>; }; export type ProbeEvidenceBehaviouralFinding = { signal: string; detail: Record }; diff --git a/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.spec.ts b/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.spec.ts index bb6c416f14..ea81dcc936 100644 --- a/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.spec.ts +++ b/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.spec.ts @@ -315,6 +315,15 @@ describe(ProviderShellProbeService.name, () => { expect(result.output).toContain("--accel"); }); + it("cuts at the last boundary, so a repeated one cannot hide collected output from the scanner", async () => { + const { service } = setup(boundary => streamOf(`--tmp\n--evidence ${boundary}\ntotal 8\n--evidence ${boundary}\n--accel\naccel: unavailable`)); + + const result = await service.run(TARGET); + + expect(result.output).toContain("total 8"); + expect(result.evidence).toBe("--accel\naccel: unavailable"); + }); + it("issues a boundary the workload has not seen before on every run", async () => { const { service, providerStreamService } = setup(streamOf("--loadavg\n1.00")); diff --git a/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.ts b/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.ts index 567406ada1..da9f14c3a6 100644 --- a/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.ts +++ b/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.ts @@ -83,9 +83,10 @@ export class ProviderShellProbeService { } } +/** Cutting at the last match means a repeat of the boundary can only widen what the scanner reads, never hide part of it. */ function splitAtEvidenceBoundary(output: string, boundary: string): { output: string; evidence: string } { const lines = output.split("\n"); - const boundaryIndex = lines.indexOf(`${EVIDENCE_BOUNDARY_PREFIX}${boundary}`); + const boundaryIndex = lines.lastIndexOf(`${EVIDENCE_BOUNDARY_PREFIX}${boundary}`); if (boundaryIndex === -1) return { output, evidence: "" }; From 42865de5feb6b54038d7fcb99ebdcd849ca4aee1 Mon Sep 17 00:00:00 2001 From: Maxime Beauchamp <15185355+baktun14@users.noreply.github.com> Date: Sun, 20 Sep 2026 16:55:31 +0400 Subject: [PATCH 4/5] fix(deployment): keep an accelerator whose driver reports a field as unavailable nvidia-smi answers some fields with a placeholder rather than a number on passthrough and virtualized cards. The parser was dropping the whole card on one of those, and with it every process attributed to that card, which is exactly the evidence the row exists for. --- .../lib/probe-evidence/parse-probe-evidence.spec.ts | 8 ++++++++ .../lib/probe-evidence/parse-probe-evidence.ts | 12 ++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.spec.ts b/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.spec.ts index 2626634080..203531a8f6 100644 --- a/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.spec.ts +++ b/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.spec.ts @@ -43,6 +43,14 @@ describe("parseProbeEvidence", () => { ]); }); + it("keeps a card and its processes when the driver reports a field as unavailable", () => { + const features = parseProbeEvidence("--accel\nGPU-0001, NVIDIA A100, [N/A], 20480, 24576\nGPU-0001, 1234, trainer, 18000\n"); + + expect(features.accelerator).toEqual([ + { name: "NVIDIA A100", utilPct: 0, memUsedMb: 20480, memTotalMb: 24576, processes: [{ pid: 1234, name: "trainer", vramMb: 18000 }] } + ]); + }); + it("returns null accelerator when the tooling is absent", () => { const features = parseProbeEvidence("--accel\naccel: unavailable\n"); diff --git a/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.ts b/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.ts index 040e5f45cb..ec54faacc0 100644 --- a/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.ts +++ b/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.ts @@ -41,7 +41,7 @@ function splitSections(output: string): Map { type AcceleratorOnGpu = Omit & { gpuUuid: string }; -/** Both accelerator queries report the gpu uuid, so a process lands on the card it actually ran on rather than on every card in the host. */ +/** Both accelerator queries report the gpu uuid, so a process lands on the card it actually ran on rather than on every card in the host, and a card whose driver reports a field as unavailable is kept with what it did report. */ function parseAccelerator(lines: string[] | undefined): ProbeEvidenceAccelerator[] | null { if (!lines) return null; if (lines.some(line => line.trim() === "accel: unavailable")) return null; @@ -51,13 +51,13 @@ function parseAccelerator(lines: string[] | undefined): ProbeEvidenceAccelerator for (const line of lines) { const fields = line.split(",").map(field => field.trim()); - if (fields.length >= 5 && !isNumeric(fields[1]) && isNumeric(fields[2]) && isNumeric(fields[3]) && isNumeric(fields[4])) { + if (fields.length >= 5 && !isNumeric(fields[1])) { accelerators.push({ gpuUuid: fields[0], - name: fields[1], - utilPct: Number(fields[2]), - memUsedMb: Number(fields[3]), - memTotalMb: Number(fields[4]) + name: fields.slice(1, -3).join(", "), + utilPct: toNumberOrZero(fields[fields.length - 3]), + memUsedMb: toNumberOrZero(fields[fields.length - 2]), + memTotalMb: toNumberOrZero(fields[fields.length - 1]) }); } else if (fields.length >= 4 && isNumeric(fields[1])) { const processes = processesByGpu.get(fields[0]) ?? []; From 22030a50b03247d776b1586c16940511658cebea Mon Sep 17 00:00:00 2001 From: Maxime Beauchamp <15185355+baktun14@users.noreply.github.com> Date: Sun, 20 Sep 2026 17:19:02 +0400 Subject: [PATCH 5/5] fix(deployment): log recorded evidence only when rows were written recordEvidence reports the rows it wrote, and the probe logs what was stored rather than what it tried to store, so a run that reached no service or failed its write no longer leaves a line claiming evidence exists. --- .../probe-evidence.service.spec.ts | 2 +- .../probe-evidence/probe-evidence.service.ts | 18 ++++++++++++++---- .../probe-trial-deployment.handler.spec.ts | 12 ++++++++++++ .../probe-trial-deployment.handler.ts | 16 +++++++++------- 4 files changed, 36 insertions(+), 12 deletions(-) diff --git a/apps/api/src/workload-abuse/services/probe-evidence/probe-evidence.service.spec.ts b/apps/api/src/workload-abuse/services/probe-evidence/probe-evidence.service.spec.ts index 2a6260cb67..be5e213d6e 100644 --- a/apps/api/src/workload-abuse/services/probe-evidence/probe-evidence.service.spec.ts +++ b/apps/api/src/workload-abuse/services/probe-evidence/probe-evidence.service.spec.ts @@ -78,7 +78,7 @@ describe(ProbeEvidenceService.name, () => { verdict: "clean", shellEvidence: [{ service: "web", provider: "akash1provider", status: "completed", evidence: "--disk\n1 /root/x\n" }] }) - ).resolves.toBeUndefined(); + ).resolves.toEqual([]); expect(instrumentation.recordEvidenceWriteFailure).toHaveBeenCalledWith("insert"); expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ event: "WORKLOAD_EVIDENCE_WRITE_FAILED", walletId: 42, dseq: "1000001" })); diff --git a/apps/api/src/workload-abuse/services/probe-evidence/probe-evidence.service.ts b/apps/api/src/workload-abuse/services/probe-evidence/probe-evidence.service.ts index 5da42b8eb4..c268079dab 100644 --- a/apps/api/src/workload-abuse/services/probe-evidence/probe-evidence.service.ts +++ b/apps/api/src/workload-abuse/services/probe-evidence/probe-evidence.service.ts @@ -2,7 +2,10 @@ import { inject, singleton } from "tsyringe"; import { type CreateLogger, LOGGER_FACTORY } from "@src/core"; import { parseProbeEvidence } from "@src/workload-abuse/lib/probe-evidence/parse-probe-evidence"; -import { WorkloadProbeEvidenceRepository } from "@src/workload-abuse/repositories/workload-probe-evidence/workload-probe-evidence.repository"; +import { + type WorkloadProbeEvidenceOutput, + WorkloadProbeEvidenceRepository +} from "@src/workload-abuse/repositories/workload-probe-evidence/workload-probe-evidence.repository"; import type { ShellEvidence } from "@src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service"; import { WorkloadAbuseConfigService } from "@src/workload-abuse/services/workload-abuse-config/workload-abuse-config.service"; import { WorkloadAbuseInstrumentationService } from "@src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service"; @@ -20,11 +23,17 @@ export class ProbeEvidenceService { this.logger = createLogger({ context: ProbeEvidenceService.name }); } - async recordEvidence(input: { walletId: number; dseq: string; verdict: string; detectionId?: string; shellEvidence: ShellEvidence[] }): Promise { - if (!input.shellEvidence.length) return; + async recordEvidence(input: { + walletId: number; + dseq: string; + verdict: string; + detectionId?: string; + shellEvidence: ShellEvidence[]; + }): Promise { + if (!input.shellEvidence.length) return []; try { - await this.evidenceRepository.insertMany( + return await this.evidenceRepository.insertMany( input.shellEvidence.map(shellEvidence => { const features = parseProbeEvidence(shellEvidence.evidence); return { @@ -45,6 +54,7 @@ export class ProbeEvidenceService { } catch (error) { this.instrumentation.recordEvidenceWriteFailure("insert"); this.logger.warn({ event: "WORKLOAD_EVIDENCE_WRITE_FAILED", error, walletId: input.walletId, dseq: input.dseq }); + return []; } } diff --git a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts index 1007b51fb0..d4a0554a7a 100644 --- a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts +++ b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts @@ -7,6 +7,7 @@ import type { WorkloadAbuseDetectionOutput, WorkloadAbuseDetectionRepository } from "@src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository"; +import type { WorkloadProbeEvidenceOutput } from "@src/workload-abuse/repositories/workload-probe-evidence/workload-probe-evidence.repository"; import { EnforceTrialAbuse } from "@src/workload-abuse/services/enforce-trial-abuse/enforce-trial-abuse.handler"; import type { ProbeEvidenceService } from "@src/workload-abuse/services/probe-evidence/probe-evidence.service"; import type { ProbeReport, TrialWorkloadProbeService } from "@src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service"; @@ -92,6 +93,14 @@ describe(ProbeTrialDeploymentHandler.name, () => { expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ event: "TRIAL_WORKLOAD_EVIDENCE_RECORDED", services: ["ssh"] })); }); + it("says nothing about recorded evidence when the probe reached no service", async () => { + const { handler, logger } = setup({ report: createReport({ verdict: "clean", shellEvidence: [] }) }); + + await handler.handle(PAYLOAD); + + expect(logger.info).not.toHaveBeenCalledWith(expect.objectContaining({ event: "TRIAL_WORKLOAD_EVIDENCE_RECORDED" })); + }); + it("records no evidence when the deployment has no live lease", async () => { const { handler, probeEvidenceService } = setup({ report: createReport({ probeStatus: "no_live_lease", shellEvidence: [] }) }); @@ -274,6 +283,9 @@ describe(ProbeTrialDeploymentHandler.name, () => { detectionRepository.create.mockResolvedValue(mock({ id: "detection-1" })); detectionRepository.findOneBy.mockResolvedValue(input.existingDetection ? mock({ id: "detection-0" }) : undefined); const probeEvidenceService = mock(); + probeEvidenceService.recordEvidence.mockImplementation(async ({ shellEvidence }) => + shellEvidence.map(entry => mock({ service: entry.service })) + ); const instrumentation = mock(); const config = mockConfigService({ WORKLOAD_ABUSE_PROBE_ENABLED: input.enabled ?? true, diff --git a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts index 306ffd98d0..3fb280a694 100644 --- a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts +++ b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts @@ -95,19 +95,21 @@ export class ProbeTrialDeploymentHandler implements JobHandler shellEvidence.service) - }); + if (evidenceRows.length) { + this.logger.info({ + event: "TRIAL_WORKLOAD_EVIDENCE_RECORDED", + ...context, + userId: wallet.userId, + services: evidenceRows.map(row => row.service) + }); + } this.logger.info({ event: "TRIAL_WORKLOAD_PROBED",