diff --git a/apps/api/drizzle/0058_workload_probe_evidence.sql b/apps/api/drizzle/0058_workload_probe_evidence.sql new file mode 100644 index 0000000000..c2ba83b868 --- /dev/null +++ b/apps/api/drizzle/0058_workload_probe_evidence.sql @@ -0,0 +1,19 @@ +CREATE TABLE "workload_probe_evidence" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4() NOT NULL, + "wallet_id" integer NOT NULL, + "dseq" varchar NOT NULL, + "provider" text NOT NULL, + "service" varchar(255) NOT NULL, + "shell_status" varchar(64) NOT NULL, + "verdict" varchar(16) NOT NULL, + "detection_id" uuid, + "accelerator" jsonb, + "artifacts" jsonb, + "process_origins" jsonb, + "net_shape" jsonb, + "behavioural_findings" jsonb, + "created_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE INDEX "workload_probe_evidence_wallet_dseq_created_idx" ON "workload_probe_evidence" USING btree ("wallet_id","dseq","created_at");--> statement-breakpoint +CREATE INDEX "workload_probe_evidence_created_idx" ON "workload_probe_evidence" USING btree ("created_at"); \ No newline at end of file diff --git a/apps/api/drizzle/meta/0058_snapshot.json b/apps/api/drizzle/meta/0058_snapshot.json new file mode 100644 index 0000000000..db5f655c94 --- /dev/null +++ b/apps/api/drizzle/meta/0058_snapshot.json @@ -0,0 +1,2053 @@ +{ + "id": "63bba39e-51aa-48a3-a83b-b019a98795f2", + "prevId": "929d72bc-0bb2-40b5-ae01-1289fda18bfa", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.user_wallets": { + "name": "user_wallets", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "varchar", + "primaryKey": false, + "notNull": false + }, + "deployment_allowance": { + "name": "deployment_allowance", + "type": "numeric(20, 2)", + "primaryKey": false, + "notNull": true, + "default": "'0.00'" + }, + "fee_allowance": { + "name": "fee_allowance", + "type": "numeric(20, 2)", + "primaryKey": false, + "notNull": true, + "default": "'0.00'" + }, + "trial": { + "name": "trial", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": true + }, + "activated_at": { + "name": "activated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "credits_low_notified_at": { + "name": "credits_low_notified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "credits_sufficient_since": { + "name": "credits_sufficient_since", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "credits_low_since": { + "name": "credits_low_since", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "abuse_locked_at": { + "name": "abuse_locked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "abuse_locked_reason": { + "name": "abuse_locked_reason", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "user_wallets_user_id_userSetting_id_fk": { + "name": "user_wallets_user_id_userSetting_id_fk", + "tableFrom": "user_wallets", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_wallets_user_id_unique": { + "name": "user_wallets_user_id_unique", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + }, + "user_wallets_address_unique": { + "name": "user_wallets_address_unique", + "nullsNotDistinct": false, + "columns": [ + "address" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.payment_methods": { + "name": "payment_methods", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "fingerprint": { + "name": "fingerprint", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "payment_method_id": { + "name": "payment_method_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "is_validated": { + "name": "is_validated", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "payment_methods_fingerprint_payment_method_id_unique": { + "name": "payment_methods_fingerprint_payment_method_id_unique", + "columns": [ + { + "expression": "fingerprint", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "payment_method_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "payment_methods_user_id_is_default_unique": { + "name": "payment_methods_user_id_is_default_unique", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_default", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"payment_methods\".\"is_default\" = true", + "concurrently": false, + "method": "btree", + "with": {} + }, + "payment_methods_fingerprint_idx": { + "name": "payment_methods_fingerprint_idx", + "columns": [ + { + "expression": "fingerprint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "payment_methods_user_id_idx": { + "name": "payment_methods_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "payment_methods_user_id_is_validated_idx": { + "name": "payment_methods_user_id_is_validated_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_validated", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "payment_methods_user_id_fingerprint_payment_method_id_idx": { + "name": "payment_methods_user_id_fingerprint_payment_method_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "fingerprint", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "payment_method_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "payment_methods_user_id_userSetting_id_fk": { + "name": "payment_methods_user_id_userSetting_id_fk", + "tableFrom": "payment_methods", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.stripe_transactions": { + "name": "stripe_transactions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "stripe_transaction_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "stripe_transaction_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'created'" + }, + "amount": { + "name": "amount", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "amount_refunded": { + "name": "amount_refunded", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "bonus_amount": { + "name": "bonus_amount", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "currency": { + "name": "currency", + "type": "varchar(3)", + "primaryKey": false, + "notNull": true, + "default": "'usd'" + }, + "stripe_payment_intent_id": { + "name": "stripe_payment_intent_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "stripe_charge_id": { + "name": "stripe_charge_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "stripe_coupon_id": { + "name": "stripe_coupon_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "stripe_promotion_code_id": { + "name": "stripe_promotion_code_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "stripe_invoice_id": { + "name": "stripe_invoice_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "stripe_idempotency_key": { + "name": "stripe_idempotency_key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "payment_method_type": { + "name": "payment_method_type", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "card_brand": { + "name": "card_brand", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "card_last4": { + "name": "card_last4", + "type": "varchar(4)", + "primaryKey": false, + "notNull": false + }, + "receipt_url": { + "name": "receipt_url", + "type": "varchar(2048)", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "varchar(500)", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "varchar(1000)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "stripe_transactions_stripe_invoice_id_unique": { + "name": "stripe_transactions_stripe_invoice_id_unique", + "columns": [ + { + "expression": "stripe_invoice_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"stripe_transactions\".\"stripe_invoice_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_stripe_idempotency_key_unique": { + "name": "stripe_transactions_stripe_idempotency_key_unique", + "columns": [ + { + "expression": "stripe_idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"stripe_transactions\".\"stripe_idempotency_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_user_id_idx": { + "name": "stripe_transactions_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_stripe_payment_intent_id_idx": { + "name": "stripe_transactions_stripe_payment_intent_id_idx", + "columns": [ + { + "expression": "stripe_payment_intent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_stripe_charge_id_idx": { + "name": "stripe_transactions_stripe_charge_id_idx", + "columns": [ + { + "expression": "stripe_charge_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_stripe_coupon_id_idx": { + "name": "stripe_transactions_stripe_coupon_id_idx", + "columns": [ + { + "expression": "stripe_coupon_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_stripe_promotion_code_id_idx": { + "name": "stripe_transactions_stripe_promotion_code_id_idx", + "columns": [ + { + "expression": "stripe_promotion_code_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_status_idx": { + "name": "stripe_transactions_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_created_at_idx": { + "name": "stripe_transactions_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_user_id_created_at_idx": { + "name": "stripe_transactions_user_id_created_at_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "stripe_transactions_user_id_userSetting_id_fk": { + "name": "stripe_transactions_user_id_userSetting_id_fk", + "tableFrom": "stripe_transactions", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.wallet_settings": { + "name": "wallet_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "wallet_id": { + "name": "wallet_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "auto_reload_enabled": { + "name": "auto_reload_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "auto_reload_mode": { + "name": "auto_reload_mode", + "type": "auto_reload_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'prediction'" + }, + "auto_reload_threshold": { + "name": "auto_reload_threshold", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 2000 + }, + "auto_reload_amount": { + "name": "auto_reload_amount", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 10000 + }, + "last_auto_charge_at": { + "name": "last_auto_charge_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "auto_reload_failure_count": { + "name": "auto_reload_failure_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "auto_reload_paused_at": { + "name": "auto_reload_paused_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "wallet_settings_user_id_idx": { + "name": "wallet_settings_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "wallet_settings_wallet_id_user_wallets_id_fk": { + "name": "wallet_settings_wallet_id_user_wallets_id_fk", + "tableFrom": "wallet_settings", + "tableTo": "user_wallets", + "columnsFrom": [ + "wallet_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "wallet_settings_user_id_userSetting_id_fk": { + "name": "wallet_settings_user_id_userSetting_id_fk", + "tableFrom": "wallet_settings", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "wallet_settings_wallet_id_unique": { + "name": "wallet_settings_wallet_id_unique", + "nullsNotDistinct": false, + "columns": [ + "wallet_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.userSetting": { + "name": "userSetting", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "userId": { + "name": "userId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "username": { + "name": "username", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "emailVerified": { + "name": "emailVerified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "stripeCustomerId": { + "name": "stripeCustomerId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "bio": { + "name": "bio", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "subscribedToNewsletter": { + "name": "subscribedToNewsletter", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "youtubeUsername": { + "name": "youtubeUsername", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "twitterUsername": { + "name": "twitterUsername", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "githubUsername": { + "name": "githubUsername", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "last_active_at": { + "name": "last_active_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "last_ip": { + "name": "last_ip", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "last_user_agent": { + "name": "last_user_agent", + "type": "varchar(500)", + "primaryKey": false, + "notNull": false + }, + "last_fingerprint": { + "name": "last_fingerprint", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "onboardingSkippedAt": { + "name": "onboardingSkippedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "fair_use_policy_accepted_at": { + "name": "fair_use_policy_accepted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "userSetting_userId_unique": { + "name": "userSetting_userId_unique", + "nullsNotDistinct": false, + "columns": [ + "userId" + ] + }, + "userSetting_username_unique": { + "name": "userSetting_username_unique", + "nullsNotDistinct": false, + "columns": [ + "username" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.template": { + "name": "template", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "userId": { + "name": "userId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "copiedFromId": { + "name": "copiedFromId", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "isPublic": { + "name": "isPublic", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "cpu": { + "name": "cpu", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "ram": { + "name": "ram", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "storage": { + "name": "storage", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "sdl": { + "name": "sdl", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "template_userId_idx": { + "name": "template_userId_idx", + "columns": [ + { + "expression": "userId", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.templateFavorite": { + "name": "templateFavorite", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "userId": { + "name": "userId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "templateId": { + "name": "templateId", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "addedDate": { + "name": "addedDate", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "templateFavorite_userId_templateId_unique": { + "name": "templateFavorite_userId_templateId_unique", + "columns": [ + { + "expression": "userId", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "templateId", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "templateFavorite_templateId_template_id_fk": { + "name": "templateFavorite_templateId_template_id_fk", + "tableFrom": "templateFavorite", + "tableTo": "template", + "columnsFrom": [ + "templateId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.deployment_settings": { + "name": "deployment_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "dseq": { + "name": "dseq", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "auto_top_up_enabled": { + "name": "auto_top_up_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "closed": { + "name": "closed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "last_funded_at": { + "name": "last_funded_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "runtime_limit_hours": { + "name": "runtime_limit_hours", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "sdl": { + "name": "sdl", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sealed_secrets": { + "name": "sealed_secrets", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "manifest_version": { + "name": "manifest_version", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "runtime_ends_at": { + "name": "runtime_ends_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "runtime_ending_notified_for": { + "name": "runtime_ending_notified_for", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "provider_unreachable_notified_for": { + "name": "provider_unreachable_notified_for", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "id_auto_top_up_enabled_closed_idx": { + "name": "id_auto_top_up_enabled_closed_idx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "auto_top_up_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "closed", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_id_id_sealed_secrets_idx": { + "name": "user_id_id_sealed_secrets_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"deployment_settings\".\"sealed_secrets\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "deployment_settings_user_id_userSetting_id_fk": { + "name": "deployment_settings_user_id_userSetting_id_fk", + "tableFrom": "deployment_settings", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "dseq_user_id_idx": { + "name": "dseq_user_id_idx", + "nullsNotDistinct": false, + "columns": [ + "dseq", + "user_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.api_keys": { + "name": "api_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "hashed_key": { + "name": "hashed_key", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "key_format": { + "name": "key_format", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "api_keys_user_id_userSetting_id_fk": { + "name": "api_keys_user_id_userSetting_id_fk", + "tableFrom": "api_keys", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "api_keys_hashed_key_unique": { + "name": "api_keys_hashed_key_unique", + "nullsNotDistinct": false, + "columns": [ + "hashed_key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.email_verification_codes": { + "name": "email_verification_codes", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "code": { + "name": "code", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "email_verification_codes_user_id_idx": { + "name": "email_verification_codes_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "email_verification_codes_user_id_userSetting_id_fk": { + "name": "email_verification_codes_user_id_userSetting_id_fk", + "tableFrom": "email_verification_codes", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_keys": { + "name": "data_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "wrapped_key": { + "name": "wrapped_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "wrapped_by_kid": { + "name": "wrapped_by_kid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "retired_at": { + "name": "retired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_keys_active_user_id_idx": { + "name": "data_keys_active_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_keys\".\"retired_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_keys_wrapped_by_kid_idx": { + "name": "data_keys_wrapped_by_kid_idx", + "columns": [ + { + "expression": "wrapped_by_kid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_keys_user_id_userSetting_id_fk": { + "name": "data_keys_user_id_userSetting_id_fk", + "tableFrom": "data_keys", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.blocked_email_domains": { + "name": "blocked_email_domains", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "domain": { + "name": "domain", + "type": "varchar(253)", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "blocked_email_domain_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'blocked'" + }, + "source": { + "name": "source", + "type": "blocked_email_domain_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'manual'" + }, + "reason": { + "name": "reason", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "triggered_by_user_id": { + "name": "triggered_by_user_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "blocked_email_domains_domain_unique": { + "name": "blocked_email_domains_domain_unique", + "columns": [ + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "blocked_email_domains_triggered_by_user_id_userSetting_id_fk": { + "name": "blocked_email_domains_triggered_by_user_id_userSetting_id_fk", + "tableFrom": "blocked_email_domains", + "tableTo": "userSetting", + "columnsFrom": [ + "triggered_by_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "blocked_email_domains_domain_normalized": { + "name": "blocked_email_domains_domain_normalized", + "value": "\"blocked_email_domains\".\"domain\" ~ '^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?(?:\\.[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)+$'" + } + }, + "isRLSEnabled": false + }, + "public.workload_abuse_detections": { + "name": "workload_abuse_detections", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "wallet_id": { + "name": "wallet_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "dseq": { + "name": "dseq", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "verdict": { + "name": "verdict", + "type": "workload_abuse_verdict", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "probe_status": { + "name": "probe_status", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "signals": { + "name": "signals", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "evidence_excerpt": { + "name": "evidence_excerpt", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "workload_abuse_action", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'detected'" + }, + "enforcement_error": { + "name": "enforcement_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workload_abuse_detections_user_id_idx": { + "name": "workload_abuse_detections_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workload_abuse_detections_dseq_idx": { + "name": "workload_abuse_detections_dseq_idx", + "columns": [ + { + "expression": "dseq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workload_abuse_detections_user_id_userSetting_id_fk": { + "name": "workload_abuse_detections_user_id_userSetting_id_fk", + "tableFrom": "workload_abuse_detections", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workload_probe_evidence": { + "name": "workload_probe_evidence", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "wallet_id": { + "name": "wallet_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "dseq": { + "name": "dseq", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service": { + "name": "service", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "shell_status": { + "name": "shell_status", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "verdict": { + "name": "verdict", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true + }, + "detection_id": { + "name": "detection_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "accelerator": { + "name": "accelerator", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "process_origins": { + "name": "process_origins", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "net_shape": { + "name": "net_shape", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "behavioural_findings": { + "name": "behavioural_findings", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workload_probe_evidence_wallet_dseq_created_idx": { + "name": "workload_probe_evidence_wallet_dseq_created_idx", + "columns": [ + { + "expression": "wallet_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dseq", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workload_probe_evidence_created_idx": { + "name": "workload_probe_evidence_created_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.stripe_transaction_status": { + "name": "stripe_transaction_status", + "schema": "public", + "values": [ + "created", + "pending", + "requires_action", + "succeeded", + "failed", + "refunded", + "canceled" + ] + }, + "public.stripe_transaction_type": { + "name": "stripe_transaction_type", + "schema": "public", + "values": [ + "payment_intent", + "coupon_claim", + "manual_credit" + ] + }, + "public.auto_reload_mode": { + "name": "auto_reload_mode", + "schema": "public", + "values": [ + "prediction", + "threshold" + ] + }, + "public.blocked_email_domain_source": { + "name": "blocked_email_domain_source", + "schema": "public", + "values": [ + "auto", + "manual" + ] + }, + "public.blocked_email_domain_status": { + "name": "blocked_email_domain_status", + "schema": "public", + "values": [ + "blocked", + "allowed" + ] + }, + "public.workload_abuse_action": { + "name": "workload_abuse_action", + "schema": "public", + "values": [ + "detected", + "enforcing", + "enforced", + "enforcement_failed" + ] + }, + "public.workload_abuse_verdict": { + "name": "workload_abuse_verdict", + "schema": "public", + "values": [ + "hard", + "soft", + "proxy" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/apps/api/drizzle/meta/_journal.json b/apps/api/drizzle/meta/_journal.json index 3484520e3b..4eb242cd4a 100644 --- a/apps/api/drizzle/meta/_journal.json +++ b/apps/api/drizzle/meta/_journal.json @@ -407,6 +407,13 @@ "when": 1789465677452, "tag": "0057_deployment_settings_user_sealed_secrets_index", "breakpoints": true + }, + { + "idx": 58, + "version": "7", + "when": 1789839246549, + "tag": "0058_workload_probe_evidence", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/api/src/workload-abuse/config/env.config.ts b/apps/api/src/workload-abuse/config/env.config.ts index 1f5dc1a664..1f7b9529a7 100644 --- a/apps/api/src/workload-abuse/config/env.config.ts +++ b/apps/api/src/workload-abuse/config/env.config.ts @@ -97,7 +97,9 @@ export const envSchema = z.object({ /** Bounds the damage of a domain match that turns out to be too broad. */ WORKLOAD_ABUSE_DOMAIN_BLOCK_MAX_SIBLINGS: z.number({ coerce: true }).int().positive().default(200), /** A domain with an account older than this predates the attack, so it is somebody's real domain. */ - WORKLOAD_ABUSE_DOMAIN_BLOCK_MIN_ACCOUNT_AGE_DAYS: z.number({ coerce: true }).int().positive().default(30) + WORKLOAD_ABUSE_DOMAIN_BLOCK_MIN_ACCOUNT_AGE_DAYS: z.number({ coerce: true }).int().positive().default(30), + /** Evidence rows feed the behavioural replay, so they must outlive its 30-day window with margin. */ + WORKLOAD_ABUSE_EVIDENCE_RETENTION_DAYS: z.number({ coerce: true }).int().positive().default(90) }); export type WorkloadAbuseConfig = z.infer; diff --git a/apps/api/src/workload-abuse/controllers/workload-abuse.controller.spec.ts b/apps/api/src/workload-abuse/controllers/workload-abuse.controller.spec.ts index d2c6b40e47..7a79090175 100644 --- a/apps/api/src/workload-abuse/controllers/workload-abuse.controller.spec.ts +++ b/apps/api/src/workload-abuse/controllers/workload-abuse.controller.spec.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { mock } from "vitest-mock-extended"; +import type { ProbeEvidenceService } from "@src/workload-abuse/services/probe-evidence/probe-evidence.service"; import type { TrialAbuseEnforcementJobService } from "@src/workload-abuse/services/trial-abuse-enforcement-job/trial-abuse-enforcement-job.service"; import type { TrialWorkloadProbeJobService } from "@src/workload-abuse/services/trial-workload-probe-job/trial-workload-probe-job.service"; import { WorkloadAbuseController } from "./workload-abuse.controller"; @@ -35,13 +36,39 @@ describe(WorkloadAbuseController.name, () => { await expect(controller.probeTrialDeployments({ dryRun: false })).rejects.toMatchObject({ errors: [probeError, enforcementError] }); }); + it("purges expired evidence once the sweeps settle", async () => { + const { controller, probeEvidenceService } = setup(); + + await controller.probeTrialDeployments({ dryRun: false }); + + expect(probeEvidenceService.purgeExpired).toHaveBeenCalledTimes(1); + }); + + it("purges expired evidence even when a sweep fails", async () => { + const { controller, probeJobService, probeEvidenceService } = setup(); + probeJobService.reconcile.mockRejectedValue(new Error("db down")); + + await expect(controller.probeTrialDeployments({ dryRun: false })).rejects.toThrow("db down"); + + expect(probeEvidenceService.purgeExpired).toHaveBeenCalledTimes(1); + }); + + it("deletes nothing on a dry run", async () => { + const { controller, probeEvidenceService } = setup(); + + await controller.probeTrialDeployments({ dryRun: true }); + + expect(probeEvidenceService.purgeExpired).not.toHaveBeenCalled(); + }); + function setup() { const probeJobService = mock(); probeJobService.reconcile.mockResolvedValue(); const enforcementJobService = mock(); enforcementJobService.reconcile.mockResolvedValue(); - const controller = new WorkloadAbuseController(probeJobService, enforcementJobService); + const probeEvidenceService = mock(); + const controller = new WorkloadAbuseController(probeJobService, enforcementJobService, probeEvidenceService); - return { controller, probeJobService, enforcementJobService }; + return { controller, probeJobService, enforcementJobService, probeEvidenceService }; } }); diff --git a/apps/api/src/workload-abuse/controllers/workload-abuse.controller.ts b/apps/api/src/workload-abuse/controllers/workload-abuse.controller.ts index 425615cd3d..b949c7651c 100644 --- a/apps/api/src/workload-abuse/controllers/workload-abuse.controller.ts +++ b/apps/api/src/workload-abuse/controllers/workload-abuse.controller.ts @@ -1,6 +1,7 @@ import { singleton } from "tsyringe"; import type { DryRunOptions } from "@src/core/types/console"; +import { ProbeEvidenceService } from "@src/workload-abuse/services/probe-evidence/probe-evidence.service"; import { TrialAbuseEnforcementJobService } from "@src/workload-abuse/services/trial-abuse-enforcement-job/trial-abuse-enforcement-job.service"; import { TrialWorkloadProbeJobService } from "@src/workload-abuse/services/trial-workload-probe-job/trial-workload-probe-job.service"; @@ -8,12 +9,16 @@ import { TrialWorkloadProbeJobService } from "@src/workload-abuse/services/trial export class WorkloadAbuseController { constructor( private readonly probeJobService: TrialWorkloadProbeJobService, - private readonly enforcementJobService: TrialAbuseEnforcementJobService + private readonly enforcementJobService: TrialAbuseEnforcementJobService, + private readonly probeEvidenceService: ProbeEvidenceService ) {} - /** Each sweep runs whether or not the other fails, so a probe outage does not leave stuck wipes waiting another run. */ + /** Each sweep runs whether or not the other fails, and retention runs whether or not the sweeps do, so nothing waits another run. */ async probeTrialDeployments(options: DryRunOptions): Promise { const sweeps = await Promise.allSettled([this.probeJobService.reconcile(options), this.enforcementJobService.reconcile(options)]); + + if (!options.dryRun) await this.probeEvidenceService.purgeExpired(); + const failures = sweeps.filter((sweep): sweep is PromiseRejectedResult => sweep.status === "rejected").map(sweep => sweep.reason); if (failures.length === 1) throw failures[0]; diff --git a/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.spec.ts b/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.spec.ts new file mode 100644 index 0000000000..203531a8f6 --- /dev/null +++ b/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.spec.ts @@ -0,0 +1,144 @@ +import { describe, expect, it } from "vitest"; + +import { parseProbeEvidence } from "./parse-probe-evidence"; + +describe("parseProbeEvidence", () => { + it("parses accelerator, processes, artifacts, process origins, and net shape from one evidence block", () => { + const raw = [ + "--accel", + "GPU-0001, NVIDIA T4, 95, 15130, 15360", + "GPU-0001, 1234, python3, 15100", + "--netl", + " 2 listen=8080", + " 3 0D05 140AB912:01BB 01", + "--disk", + "73014444032 /root/.cache/model.safetensors", + "--procorig", + "btime=1740000000", + "1234 ppid=1 starttime=650000 comm=python3", + "" + ].join("\n"); + + const features = parseProbeEvidence(raw); + + expect(features.accelerator).toEqual([ + { name: "NVIDIA T4", utilPct: 95, memUsedMb: 15130, memTotalMb: 15360, processes: [{ pid: 1234, name: "python3", vramMb: 15100 }] } + ]); + expect(features.artifacts).toEqual([{ path: "/root/.cache/model.safetensors", sizeBytes: 73014444032 }]); + expect(features.processOrigins).toEqual([{ pid: 1234, ppid: 1, comm: "python3", startedAtEpochMs: 1740006500000 }]); + expect(features.netShape).toEqual({ + listenPorts: [8080], + connections: [{ localPort: 3333, remoteIp: "18.185.10.20", remotePort: 443, count: 3, state: "established" }] + }); + }); + + it("keeps each process on the card it ran on when the host has more than one", () => { + const features = parseProbeEvidence( + ["--accel", "GPU-0001, NVIDIA A100, 99, 20480, 24576", "GPU-0002, NVIDIA A100, 0, 4, 24576", "GPU-0001, 1234, trainer, 18000", ""].join("\n") + ); + + expect(features.accelerator).toEqual([ + { name: "NVIDIA A100", utilPct: 99, memUsedMb: 20480, memTotalMb: 24576, processes: [{ pid: 1234, name: "trainer", vramMb: 18000 }] }, + { name: "NVIDIA A100", utilPct: 0, memUsedMb: 4, memTotalMb: 24576, processes: [] } + ]); + }); + + it("keeps a card and its processes when the driver reports a field as unavailable", () => { + const features = parseProbeEvidence("--accel\nGPU-0001, NVIDIA A100, [N/A], 20480, 24576\nGPU-0001, 1234, trainer, 18000\n"); + + expect(features.accelerator).toEqual([ + { name: "NVIDIA A100", utilPct: 0, memUsedMb: 20480, memTotalMb: 24576, processes: [{ pid: 1234, name: "trainer", vramMb: 18000 }] } + ]); + }); + + it("returns null accelerator when the tooling is absent", () => { + const features = parseProbeEvidence("--accel\naccel: unavailable\n"); + + expect(features.accelerator).toBeNull(); + }); + + it("returns nulls when the evidence block is empty", () => { + const features = parseProbeEvidence(""); + + expect(features).toEqual({ accelerator: null, artifacts: null, processOrigins: null, netShape: null }); + }); + + it("tolerates non-numeric vram placeholders in compute app lines", () => { + const features = parseProbeEvidence("--accel\nGPU-0001, NVIDIA T4, 0, 0, 15360\nGPU-0001, 1234, python3, [N/A]\n"); + + expect(features.accelerator).toEqual([ + { name: "NVIDIA T4", utilPct: 0, memUsedMb: 0, memTotalMb: 15360, processes: [{ pid: 1234, name: "python3", vramMb: 0 }] } + ]); + }); + + it("keeps commas in process names within one process entry", () => { + const features = parseProbeEvidence("--accel\nGPU-0001, NVIDIA T4, 5, 100, 15360\nGPU-0001, 42, trainer, extra, part, 96\n"); + + expect(features.accelerator).toEqual([ + { name: "NVIDIA T4", utilPct: 5, memUsedMb: 100, memTotalMb: 15360, processes: [{ pid: 42, name: "trainer, extra, part", vramMb: 96 }] } + ]); + }); + + it("keeps spaces in artifact paths and process comm fields", () => { + const features = parseProbeEvidence( + ["--disk", "104857600 /root/My Folder/model file.bin", "--procorig", "btime=100", "7 ppid=1 starttime=50 comm=my worker", ""].join("\n") + ); + + expect(features.artifacts).toEqual([{ path: "/root/My Folder/model file.bin", sizeBytes: 104857600 }]); + expect(features.processOrigins).toEqual([{ pid: 7, ppid: 1, comm: "my worker", startedAtEpochMs: 100500 }]); + }); + + it("decodes v4-mapped and native ipv6 remotes from netl lines", () => { + const raw = [ + "--netl", + " 1 listen=22", + " 1 0016 0000000000000000FFFF00000100007F:01BB 01", + " 2 0EA7 F804012A8F0B170C0000000002000000:0050 01", + "" + ].join("\n"); + + const features = parseProbeEvidence(raw); + + expect(features.netShape?.listenPorts).toEqual([22]); + expect(features.netShape?.connections).toEqual([ + { localPort: 22, remoteIp: "127.0.0.1", remotePort: 443, count: 1, state: "established" }, + { localPort: 3751, remoteIp: "2a01:04f8:0c17:0b8f:0000:0000:0000:0002", remotePort: 80, count: 2, state: "established" } + ]); + }); + + it("merges duplicate netl keys into one counted entry", () => { + const raw = ["--netl", " 2 0D05 140AB912:01BB 01", " 3 0D05 140AB912:01BB 01", ""].join("\n"); + + const features = parseProbeEvidence(raw); + + expect(features.netShape?.connections).toEqual([{ localPort: 3333, remoteIp: "18.185.10.20", remotePort: 443, count: 5, state: "established" }]); + }); + + it("keeps a half open socket apart from a live one", () => { + const raw = ["--netl", " 1 0D05 140AB912:01BB 01", " 1 0D06 140AB912:01BB 02", ""].join("\n"); + + const features = parseProbeEvidence(raw); + + expect(features.netShape?.connections).toEqual([ + { localPort: 3333, remoteIp: "18.185.10.20", remotePort: 443, count: 1, state: "established" }, + { localPort: 3334, remoteIp: "18.185.10.20", remotePort: 443, count: 1, state: "connecting" } + ]); + }); + + it("ignores malformed section lines instead of throwing", () => { + const features = parseProbeEvidence( + ["--accel", "not,a,gpu,line", "--disk", "notasize /path", "--procorig", "nonsense line", "--netl", "garbage", ""].join("\n") + ); + + expect(features.accelerator).toEqual([]); + expect(features.artifacts).toEqual([]); + expect(features.processOrigins).toEqual([]); + expect(features.netShape).toEqual({ listenPorts: [], connections: [] }); + }); + + it("reports zero startedAtEpochMs when the btime header is missing", () => { + const features = parseProbeEvidence("--procorig\n999 ppid=1 starttime=100 comm=sh\n"); + + expect(features.processOrigins).toEqual([{ pid: 999, ppid: 1, comm: "sh", startedAtEpochMs: 0 }]); + }); +}); diff --git a/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.ts b/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.ts new file mode 100644 index 0000000000..ec54faacc0 --- /dev/null +++ b/apps/api/src/workload-abuse/lib/probe-evidence/parse-probe-evidence.ts @@ -0,0 +1,177 @@ +import type { ProbeEvidenceAccelerator, ProbeEvidenceArtifact, ProbeEvidenceNetShape, ProbeEvidenceProcessOrigin } from "@src/workload-abuse/model-schemas"; + +export type ProbeEvidenceFeatures = { + accelerator: ProbeEvidenceAccelerator[] | null; + artifacts: ProbeEvidenceArtifact[] | null; + processOrigins: ProbeEvidenceProcessOrigin[] | null; + netShape: ProbeEvidenceNetShape | null; +}; + +/** The kernel numbers a live socket 01 and a half open one 02, and the collector reports both. */ +const ESTABLISHED_SOCKET_STATE = "01"; + +const SECTION_MARKERS = ["--accel", "--netl", "--disk", "--procorig"] as const; + +export function parseProbeEvidence(evidenceOutput: string): ProbeEvidenceFeatures { + const sections = splitSections(evidenceOutput); + + return { + accelerator: parseAccelerator(sections.get("--accel")), + artifacts: parseArtifacts(sections.get("--disk")), + processOrigins: parseProcessOrigins(sections.get("--procorig")), + netShape: parseNetShape(sections.get("--netl")) + }; +} + +function splitSections(output: string): Map { + const sections = new Map(); + let currentLines: string[] | null = null; + + for (const line of output.split("\n")) { + if ((SECTION_MARKERS as readonly string[]).includes(line)) { + currentLines = sections.get(line) ?? []; + sections.set(line, currentLines); + continue; + } + currentLines?.push(line); + } + + return sections; +} + +type AcceleratorOnGpu = Omit & { gpuUuid: string }; + +/** Both accelerator queries report the gpu uuid, so a process lands on the card it actually ran on rather than on every card in the host, and a card whose driver reports a field as unavailable is kept with what it did report. */ +function parseAccelerator(lines: string[] | undefined): ProbeEvidenceAccelerator[] | null { + if (!lines) return null; + if (lines.some(line => line.trim() === "accel: unavailable")) return null; + + const accelerators: AcceleratorOnGpu[] = []; + const processesByGpu = new Map(); + + for (const line of lines) { + const fields = line.split(",").map(field => field.trim()); + if (fields.length >= 5 && !isNumeric(fields[1])) { + accelerators.push({ + gpuUuid: fields[0], + name: fields.slice(1, -3).join(", "), + utilPct: toNumberOrZero(fields[fields.length - 3]), + memUsedMb: toNumberOrZero(fields[fields.length - 2]), + memTotalMb: toNumberOrZero(fields[fields.length - 1]) + }); + } else if (fields.length >= 4 && isNumeric(fields[1])) { + const processes = processesByGpu.get(fields[0]) ?? []; + processes.push({ + pid: Number(fields[1]), + name: fields.slice(2, -1).join(", "), + vramMb: toNumberOrZero(fields[fields.length - 1]) + }); + processesByGpu.set(fields[0], processes); + } + } + + return accelerators.map(accelerator => ({ + name: accelerator.name, + utilPct: accelerator.utilPct, + memUsedMb: accelerator.memUsedMb, + memTotalMb: accelerator.memTotalMb, + processes: processesByGpu.get(accelerator.gpuUuid) ?? [] + })); +} + +function parseArtifacts(lines: string[] | undefined): ProbeEvidenceArtifact[] | null { + if (!lines) return null; + + const artifacts: ProbeEvidenceArtifact[] = []; + for (const line of lines) { + const match = line.match(/^(\d+) (.+)$/); + if (match) artifacts.push({ sizeBytes: Number(match[1]), path: match[2] }); + } + + return artifacts; +} + +function parseProcessOrigins(lines: string[] | undefined): ProbeEvidenceProcessOrigin[] | null { + if (!lines) return null; + + let bootTimeSeconds: number | null = null; + const origins: ProbeEvidenceProcessOrigin[] = []; + + for (const line of lines) { + const bootTimeMatch = line.match(/^btime=(\d+)$/); + if (bootTimeMatch) { + bootTimeSeconds = Number(bootTimeMatch[1]); + continue; + } + const match = line.match(/^(\d+) ppid=(\d+) starttime=(\d+) comm=(.*)$/); + if (!match) continue; + const starttimeTicks = Number(match[3]); + origins.push({ + pid: Number(match[1]), + ppid: Number(match[2]), + comm: match[4], + startedAtEpochMs: bootTimeSeconds === null ? 0 : Math.round((bootTimeSeconds + starttimeTicks / 100) * 1000) + }); + } + + return origins; +} + +function parseNetShape(netlLines: string[] | undefined): ProbeEvidenceNetShape | null { + if (!netlLines) return null; + + const listenPorts = new Set(); + for (const line of netlLines) { + const match = line.trim().match(/^\d+\s+listen=(\d+)$/); + if (match) listenPorts.add(Number(match[1])); + } + + const connectionsByKey = new Map(); + for (const line of netlLines) { + const match = line.trim().match(/^(\d+)\s+([0-9A-Fa-f]+)\s+([0-9A-Fa-f]+):([0-9A-Fa-f]+)\s+(\d+)$/); + if (!match) continue; + const localPort = parseInt(match[2], 16); + const remoteIp = decodeHexIp(match[3]); + const remotePort = parseInt(match[4], 16); + if (!remoteIp) continue; + const state = match[5] === ESTABLISHED_SOCKET_STATE ? "established" : "connecting"; + const key = `${localPort}|${remoteIp}|${remotePort}|${state}`; + const existing = connectionsByKey.get(key); + if (existing) existing.count += Number(match[1]); + else connectionsByKey.set(key, { localPort, remoteIp, remotePort, count: Number(match[1]), state }); + } + + return { + listenPorts: [...listenPorts].sort((a, b) => a - b), + connections: [...connectionsByKey.values()] + }; +} + +function decodeHexIp(hex: string): string | null { + if (hex.length === 8) { + return `${parseInt(hex.slice(6, 8), 16)}.${parseInt(hex.slice(4, 6), 16)}.${parseInt(hex.slice(2, 4), 16)}.${parseInt(hex.slice(0, 2), 16)}`; + } + if (hex.length === 32) { + const lowered = hex.toLowerCase(); + if (lowered.startsWith("0000000000000000ffff0000")) return decodeHexIp(lowered.slice(24)); + return formatIpv6(lowered); + } + return null; +} + +function formatIpv6(hex: string): string { + const hextets: string[] = []; + for (let index = 0; index < 32; index += 8) { + const group = hex.slice(index, index + 8); + hextets.push(`${group.slice(6, 8)}${group.slice(4, 6)}`, `${group.slice(2, 4)}${group.slice(0, 2)}`); + } + return hextets.join(":").toLowerCase(); +} + +function toNumberOrZero(value: string): number { + return isNumeric(value) ? Number(value) : 0; +} + +function isNumeric(value: string): boolean { + return /^\d+(\.\d+)?$/.test(value); +} diff --git a/apps/api/src/workload-abuse/model-schemas/index.ts b/apps/api/src/workload-abuse/model-schemas/index.ts index 0d27584ae4..869922b71e 100644 --- a/apps/api/src/workload-abuse/model-schemas/index.ts +++ b/apps/api/src/workload-abuse/model-schemas/index.ts @@ -1,2 +1,3 @@ export * from "./blocked-email-domain/blocked-email-domain.schema"; export * from "./workload-abuse-detection/workload-abuse-detection.schema"; +export * from "./workload-probe-evidence/workload-probe-evidence.schema"; diff --git a/apps/api/src/workload-abuse/model-schemas/workload-probe-evidence/workload-probe-evidence.schema.ts b/apps/api/src/workload-abuse/model-schemas/workload-probe-evidence/workload-probe-evidence.schema.ts new file mode 100644 index 0000000000..f6f83734cc --- /dev/null +++ b/apps/api/src/workload-abuse/model-schemas/workload-probe-evidence/workload-probe-evidence.schema.ts @@ -0,0 +1,50 @@ +import { sql } from "drizzle-orm"; +import { index, integer, jsonb, pgTable, text, timestamp, uuid, varchar } from "drizzle-orm/pg-core"; + +export type ProbeEvidenceAccelerator = { + name: string; + utilPct: number; + memUsedMb: number; + memTotalMb: number; + processes: Array<{ pid: number; name: string; vramMb: number }>; +}; + +export type ProbeEvidenceArtifact = { path: string; sizeBytes: number }; + +export type ProbeEvidenceProcessOrigin = { pid: number; ppid: number; comm: string; startedAtEpochMs: number }; + +export type ProbeEvidenceConnectionState = "established" | "connecting"; + +export type ProbeEvidenceNetShape = { + listenPorts: number[]; + connections: Array<{ localPort: number; remoteIp: string; remotePort: number; count: number; state: ProbeEvidenceConnectionState }>; +}; + +export type ProbeEvidenceBehaviouralFinding = { signal: string; detail: Record }; + +export const WorkloadProbeEvidence = pgTable( + "workload_probe_evidence", + { + id: uuid("id") + .primaryKey() + .notNull() + .default(sql`uuid_generate_v4()`), + walletId: integer("wallet_id").notNull(), + dseq: varchar("dseq").notNull(), + provider: text("provider").notNull(), + service: varchar("service", { length: 255 }).notNull(), + shellStatus: varchar("shell_status", { length: 64 }).notNull(), + verdict: varchar("verdict", { length: 16 }).notNull(), + detectionId: uuid("detection_id"), + accelerator: jsonb("accelerator").$type(), + artifacts: jsonb("artifacts").$type(), + processOrigins: jsonb("process_origins").$type(), + netShape: jsonb("net_shape").$type(), + behaviouralFindings: jsonb("behavioural_findings").$type(), + createdAt: timestamp("created_at", { withTimezone: true }).defaultNow().notNull() + }, + table => ({ + walletDseqCreatedIdx: index("workload_probe_evidence_wallet_dseq_created_idx").on(table.walletId, table.dseq, table.createdAt), + createdIdx: index("workload_probe_evidence_created_idx").on(table.createdAt) + }) +); diff --git a/apps/api/src/workload-abuse/repositories/workload-probe-evidence/workload-probe-evidence.repository.ts b/apps/api/src/workload-abuse/repositories/workload-probe-evidence/workload-probe-evidence.repository.ts new file mode 100644 index 0000000000..8896df592a --- /dev/null +++ b/apps/api/src/workload-abuse/repositories/workload-probe-evidence/workload-probe-evidence.repository.ts @@ -0,0 +1,51 @@ +import { and, asc, eq, gt, lt, sql } from "drizzle-orm"; +import { singleton } from "tsyringe"; + +import { type ApiPgDatabase, type ApiPgTables, InjectPg, InjectPgTable } from "@src/core/providers"; +import { type AbilityParams, BaseRepository } from "@src/core/repositories/base.repository"; +import { TxService } from "@src/core/services"; + +type Table = ApiPgTables["WorkloadProbeEvidence"]; +export type WorkloadProbeEvidenceInput = Partial; +export type WorkloadProbeEvidenceInsert = Table["$inferInsert"]; +export type WorkloadProbeEvidenceOutput = Table["$inferSelect"]; + +@singleton() +export class WorkloadProbeEvidenceRepository extends BaseRepository { + constructor( + @InjectPg() protected readonly pg: ApiPgDatabase, + @InjectPgTable("WorkloadProbeEvidence") protected readonly table: Table, + protected readonly txManager: TxService + ) { + super(pg, table, txManager, "WorkloadProbeEvidence", "WorkloadProbeEvidence"); + } + + accessibleBy(...abilityParams: AbilityParams) { + return new WorkloadProbeEvidenceRepository(this.pg, this.table, this.txManager).withAbility(...abilityParams) as this; + } + + async insertMany(rows: WorkloadProbeEvidenceInsert[]): Promise { + if (!rows.length) return []; + return await this.cursor.insert(this.table).values(rows).returning(); + } + + async findRecentForDeployment({ walletId, dseq, since }: { walletId: number; dseq: string; since: Date }): Promise { + return await this.cursor + .select() + .from(this.table) + .where(and(eq(this.table.walletId, walletId), eq(this.table.dseq, dseq), gt(this.table.createdAt, since))) + .orderBy(asc(this.table.createdAt)); + } + + async countDistinctDeploymentsSince({ since }: { since: Date }): Promise { + const [row] = await this.cursor + .select({ deployments: sql`count(distinct (${this.table.walletId}, ${this.table.dseq}))` }) + .from(this.table) + .where(gt(this.table.createdAt, since)); + return Number(row?.deployments ?? 0); + } + + async deleteOlderThan({ before }: { before: Date }): Promise { + await this.cursor.delete(this.table).where(lt(this.table.createdAt, before)); + } +} diff --git a/apps/api/src/workload-abuse/services/probe-evidence/probe-evidence.service.spec.ts b/apps/api/src/workload-abuse/services/probe-evidence/probe-evidence.service.spec.ts new file mode 100644 index 0000000000..be5e213d6e --- /dev/null +++ b/apps/api/src/workload-abuse/services/probe-evidence/probe-evidence.service.spec.ts @@ -0,0 +1,115 @@ +import { describe, expect, it, vi } from "vitest"; +import { mock } from "vitest-mock-extended"; + +import { type CreateLogger } from "@src/core"; +import type { WorkloadProbeEvidenceRepository } from "@src/workload-abuse/repositories/workload-probe-evidence/workload-probe-evidence.repository"; +import type { WorkloadAbuseConfigService } from "@src/workload-abuse/services/workload-abuse-config/workload-abuse-config.service"; +import type { WorkloadAbuseInstrumentationService } from "@src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service"; +import { ProbeEvidenceService } from "./probe-evidence.service"; + +import { mockConfigService } from "@test/mocks/config-service.mock"; + +describe(ProbeEvidenceService.name, () => { + it("records one row per probed service with parsed features", async () => { + const { service, evidenceRepository } = setup(); + evidenceRepository.insertMany.mockResolvedValue([]); + + await service.recordEvidence({ + walletId: 42, + dseq: "1000001", + verdict: "clean", + shellEvidence: [ + { + service: "web", + provider: "akash1provider", + status: "completed", + evidence: "--accel\nGPU-0001, NVIDIA T4, 90, 14000, 15360\nGPU-0001, 1234, python3, 14000\n--disk\n1073741824 /root/model.bin\n" + }, + { service: "sidecar", provider: "akash1provider", status: "output_capped", evidence: "" } + ] + }); + + expect(evidenceRepository.insertMany).toHaveBeenCalledWith([ + expect.objectContaining({ + walletId: 42, + dseq: "1000001", + provider: "akash1provider", + service: "web", + shellStatus: "completed", + verdict: "clean", + accelerator: [expect.objectContaining({ name: "NVIDIA T4", utilPct: 90 })], + artifacts: [{ path: "/root/model.bin", sizeBytes: 1073741824 }] + }), + expect.objectContaining({ service: "sidecar", shellStatus: "output_capped", accelerator: null, artifacts: null }) + ]); + }); + + it("skips the write when no service was reached", async () => { + const { service, evidenceRepository } = setup(); + + await service.recordEvidence({ walletId: 42, dseq: "1000001", verdict: "clean", shellEvidence: [] }); + + expect(evidenceRepository.insertMany).not.toHaveBeenCalled(); + }); + + it("passes the detection id through to every row", async () => { + const { service, evidenceRepository } = setup(); + evidenceRepository.insertMany.mockResolvedValue([]); + + await service.recordEvidence({ + walletId: 42, + dseq: "1000001", + verdict: "hard", + detectionId: "detection-uuid", + shellEvidence: [{ service: "web", provider: "akash1provider", status: "completed", evidence: "--accel\nGPU-0001, NVIDIA T4, 1, 2, 3\n" }] + }); + + expect(evidenceRepository.insertMany).toHaveBeenCalledWith([expect.objectContaining({ detectionId: "detection-uuid" })]); + }); + + it("logs and counts a write failure without rethrowing", async () => { + const { service, evidenceRepository, instrumentation, logger } = setup(); + evidenceRepository.insertMany.mockRejectedValue(new Error("connection refused")); + + await expect( + service.recordEvidence({ + walletId: 42, + dseq: "1000001", + verdict: "clean", + shellEvidence: [{ service: "web", provider: "akash1provider", status: "completed", evidence: "--disk\n1 /root/x\n" }] + }) + ).resolves.toEqual([]); + + expect(instrumentation.recordEvidenceWriteFailure).toHaveBeenCalledWith("insert"); + expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ event: "WORKLOAD_EVIDENCE_WRITE_FAILED", walletId: 42, dseq: "1000001" })); + }); + + it("purges rows older than the retention window", async () => { + const { service, evidenceRepository } = setup(); + + await service.purgeExpired(); + + expect(evidenceRepository.deleteOlderThan).toHaveBeenCalledWith({ before: expect.any(Date) }); + }); + + it("logs a purge failure without rethrowing", async () => { + const { service, evidenceRepository, instrumentation, logger } = setup(); + evidenceRepository.deleteOlderThan.mockRejectedValue(new Error("deadlock")); + + await expect(service.purgeExpired()).resolves.toBeUndefined(); + + expect(instrumentation.recordEvidenceWriteFailure).toHaveBeenCalledWith("purge"); + expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ event: "WORKLOAD_EVIDENCE_PURGE_FAILED" })); + }); + + function setup(input?: { retentionDays?: number }) { + const evidenceRepository = mock(); + const instrumentation = mock(); + const logger = mock>(); + const createLogger = vi.fn(() => logger); + const config = mockConfigService({ WORKLOAD_ABUSE_EVIDENCE_RETENTION_DAYS: input?.retentionDays ?? 90 }); + const service = new ProbeEvidenceService(evidenceRepository, instrumentation, config, createLogger); + + return { service, evidenceRepository, instrumentation, logger, config }; + } +}); diff --git a/apps/api/src/workload-abuse/services/probe-evidence/probe-evidence.service.ts b/apps/api/src/workload-abuse/services/probe-evidence/probe-evidence.service.ts new file mode 100644 index 0000000000..c268079dab --- /dev/null +++ b/apps/api/src/workload-abuse/services/probe-evidence/probe-evidence.service.ts @@ -0,0 +1,72 @@ +import { inject, singleton } from "tsyringe"; + +import { type CreateLogger, LOGGER_FACTORY } from "@src/core"; +import { parseProbeEvidence } from "@src/workload-abuse/lib/probe-evidence/parse-probe-evidence"; +import { + type WorkloadProbeEvidenceOutput, + WorkloadProbeEvidenceRepository +} from "@src/workload-abuse/repositories/workload-probe-evidence/workload-probe-evidence.repository"; +import type { ShellEvidence } from "@src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service"; +import { WorkloadAbuseConfigService } from "@src/workload-abuse/services/workload-abuse-config/workload-abuse-config.service"; +import { WorkloadAbuseInstrumentationService } from "@src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service"; + +@singleton() +export class ProbeEvidenceService { + private readonly logger: ReturnType; + + constructor( + private readonly evidenceRepository: WorkloadProbeEvidenceRepository, + private readonly instrumentation: WorkloadAbuseInstrumentationService, + private readonly config: WorkloadAbuseConfigService, + @inject(LOGGER_FACTORY) createLogger: CreateLogger + ) { + this.logger = createLogger({ context: ProbeEvidenceService.name }); + } + + async recordEvidence(input: { + walletId: number; + dseq: string; + verdict: string; + detectionId?: string; + shellEvidence: ShellEvidence[]; + }): Promise { + if (!input.shellEvidence.length) return []; + + try { + return await this.evidenceRepository.insertMany( + input.shellEvidence.map(shellEvidence => { + const features = parseProbeEvidence(shellEvidence.evidence); + return { + walletId: input.walletId, + dseq: input.dseq, + provider: shellEvidence.provider, + service: shellEvidence.service, + shellStatus: shellEvidence.status, + verdict: input.verdict, + detectionId: input.detectionId, + accelerator: features.accelerator, + artifacts: features.artifacts, + processOrigins: features.processOrigins, + netShape: features.netShape + }; + }) + ); + } catch (error) { + this.instrumentation.recordEvidenceWriteFailure("insert"); + this.logger.warn({ event: "WORKLOAD_EVIDENCE_WRITE_FAILED", error, walletId: input.walletId, dseq: input.dseq }); + return []; + } + } + + async purgeExpired(): Promise { + const retentionDays = this.config.get("WORKLOAD_ABUSE_EVIDENCE_RETENTION_DAYS"); + const before = new Date(Date.now() - retentionDays * 24 * 60 * 60 * 1000); + + try { + await this.evidenceRepository.deleteOlderThan({ before }); + } catch (error) { + this.instrumentation.recordEvidenceWriteFailure("purge"); + this.logger.warn({ event: "WORKLOAD_EVIDENCE_PURGE_FAILED", error, before }); + } + } +} diff --git a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.integration.ts b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.integration.ts index 148af985fa..0a1164ada9 100644 --- a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.integration.ts +++ b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.integration.ts @@ -5,6 +5,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { JOB_NAME } from "@src/core"; import { WorkloadAbuseDetectionRepository } from "@src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository"; +import { WorkloadProbeEvidenceRepository } from "@src/workload-abuse/repositories/workload-probe-evidence/workload-probe-evidence.repository"; import { type ProbeReport, TrialWorkloadProbeService } from "@src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service"; import { ProbeTrialDeployment, probeTrialDeploymentKeyFor } from "@src/workload-abuse/services/trial-workload-probe-job/trial-workload-probe-job.service"; import { WorkloadAbuseConfigService } from "@src/workload-abuse/services/workload-abuse-config/workload-abuse-config.service"; @@ -16,6 +17,8 @@ import { expectJobCompleted, findJobRows, useJobWorkers } from "@test/services/j const MAX_ATTEMPTS = 30; +const ACCELERATED_EVIDENCE = "--accel\nGPU-0001, NVIDIA A100, 95, 20480, 24576\nGPU-0001, 1234, python3, 18000"; + const jobWorkers = useJobWorkers(() => [container.resolve(ProbeTrialDeploymentHandler)]); describe(ProbeTrialDeploymentHandler.name, () => { @@ -64,6 +67,43 @@ describe(ProbeTrialDeploymentHandler.name, () => { expect(await findDetections()).toHaveLength(0); }); + it("records evidence for every probed service on a clean probe", async () => { + const { probeDeployment, findEvidence } = await setup({ verdict: "clean" }); + + await probeDeployment(); + + expect(await findEvidence()).toMatchObject([ + { + verdict: "clean", + shellStatus: "completed", + detectionId: null, + provider: "akash1provider", + service: "ssh", + accelerator: [{ name: "NVIDIA A100", utilPct: 95, memUsedMb: 20480, memTotalMb: 24576, processes: [{ pid: 1234, name: "python3", vramMb: 18000 }] }], + artifacts: null, + processOrigins: null, + netShape: null + } + ]); + }); + + it("links the evidence rows to the detection the probe recorded", async () => { + const { probeDeployment, findDetections, findEvidence } = await setup({ verdict: "hard" }); + + await probeDeployment(); + + const [detection] = await findDetections(); + expect(await findEvidence()).toMatchObject([{ verdict: "hard", detectionId: detection.id }]); + }); + + it("records no evidence when the deployment has no live lease", async () => { + const { probeDeployment, findEvidence } = await setup({ verdict: "clean", probeStatus: "no_live_lease" }); + + await probeDeployment(); + + expect(await findEvidence()).toHaveLength(0); + }); + it("records nothing more for a deployment already judged abusive", async () => { const { probeDeployment, findDetections, probe, seedExistingDetection } = await setup({ verdict: "hard" }); await seedExistingDetection(); @@ -74,9 +114,16 @@ describe(ProbeTrialDeploymentHandler.name, () => { expect(await findDetections()).toHaveLength(1); }); - async function setup(input: { verdict: ProbeReport["verdict"]; attempt?: number; isTrialing?: boolean }) { + async function setup(input: { + verdict: ProbeReport["verdict"]; + attempt?: number; + isTrialing?: boolean; + probeStatus?: ProbeReport["probeStatus"]; + shellEvidence?: ProbeReport["shellEvidence"]; + }) { const { enqueue, startWorkers } = await jobWorkers(); const detectionRepository = container.resolve(WorkloadAbuseDetectionRepository); + const evidenceRepository = container.resolve(WorkloadProbeEvidenceRepository); const { user, wallet, address } = await seedUserWithWallet({ isTrialing: input.isTrialing ?? true }); const dseq = createDseq(); @@ -86,10 +133,11 @@ describe(ProbeTrialDeploymentHandler.name, () => { const probe = vi.spyOn(container.resolve(TrialWorkloadProbeService), "probe").mockResolvedValue({ verdict: input.verdict, - probeStatus: "probed", + probeStatus: input.probeStatus ?? "probed", signals: [], excerpt: "denied", - leases: [] + leases: [], + shellEvidence: input.shellEvidence ?? [{ service: "ssh", provider: "akash1provider", status: "completed", evidence: ACCELERATED_EVIDENCE }] }); const probeKey = probeTrialDeploymentKeyFor({ walletId: wallet.id, dseq }); @@ -98,6 +146,7 @@ describe(ProbeTrialDeploymentHandler.name, () => { dseq, probe, findDetections: () => detectionRepository.find({ walletId: wallet.id, dseq }), + findEvidence: () => evidenceRepository.find({ walletId: wallet.id, dseq }), seedExistingDetection: () => detectionRepository.create({ userId: user.id, diff --git a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts index 9a60666c63..d4a0554a7a 100644 --- a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts +++ b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts @@ -7,7 +7,9 @@ import type { WorkloadAbuseDetectionOutput, WorkloadAbuseDetectionRepository } from "@src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository"; +import type { WorkloadProbeEvidenceOutput } from "@src/workload-abuse/repositories/workload-probe-evidence/workload-probe-evidence.repository"; import { EnforceTrialAbuse } from "@src/workload-abuse/services/enforce-trial-abuse/enforce-trial-abuse.handler"; +import type { ProbeEvidenceService } from "@src/workload-abuse/services/probe-evidence/probe-evidence.service"; import type { ProbeReport, TrialWorkloadProbeService } from "@src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service"; import type { TrialWorkloadProbeJobService } from "@src/workload-abuse/services/trial-workload-probe-job/trial-workload-probe-job.service"; import type { WorkloadAbuseConfigService } from "@src/workload-abuse/services/workload-abuse-config/workload-abuse-config.service"; @@ -50,7 +52,7 @@ describe(ProbeTrialDeploymentHandler.name, () => { expect(detectionRepository.create).not.toHaveBeenCalled(); }); - it("reschedules a clean probe and records nothing", async () => { + it("reschedules a clean probe and records no detection", async () => { const { handler, probeJobService, detectionRepository, instrumentation } = setup({ report: createReport({ verdict: "clean" }) }); await handler.handle(PAYLOAD); @@ -60,6 +62,53 @@ describe(ProbeTrialDeploymentHandler.name, () => { expect(instrumentation.recordProbe).toHaveBeenCalledWith(expect.objectContaining({ verdict: "clean", probeStatus: "probed" })); }); + it("records evidence on a clean probe with the collected evidence and no detection id", async () => { + const shellEvidence = [{ service: "web", provider: "akash1provider", status: "completed" as const, evidence: "--accel\naccel: unavailable" }]; + const { handler, wallet, probeEvidenceService } = setup({ report: createReport({ verdict: "clean", shellEvidence }) }); + + await handler.handle(PAYLOAD); + + expect(probeEvidenceService.recordEvidence).toHaveBeenCalledWith({ + walletId: wallet.id, + dseq: PAYLOAD.dseq, + verdict: "clean", + detectionId: undefined, + shellEvidence + }); + }); + + it("records evidence with the detection id on a probe that lands a detection", async () => { + const { handler, probeEvidenceService } = setup({ report: createReport({ verdict: "hard" }) }); + + await handler.handle(PAYLOAD); + + expect(probeEvidenceService.recordEvidence).toHaveBeenCalledWith(expect.objectContaining({ detectionId: "detection-1" })); + }); + + it("logs the recorded evidence services for the probe run", async () => { + const { handler, logger } = setup({ report: createReport({ verdict: "clean" }) }); + + await handler.handle(PAYLOAD); + + expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ event: "TRIAL_WORKLOAD_EVIDENCE_RECORDED", services: ["ssh"] })); + }); + + it("says nothing about recorded evidence when the probe reached no service", async () => { + const { handler, logger } = setup({ report: createReport({ verdict: "clean", shellEvidence: [] }) }); + + await handler.handle(PAYLOAD); + + expect(logger.info).not.toHaveBeenCalledWith(expect.objectContaining({ event: "TRIAL_WORKLOAD_EVIDENCE_RECORDED" })); + }); + + it("records no evidence when the deployment has no live lease", async () => { + const { handler, probeEvidenceService } = setup({ report: createReport({ probeStatus: "no_live_lease", shellEvidence: [] }) }); + + await handler.handle(PAYLOAD); + + expect(probeEvidenceService.recordEvidence).not.toHaveBeenCalled(); + }); + it("logs what the shell saw for every verdict, cut so the line survives log shipping", async () => { const { handler, logger } = setup({ report: createReport({ verdict: "clean", excerpt: "x".repeat(5_000) }) }); @@ -182,6 +231,7 @@ describe(ProbeTrialDeploymentHandler.name, () => { logStatus: "completed" } ], + shellEvidence: [{ service: "ssh", provider: "akash1provider", status: "completed", evidence: "--accel\naccel: unavailable" }], ...overrides }; } @@ -232,6 +282,10 @@ describe(ProbeTrialDeploymentHandler.name, () => { const detectionRepository = mock(); detectionRepository.create.mockResolvedValue(mock({ id: "detection-1" })); detectionRepository.findOneBy.mockResolvedValue(input.existingDetection ? mock({ id: "detection-0" }) : undefined); + const probeEvidenceService = mock(); + probeEvidenceService.recordEvidence.mockImplementation(async ({ shellEvidence }) => + shellEvidence.map(entry => mock({ service: entry.service })) + ); const instrumentation = mock(); const config = mockConfigService({ WORKLOAD_ABUSE_PROBE_ENABLED: input.enabled ?? true, @@ -247,12 +301,24 @@ describe(ProbeTrialDeploymentHandler.name, () => { probeService, probeJobService, detectionRepository, + probeEvidenceService, instrumentation, config, jobQueueService, createLogger ); - return { handler, wallet: wallet!, userWalletRepository, probeService, probeJobService, detectionRepository, instrumentation, jobQueueService, logger }; + return { + handler, + wallet: wallet!, + userWalletRepository, + probeService, + probeJobService, + detectionRepository, + probeEvidenceService, + instrumentation, + jobQueueService, + logger + }; } }); diff --git a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts index 7de693700a..3fb280a694 100644 --- a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts +++ b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts @@ -6,6 +6,7 @@ import { withoutFileContents } from "@src/workload-abuse/lib/evidence-scanner/ev import { truncateToUtf8Bytes } from "@src/workload-abuse/lib/utf8-text/utf8-text"; import { WorkloadAbuseDetectionRepository } from "@src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository"; import { EnforceTrialAbuse, enforceTrialAbuseKeyFor } from "@src/workload-abuse/services/enforce-trial-abuse/enforce-trial-abuse.handler"; +import { ProbeEvidenceService } from "@src/workload-abuse/services/probe-evidence/probe-evidence.service"; import { type ProbeReport, TrialWorkloadProbeService } from "@src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service"; import { ProbeTrialDeployment, TrialWorkloadProbeJobService } from "@src/workload-abuse/services/trial-workload-probe-job/trial-workload-probe-job.service"; import { WorkloadAbuseConfigService } from "@src/workload-abuse/services/workload-abuse-config/workload-abuse-config.service"; @@ -31,6 +32,7 @@ export class ProbeTrialDeploymentHandler implements JobHandler row.service) + }); + } + this.logger.info({ event: "TRIAL_WORKLOAD_PROBED", ...context, diff --git a/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.spec.ts b/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.spec.ts index 1d42c855e1..ea81dcc936 100644 --- a/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.spec.ts +++ b/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.spec.ts @@ -1,5 +1,5 @@ import { spawnSync } from "node:child_process"; -import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, symlinkSync, truncateSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; @@ -8,17 +8,19 @@ import { mock } from "vitest-mock-extended"; import { withoutFileContents } from "@src/workload-abuse/lib/evidence-scanner/evidence-scanner"; import type { ProviderStreamResult, ProviderStreamService } from "@src/workload-abuse/services/provider-stream/provider-stream.service"; import type { WorkloadAbuseConfigService } from "@src/workload-abuse/services/workload-abuse-config/workload-abuse-config.service"; -import { buildShellProbeUrl, ProviderShellProbeService, SHELL_PROBE_COLLECTORS, SHELL_PROBE_SCRIPT } from "./provider-shell-probe.service"; +import { buildShellProbeScript, buildShellProbeUrl, ProviderShellProbeService, SHELL_PROBE_COLLECTORS } from "./provider-shell-probe.service"; import { mockConfigService } from "@test/mocks/config-service.mock"; const TMP_FILE_GLOBS = "/tmp/*.json /tmp/*.conf /tmp/*.txt /tmp/*/*.json /tmp/*/*.conf"; const TARGET = { hostUri: "https://provider.example:8443", providerAddress: "akash1provider", token: "jwt", dseq: "123", gseq: 1, oseq: 2, service: "ssh box" }; +const BOUNDARY = "0f1e2d3c4b5a69788796a5b4c3d2e1f0"; +const SCRIPT = buildShellProbeScript(BOUNDARY); describe(ProviderShellProbeService.name, () => { describe("buildShellProbeUrl", () => { it("runs the collector script through sh without stdin or a tty on the first pod of the service", () => { - const url = new URL(buildShellProbeUrl(TARGET)); + const url = new URL(buildShellProbeUrl(TARGET, BOUNDARY)); expect(url.origin + url.pathname).toBe("https://provider.example:8443/lease/123/1/2/shell"); expect(url.searchParams.get("stdin")).toBe("0"); @@ -26,51 +28,57 @@ describe(ProviderShellProbeService.name, () => { expect(url.searchParams.get("podIndex")).toBe("0"); expect(url.searchParams.get("cmd0")).toBe("sh"); expect(url.searchParams.get("cmd1")).toBe("-c"); - expect(url.searchParams.get("cmd2")).toBe(SHELL_PROBE_SCRIPT); + expect(url.searchParams.get("cmd2")).toBe(SCRIPT); expect(url.searchParams.get("service")).toBe("ssh box"); }); it("prints expanded lines with printf so a dash echo cannot turn the script's own cmdline into NUL bytes", () => { - expect(SHELL_PROBE_SCRIPT).toContain("printf '%s\\n' \"${p#/proc/} cpu_s="); - expect(SHELL_PROBE_SCRIPT).toContain("printf '%s\\n' \"== $f\""); - expect(SHELL_PROBE_SCRIPT).not.toMatch(/echo "/); + expect(SCRIPT).toContain("printf '%s\\n' \"${p#/proc/} cpu_s="); + expect(SCRIPT).toContain("printf '%s\\n' \"== $f\""); + expect(SCRIPT).not.toMatch(/echo "/); }); it("marks every file body line as it reads it, so no body line can pass for a section or a file header", () => { - expect(SHELL_PROBE_SCRIPT).not.toMatch(/cat "\$f"/); - expect(SHELL_PROBE_SCRIPT.match(/printf '\| %s\\n' "\$l"/g)).toHaveLength(2); + expect(SCRIPT).not.toMatch(/cat "\$f"/); + expect(SCRIPT.match(/printf '\| %s\\n' "\$l"/g)).toHaveLength(2); }); it("reads a whole stat file before parsing it, since a workload can put a newline in its own process name", () => { - expect(SHELL_PROBE_SCRIPT).toContain('{ s=; while IFS= read -r x; do s="$s$x "; done < "$p/stat"; } 2>/dev/null; [ -n "$s" ] || continue'); + expect(SCRIPT).toContain('{ s=; while IFS= read -r x; do s="$s$x "; done < "$p/stat"; } 2>/dev/null; [ -n "$s" ] || continue'); }); it("streams the process listing line by line and bounds each cmdline read, so a hung or starved process still leaves the rest visible", () => { - expect(SHELL_PROBE_SCRIPT).toContain("T=$(command -v timeout >/dev/null 2>&1 && printf 'timeout 2')"); - expect(SHELL_PROBE_SCRIPT).toContain("c=$({ $T tr '\\0' ' ' < \"$p/cmdline\"; } 2>/dev/null)"); - expect(SHELL_PROBE_SCRIPT).toContain('n=$((n + 1)); [ "$n" -ge 150 ] && break; done'); - expect(SHELL_PROBE_SCRIPT).not.toContain("head -150"); + expect(SCRIPT).toContain("T=$(command -v timeout >/dev/null 2>&1 && printf 'timeout 2')"); + expect(SCRIPT).toContain("c=$({ $T tr '\\0' ' ' < \"$p/cmdline\"; } 2>/dev/null)"); + expect(SCRIPT).toContain('n=$((n + 1)); [ "$n" -ge 150 ] && break; done'); + expect(SCRIPT).not.toContain("head -150"); }); it("leaves its own shell and that shell's children out of the process listing by pid, not by what they run", () => { - expect(SHELL_PROBE_SCRIPT).toContain('[ "${p#/proc/}" = "$$" ] && continue'); - expect(SHELL_PROBE_SCRIPT).toContain('[ "${2:-}" = "$$" ] && continue'); + expect(SCRIPT).toContain('[ "${p#/proc/}" = "$$" ] && continue'); + expect(SCRIPT).toContain('[ "${2:-}" = "$$" ] && continue'); }); it("records cpu time and memory per process, listening ports, and files written since the container started", () => { - expect(SHELL_PROBE_SCRIPT).toContain("cpu_s=$(( (${12:-0} + ${13:-0}) / 100 )) rss_mb=$(( ${22:-0} * 4 / 1024 ))"); - expect(SHELL_PROBE_SCRIPT).toContain("printf 'listen=%d\\n' \"0x${la#*:}\""); - expect(SHELL_PROBE_SCRIPT).toContain( + expect(SCRIPT).toContain("cpu_s=$(( (${12:-0} + ${13:-0}) / 100 )) rss_mb=$(( ${22:-0} * 4 / 1024 ))"); + expect(SCRIPT).toContain("printf 'listen=%d\\n' \"0x${la#*:}\""); + expect(SCRIPT).toContain( "echo '--recent-exec'; find / \\( -path /proc -o -path /sys -o -path /dev -o -name node_modules \\) -prune -o -type f -perm -100 -newer /proc/1 -print" ); - expect(SHELL_PROBE_SCRIPT).toContain( + expect(SCRIPT).toContain( "echo '--recent-conf'; find / \\( -path /proc -o -path /sys -o -path /dev -o -path /etc -o -path /tmp -o -name node_modules \\) -prune" ); }); + it("closes the collected sections with the boundary it was given, before the sections only the evidence table reads", () => { + expect(SCRIPT).toContain(`echo '--evidence ${BOUNDARY}'`); + expect(SCRIPT.indexOf(`--evidence ${BOUNDARY}`)).toBeLessThan(SCRIPT.indexOf("--accel")); + expect(SCRIPT.indexOf("--recent-conf")).toBeLessThan(SCRIPT.indexOf(`--evidence ${BOUNDARY}`)); + }); + it("only reads the container and never changes, fetches or runs anything in it", () => { - expect(SHELL_PROBE_SCRIPT).not.toMatch(/\b(curl|wget|chmod|chown|kill|rm|mv|cp|apt|apk|pip|npm)\b/); - expect(SHELL_PROBE_SCRIPT).not.toMatch(/[^2<]>\s*\/(?!dev\/null)/); + expect(SCRIPT).not.toMatch(/\b(curl|wget|chmod|chown|kill|rm|mv|cp|apt|apk|pip|npm)\b/); + expect(SCRIPT).not.toMatch(/[^2<]>\s*\/(?!dev\/null)/); }); }); @@ -184,6 +192,92 @@ describe(ProviderShellProbeService.name, () => { }); }); + describe("the --accel collector", () => { + it("reports the accelerator model, utilization and memory along with the processes holding it", () => { + const binDirectory = mkdtempSync(join(tmpdir(), "shell-probe-accel-")); + writeFileSync( + join(binDirectory, "nvidia-smi"), + [ + "#!/bin/sh", + 'case "$1" in', + " --query-gpu*) printf 'GPU-0001, NVIDIA A100, 95, 20480, 24576\\n' ;;", + " --query-compute-apps*) printf 'GPU-0001, 1234, python3, 18000\\n' ;;", + "esac" + ].join("\n"), + { mode: 0o755 } + ); + + const reported = runCollector("--accel", collector => collector, { ...process.env, PATH: `${binDirectory}:${process.env.PATH}` }); + + expect(reported).toEqual(["--accel", "GPU-0001, NVIDIA A100, 95, 20480, 24576", "GPU-0001, 1234, python3, 18000"]); + }); + + it("reports the accelerator as unavailable when the container has no nvidia-smi", () => { + const binDirectory = mkdtempSync(join(tmpdir(), "shell-probe-accel-")); + symlinkSync("/bin/sh", join(binDirectory, "sh")); + + const reported = runCollector("--accel", collector => collector, { PATH: binDirectory }); + + expect(reported).toEqual(["--accel", "accel: unavailable"]); + }); + }); + + describe("the --netl collector", () => { + it("reports the local port and remote endpoint of each connected socket, counted per peer", () => { + const reported = runNetlCollector({ + tcp: [ + " 0: 0100007F:9C4E 140AB912:0D05 01 00000000:00000000 00:00000000 00000000 0 0 2 1 0 0 0", + " 1: 0100007F:9C4E 140AB912:0D05 01 00000000:00000000 00:00000000 00000000 0 0 2 1 0 0 0", + " 2: 00000000:1F90 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 3 1 0 0 0" + ], + tcp6: [ + " 0: 0000000000000000FFFF00000100007F:A0F0 0000000000000000FFFF000009030E34:0D05 01 00000000:00000000 00:00000000 00000000 0 0 4 1 0 0 0" + ] + }); + + expect(reported).toEqual(expect.arrayContaining(["2 9C4E 140AB912:0D05 01", "1 A0F0 0000000000000000FFFF000009030E34:0D05 01", "1 listen=8080"])); + expect(reported).toHaveLength(4); + }); + + it("leaves out a socket that is neither established nor connecting", () => { + const reported = runNetlCollector({ + tcp: [" 0: 0100007F:9C4E 140AB912:0D05 06 00000000:00000000 00:00000000 00000000 0 0 7 1 0 0 0"] + }); + + expect(reported).toEqual(["--netl"]); + }); + }); + + describe("the --disk collector", () => { + it("reports the size and path of a file over 64 MiB and leaves smaller files out", () => { + const directory = mkdtempSync(join(tmpdir(), "shell-probe-disk-")); + writeFileSync(join(directory, "weights.bin"), ""); + truncateSync(join(directory, "weights.bin"), 70 * 1024 * 1024); + writeFileSync(join(directory, "notes.txt"), "notes"); + + const reported = runCollector("--disk", collector => collector.replace("find / -xdev", `find ${directory} -xdev`)).map(line => line.trim()); + + expect(reported).toEqual(["--disk", `73400320 ${join(directory, "weights.bin")}`]); + }); + }); + + describe("the --procorig collector", () => { + it("reports the boot time and the parent, start time and name of each process it finds", () => { + const reported = runProcorigCollector([{ pid: "9000001", comm: "node", ppid: 1, starttimeTicks: 100, cmdline: ["node", "app.js"] }]); + + expect(reported).toEqual(["--procorig", "btime=1740000000", "9000001 ppid=1 starttime=100 comm=node"]); + }); + + it("leaves out a process with no command line, so kernel threads do not crowd out the workload", () => { + const reported = runProcorigCollector([ + { pid: "9000001", comm: "kthreadd", ppid: 2, starttimeTicks: 5, cmdline: [] }, + { pid: "9000002", comm: "node", ppid: 1, starttimeTicks: 200, cmdline: ["node", "app.js"] } + ]); + + expect(reported).toEqual(["--procorig", "btime=1740000000", "9000002 ppid=1 starttime=200 comm=node"]); + }); + }); + describe("run", () => { it("joins stdout and stderr into the output and keeps the stream status", async () => { const { service } = setup({ @@ -198,7 +292,48 @@ describe(ProviderShellProbeService.name, () => { const result = await service.run(TARGET); - expect(result).toEqual({ status: "completed", output: "--loadavg\n1.00\nwarn", exitCode: 0 }); + expect(result).toEqual({ status: "completed", output: "--loadavg\n1.00\nwarn", evidence: "", exitCode: 0 }); + }); + + it("splits the collected output from the evidence block at the boundary the run issued", async () => { + const { service } = setup(boundary => streamOf(`--loadavg\n1.00\n--evidence ${boundary}\n--accel\naccel: unavailable`)); + + expect(await service.run(TARGET)).toEqual({ + status: "completed", + output: "--loadavg\n1.00\n", + evidence: "--accel\naccel: unavailable", + exitCode: 0 + }); + }); + + it("keeps a collected line that reads like a boundary with the collected output", async () => { + const { service } = setup(streamOf("--tmp\n--evidence 0f1e2d3c4b5a69788796a5b4c3d2e1f0\n--accel\nGPU-0001, NVIDIA A100, 95, 20480, 24576")); + + const result = await service.run(TARGET); + + expect(result.evidence).toBe(""); + expect(result.output).toContain("--accel"); + }); + + it("cuts at the last boundary, so a repeated one cannot hide collected output from the scanner", async () => { + const { service } = setup(boundary => streamOf(`--tmp\n--evidence ${boundary}\ntotal 8\n--evidence ${boundary}\n--accel\naccel: unavailable`)); + + const result = await service.run(TARGET); + + expect(result.output).toContain("total 8"); + expect(result.evidence).toBe("--accel\naccel: unavailable"); + }); + + it("issues a boundary the workload has not seen before on every run", async () => { + const { service, providerStreamService } = setup(streamOf("--loadavg\n1.00")); + + await service.run(TARGET); + await service.run(TARGET); + + const [first, second] = providerStreamService.collect.mock.calls.map(([input]) => boundaryOf(input.url)); + + expect(first).toMatch(/^[0-9a-f]{32}$/); + expect(second).not.toBe(first); }); it("reports the shell as unavailable when the provider fails to exec or nothing comes back", async () => { @@ -242,9 +377,9 @@ describe(ProviderShellProbeService.name, () => { return { reported: runCollector("--files", collector => collector.replace(TMP_FILE_GLOBS, `${directory}/*.conf`)), path }; } - function runCollector(section: string, toRunnable: (collector: string) => string) { + function runCollector(section: string, toRunnable: (collector: string) => string, env?: NodeJS.ProcessEnv) { const collector = SHELL_PROBE_COLLECTORS.find(entry => entry.includes(section)) ?? ""; - const { stdout } = spawnSync("sh", ["-c", toRunnable(collector)], { encoding: "utf8" }); + const { stdout } = spawnSync("sh", ["-c", toRunnable(collector)], { encoding: "utf8", env }); return stdout .split("\n") @@ -263,9 +398,49 @@ describe(ProviderShellProbeService.name, () => { return runCollector("--net", collector => collector.replace("/proc/net/tcp /proc/net/tcp6", `${directory}/tcp ${directory}/tcp6`)).map(line => line.trim()); } - function setup(result: ProviderStreamResult) { + function runNetlCollector(procFiles: { tcp?: string[]; tcp6?: string[] }) { + const header = " sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode"; + const directory = mkdtempSync(join(tmpdir(), "shell-probe-netl-")); + + for (const name of ["tcp", "tcp6"] as const) { + writeFileSync(join(directory, name), [header, ...(procFiles[name] ?? [])].join("\n") + "\n"); + } + + return runCollector("--netl", collector => collector.replace("/proc/net/tcp /proc/net/tcp6", `${directory}/tcp ${directory}/tcp6`)).map(line => + line.trim() + ); + } + + function runProcorigCollector(processes: { pid: string; comm: string; ppid: number; starttimeTicks: number; cmdline: string[] }[]) { + const directory = mkdtempSync(join(tmpdir(), "shell-probe-procorig-")); + writeFileSync(join(directory, "stat"), "btime 1740000000\n"); + + for (const fixture of processes) { + const fieldsAfterComm: (string | number)[] = Array(22).fill(0); + fieldsAfterComm[0] = "S"; + fieldsAfterComm[1] = fixture.ppid; + fieldsAfterComm[19] = fixture.starttimeTicks; + mkdirSync(join(directory, fixture.pid)); + writeFileSync(join(directory, fixture.pid, "stat"), `${fixture.pid} (${fixture.comm}) ${fieldsAfterComm.join(" ")}\n`); + writeFileSync(join(directory, fixture.pid, "cmdline"), fixture.cmdline.map(argument => `${argument}\0`).join("")); + } + + return runCollector("--procorig", collector => collector.replaceAll("/proc/", `${directory}/`)); + } + + function streamOf(payload: string): ProviderStreamResult { + return { status: "completed", exitCode: 0, frames: [{ kind: "shell", stream: "stdout", payload }] }; + } + + function boundaryOf(url: string): string { + const script = new URL(url).searchParams.get("cmd2") ?? ""; + + return script.match(/--evidence ([0-9a-f]+)/)?.[1] ?? ""; + } + + function setup(result: ProviderStreamResult | ((boundary: string) => ProviderStreamResult)) { const providerStreamService = mock(); - providerStreamService.collect.mockResolvedValue(result); + providerStreamService.collect.mockImplementation(async input => (typeof result === "function" ? result(boundaryOf(input.url)) : result)); const config = mockConfigService({ WORKLOAD_ABUSE_PROBE_IDLE_TIMEOUT_MS: 5_000, WORKLOAD_ABUSE_PROBE_HARD_TIMEOUT_MS: 30_000, diff --git a/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.ts b/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.ts index 4be465eece..da9f14c3a6 100644 --- a/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.ts +++ b/apps/api/src/workload-abuse/services/provider-shell-probe/provider-shell-probe.service.ts @@ -1,3 +1,4 @@ +import { randomBytes } from "node:crypto"; import { singleton } from "tsyringe"; import { FILE_BODY_PREFIX } from "@src/workload-abuse/lib/evidence-scanner/evidence-scanner"; @@ -5,7 +6,7 @@ import { ProviderStreamService, type ProviderStreamStatus } from "@src/workload- import { WorkloadAbuseConfigService } from "@src/workload-abuse/services/workload-abuse-config/workload-abuse-config.service"; /** Collect-only and printf-only: argv is visible to the workload through /proc, and dash's echo expands the `\0` in this script's own cmdline into a NUL byte. */ -export const SHELL_PROBE_COLLECTORS = [ +const WORKLOAD_COLLECTORS = [ "echo '--loadavg'; cat /proc/loadavg 2>/dev/null", "echo '--nproc'; nproc 2>/dev/null || grep -c ^processor /proc/cpuinfo 2>/dev/null", 'echo \'--procs\'; T=$(command -v timeout >/dev/null 2>&1 && printf \'timeout 2\'); n=0; for p in /proc/[0-9]*; do [ "${p#/proc/}" = "$$" ] && continue; { s=; while IFS= read -r x; do s="$s$x "; done < "$p/stat"; } 2>/dev/null; [ -n "$s" ] || continue; m=${s#*(}; m=${m%)*}; set -- ${s##*) }; [ "${2:-}" = "$$" ] && continue; c=$({ $T tr \'\\0\' \' \' < "$p/cmdline"; } 2>/dev/null); [ -n "$c" ] || continue; printf \'%s\\n\' "${p#/proc/} cpu_s=$(( (${12:-0} + ${13:-0}) / 100 )) rss_mb=$(( ${22:-0} * 4 / 1024 )) comm=$m exe=$(readlink "$p/exe" 2>/dev/null) cwd=$(readlink "$p/cwd" 2>/dev/null) cmd=$c"; n=$((n + 1)); [ "$n" -ge 150 ] && break; done', @@ -17,11 +18,25 @@ export const SHELL_PROBE_COLLECTORS = [ `echo '--recent-conf'; find / \\( -path /proc -o -path /sys -o -path /dev -o -path /etc -o -path /tmp -o -name node_modules \\) -prune -o -type f -newer /proc/1 \\( -name '*.json' -o -name '*.conf' -o -name '*.ini' -o -name '*.txt' \\) -size -16k -print 2>/dev/null | head -20 | while read -r f; do printf '%s\\n' "== $f"; [ -r "$f" ] || continue; while IFS= read -r l || [ -n "$l" ]; do printf '${FILE_BODY_PREFIX}%s\\n' "$l"; l=; done < "$f"; done | head -400` ]; -export const SHELL_PROBE_SCRIPT = SHELL_PROBE_COLLECTORS.join("; "); +const EVIDENCE_COLLECTORS = [ + "echo '--accel'; T=$(command -v timeout >/dev/null 2>&1 && printf 'timeout 2'); command -v nvidia-smi >/dev/null 2>&1 && { $T nvidia-smi --query-gpu=uuid,name,utilization.gpu,memory.used,memory.total --format=csv,noheader,nounits 2>/dev/null; $T nvidia-smi --query-compute-apps=gpu_uuid,pid,process_name,used_gpu_memory --format=csv,noheader,nounits 2>/dev/null; } || printf 'accel: unavailable\\n'", + 'echo \'--netl\'; for f in /proc/net/tcp /proc/net/tcp6; do [ -r "$f" ] || continue; while read -r sl la ra st rest; do case $st in 0A) printf \'listen=%d\\n\' "0x${la#*:}"; continue;; 01|02) ;; *) continue;; esac; printf \'%s %s %s\\n\' "${la#*:}" "$ra" "$st"; done < "$f"; done 2>/dev/null | sort | uniq -c | head -80', + 'echo \'--disk\'; find / -xdev -type f -size +64M 2>/dev/null | head -200 | while read -r f; do printf \'%s %s\\n\' "$(wc -c < "$f" 2>/dev/null)" "$f"; done 2>/dev/null | sort -rn | head -40', + 'echo \'--procorig\'; b=$(grep ^btime /proc/stat 2>/dev/null); printf \'btime=%s\\n\' "${b#btime }"; T=$(command -v timeout >/dev/null 2>&1 && printf \'timeout 2\'); n=0; for p in /proc/[0-9]*; do [ "${p#/proc/}" = "$$" ] && continue; { s=; while IFS= read -r x; do s="$s$x "; done < "$p/stat"; } 2>/dev/null; [ -n "$s" ] || continue; m=${s#*(}; m=${m%)*}; set -- ${s##*) }; [ "${2:-}" = "$$" ] && continue; c=$({ $T tr \'\\0\' \' \' < "$p/cmdline"; } 2>/dev/null); [ -n "$c" ] || continue; printf \'%s\\n\' "${p#/proc/} ppid=${2:-0} starttime=${20:-0} comm=$m"; n=$((n + 1)); [ "$n" -ge 150 ] && break; done' +]; + +export const SHELL_PROBE_COLLECTORS = [...WORKLOAD_COLLECTORS, ...EVIDENCE_COLLECTORS]; + +const EVIDENCE_BOUNDARY_PREFIX = "--evidence "; + +/** The workload writes part of what the collectors print, so the line that closes their output carries a token it cannot predict. */ +export function buildShellProbeScript(boundary: string): string { + return [...WORKLOAD_COLLECTORS, `echo '${EVIDENCE_BOUNDARY_PREFIX}${boundary}'`, ...EVIDENCE_COLLECTORS].join("; "); +} export type ShellProbeStatus = ProviderStreamStatus | "shell_unavailable"; -export type ShellProbeResult = { status: ShellProbeStatus; output: string; exitCode?: number }; +export type ShellProbeResult = { status: ShellProbeStatus; output: string; evidence: string; exitCode?: number }; export type ShellProbeTarget = { hostUri: string; @@ -33,8 +48,8 @@ export type ShellProbeTarget = { service: string; }; -export function buildShellProbeUrl(target: ShellProbeTarget): string { - const command = ["sh", "-c", SHELL_PROBE_SCRIPT].map((part, index) => `cmd${index}=${encodeURIComponent(part)}`).join("&"); +export function buildShellProbeUrl(target: ShellProbeTarget, boundary: string): string { + const command = ["sh", "-c", buildShellProbeScript(boundary)].map((part, index) => `cmd${index}=${encodeURIComponent(part)}`).join("&"); return `${target.hostUri}/lease/${target.dseq}/${target.gseq}/${target.oseq}/shell?stdin=0&tty=0&podIndex=0&${command}&service=${encodeURIComponent(target.service)}`; } @@ -47,8 +62,9 @@ export class ProviderShellProbeService { ) {} async run(target: ShellProbeTarget): Promise { + const boundary = randomBytes(16).toString("hex"); const result = await this.providerStreamService.collect({ - url: buildShellProbeUrl(target), + url: buildShellProbeUrl(target, boundary), providerAddress: target.providerAddress, token: target.token, idleTimeoutMs: this.config.get("WORKLOAD_ABUSE_PROBE_IDLE_TIMEOUT_MS"), @@ -63,6 +79,18 @@ export class ProviderShellProbeService { const failed = result.frames.some(frame => frame.kind === "shell" && frame.stream === "failure"); const status: ShellProbeStatus = failed || (result.status === "completed" && output.length === 0) ? "shell_unavailable" : result.status; - return { status, output, exitCode: result.exitCode }; + return { status, ...splitAtEvidenceBoundary(output, boundary), exitCode: result.exitCode }; } } + +/** Cutting at the last match means a repeat of the boundary can only widen what the scanner reads, never hide part of it. */ +function splitAtEvidenceBoundary(output: string, boundary: string): { output: string; evidence: string } { + const lines = output.split("\n"); + const boundaryIndex = lines.lastIndexOf(`${EVIDENCE_BOUNDARY_PREFIX}${boundary}`); + + if (boundaryIndex === -1) return { output, evidence: "" }; + + const collected = lines.slice(0, boundaryIndex); + + return { output: collected.length ? `${collected.join("\n")}\n` : "", evidence: lines.slice(boundaryIndex + 1).join("\n") }; +} diff --git a/apps/api/src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service.spec.ts b/apps/api/src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service.spec.ts index e4a381fedb..82b6eb5309 100644 --- a/apps/api/src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service.spec.ts +++ b/apps/api/src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service.spec.ts @@ -31,7 +31,7 @@ describe(TrialWorkloadProbeService.name, () => { const report = await service.probe({ wallet, dseq: DSEQ }); - expect(report).toEqual({ verdict: "clean", signals: [], excerpt: "", probeStatus: "no_live_lease", leases: [] }); + expect(report).toEqual({ verdict: "clean", signals: [], excerpt: "", probeStatus: "no_live_lease", leases: [], shellEvidence: [] }); expect(providerService.toProviderAuth).not.toHaveBeenCalled(); }); @@ -177,6 +177,32 @@ describe(TrialWorkloadProbeService.name, () => { expect(shellProbeService.run).not.toHaveBeenCalled(); }); + it("records the evidence block the probe split off and scans only what came before it", async () => { + const evidence = [ + "--accel", + "GPU-0001, NVIDIA T4, 95, 14000, 15360", + "GPU-0001, 1234, stratum+tcp://pool.example:3333, 14000", + "--disk", + "1073741824 /tmp/stratum+tcp://pool.example:3333", + "--procorig", + "btime=1740000000", + "1234 ppid=1 starttime=650000 comm=stratum+tcp://pool.example:3333" + ].join("\n"); + const { service, wallet } = setup({ + leases: [createRpcLease()], + services: { web: 1 }, + shell: { status: "completed", output: "--loadavg\n0.10 0.20 0.30\n", evidence } + }); + + const report = await service.probe({ wallet, dseq: DSEQ }); + + expect(report.verdict).toBe("clean"); + expect(report.signals).toEqual([]); + expect(report.excerpt).not.toContain("--accel"); + expect(report.excerpt).not.toContain("stratum+tcp"); + expect(report.shellEvidence).toEqual([{ service: "web", provider: PROVIDER, status: "completed", evidence }]); + }); + function createRpcLease(overrides: Partial = {}): RpcLease { return mock({ lease: { id: { owner: "akash1owner", dseq: DSEQ, gseq: 1, oseq: 1, provider: PROVIDER, bseq: 1, ...overrides }, state: "active" } @@ -188,7 +214,7 @@ describe(TrialWorkloadProbeService.name, () => { provider?: Provider | null; services?: Record; sdl?: string | null; - shell?: ShellProbeResult; + shell?: { status: ShellProbeResult["status"]; output: string; evidence?: string }; logs?: string[]; }) { const wallet = { ...createUserWallet({ isTrialing: true }), address: "akash1owner" }; @@ -211,7 +237,7 @@ describe(TrialWorkloadProbeService.name, () => { const deploymentSettingRepository = mock(); deploymentSettingRepository.findOneBy.mockResolvedValue(mock({ sdl: input.sdl ?? null })); const shellProbeService = mock(); - shellProbeService.run.mockResolvedValue(input.shell ?? { status: "completed", output: "--loadavg\n0.10" }); + shellProbeService.run.mockResolvedValue({ evidence: "", ...(input.shell ?? { status: "completed", output: "--loadavg\n0.10" }) }); const logTailService = mock(); logTailService.collect.mockResolvedValue({ status: "completed", lines: input.logs ?? [] }); const config = mockConfigService({ diff --git a/apps/api/src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service.ts b/apps/api/src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service.ts index 3586eec2c7..31afb1c732 100644 --- a/apps/api/src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service.ts +++ b/apps/api/src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service.ts @@ -37,8 +37,11 @@ export type ProbeReport = { excerpt: string; probeStatus: ProbeStatus; leases: ProbedLease[]; + shellEvidence: ShellEvidence[]; }; +export type ShellEvidence = { service: string; provider: string; status: ShellProbeStatus; evidence: string }; + const PROVIDER_SCOPES = ["status", "logs", "shell"] as const; /** The probed provider reports its own service list, so a hostile one must not be able to stretch a run past this many shell sessions. */ const MAX_PROBED_SERVICES_PER_LEASE = 8; @@ -69,10 +72,11 @@ export class TrialWorkloadProbeService { const leases = await this.#findLiveLeases(input.wallet.address, input.dseq); if (leases.length === 0) { - return { verdict: "clean", signals: [], excerpt: "", probeStatus: "no_live_lease", leases: [] }; + return { verdict: "clean", signals: [], excerpt: "", probeStatus: "no_live_lease", leases: [], shellEvidence: [] }; } const sources: EvidenceSource[] = []; + const shellEvidence: ShellEvidence[] = []; const setting = await this.deploymentSettingRepository.findOneBy({ userId: input.wallet.userId, dseq: input.dseq }); if (setting?.sdl) sources.push({ kind: "sdl", text: setting.sdl }); @@ -91,7 +95,7 @@ export class TrialWorkloadProbeService { } for (const lease of leasesToProbe) { - const probed = await this.#probeLease(input.wallet, lease, sources); + const probed = await this.#probeLease(input.wallet, lease, sources, shellEvidence); statuses.push(probed.status); if (probed.lease) probedLeases.push(probed.lease); } @@ -103,7 +107,8 @@ export class TrialWorkloadProbeService { signals, excerpt: buildExcerpt(sources, signals), probeStatus: statuses.includes("probed") ? "probed" : statuses[0] ?? "stream_failed", - leases: probedLeases + leases: probedLeases, + shellEvidence }; } @@ -113,7 +118,12 @@ export class TrialWorkloadProbeService { return responses.flatMap(response => response.leases); } - async #probeLease(wallet: WalletInitialized, lease: RpcLease, sources: EvidenceSource[]): Promise<{ status: ProbeStatus; lease?: ProbedLease }> { + async #probeLease( + wallet: WalletInitialized, + lease: RpcLease, + sources: EvidenceSource[], + shellEvidence: ShellEvidence[] + ): Promise<{ status: ProbeStatus; lease?: ProbedLease }> { const { provider: providerAddress, dseq, gseq, oseq } = lease.lease.id; const provider = await this.providerRepository.findActiveByAddress(providerAddress); @@ -148,6 +158,9 @@ export class TrialWorkloadProbeService { const shell = await this.shellProbeService.run({ ...target, service }); probedLease.shellStatuses.push(shell.status); if (shell.output) sources.push({ kind: "shell", service, text: shell.output }); + if (shell.output || shell.evidence) { + shellEvidence.push({ service, provider: providerAddress, status: shell.status, evidence: shell.evidence }); + } } const logs = await this.logTailService.collect({ ...target, services: probedServices }); diff --git a/apps/api/src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service.ts b/apps/api/src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service.ts index 6307f13c87..8de0d3b7d6 100644 --- a/apps/api/src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service.ts +++ b/apps/api/src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service.ts @@ -6,6 +6,8 @@ import type { WorkloadVerdict } from "@src/workload-abuse/lib/evidence-scanner/e export type DomainBlockResult = "blocked" | "raced" | "skipped" | "dry_run" | "failed" | "sibling_limit_reached"; +export type EvidenceWriteOperation = "insert" | "purge"; + @singleton() export class WorkloadAbuseInstrumentationService { private readonly meter: Meter; @@ -14,6 +16,7 @@ export class WorkloadAbuseInstrumentationService { private readonly enforcements: Counter; private readonly blockedDomainLookupFailures: Counter; private readonly domainBlocks: Counter; + private readonly evidenceWriteFailures: Counter; constructor(metricsService: MetricsService) { this.meter = metricsService.getMeter("workload-abuse", "1.0.0"); @@ -32,6 +35,9 @@ export class WorkloadAbuseInstrumentationService { this.domainBlocks = metricsService.createCounter(this.meter, "workload_abuse_domain_blocks_total", { description: "Email domain auto-block outcomes, by result and (on a skip) reason" }); + this.evidenceWriteFailures = metricsService.createCounter(this.meter, "workload_abuse_evidence_write_failures_total", { + description: "Probe evidence statements that failed to persist, by operation" + }); } recordProbe(input: { verdict: WorkloadVerdict; probeStatus: string }): void { @@ -53,4 +59,8 @@ export class WorkloadAbuseInstrumentationService { recordDomainBlock(result: DomainBlockResult, reason?: string): void { this.domainBlocks.add(1, reason ? { result, reason } : { result }); } + + recordEvidenceWriteFailure(operation: EvidenceWriteOperation): void { + this.evidenceWriteFailures.add(1, { operation }); + } }