From 1e8633a63bb25eb5cc19365c8e3f2d752eda363c Mon Sep 17 00:00:00 2001 From: Maxime Beauchamp <15185355+baktun14@users.noreply.github.com> Date: Sun, 20 Sep 2026 15:00:08 +0400 Subject: [PATCH 1/6] feat(deployment): require multi-probe agreement before behavioural enforcement A repeated shape can now stand on its own: when consecutive probes of one service see it and the streak spans real time rather than minutes, the probe records a detection and hands the deployment to the wipe that already exists. A probe that collected nothing steps aside rather than resetting the streak. Those detections carry a verdict value of their own so the audit trail keeps them apart from a string match, and the deployment stops being re-probed once one lands. The flag is off, so nothing changes until it is set. --- ...059_workload_abuse_behavioural_verdict.sql | 1 + apps/api/drizzle/meta/0059_snapshot.json | 2054 +++++++++++++++++ apps/api/drizzle/meta/_journal.json | 7 + .../workload-abuse/config/env.config.spec.ts | 8 + .../src/workload-abuse/config/env.config.ts | 9 +- .../lib/behavioural-signals/agreement.spec.ts | 76 + .../lib/behavioural-signals/agreement.ts | 72 + .../workload-abuse-detection.schema.ts | 2 +- ...-abuse-detection.repository.integration.ts | 15 +- .../workload-abuse-detection.repository.ts | 4 +- ...be-trial-deployment.handler.integration.ts | 50 +- .../probe-trial-deployment.handler.spec.ts | 88 +- .../probe-trial-deployment.handler.ts | 62 + .../trial-workload-probe-job.service.spec.ts | 4 +- .../trial-workload-probe-job.service.ts | 2 +- .../workload-abuse-instrumentation.service.ts | 4 +- 16 files changed, 2444 insertions(+), 14 deletions(-) create mode 100644 apps/api/drizzle/0059_workload_abuse_behavioural_verdict.sql create mode 100644 apps/api/drizzle/meta/0059_snapshot.json create mode 100644 apps/api/src/workload-abuse/lib/behavioural-signals/agreement.spec.ts create mode 100644 apps/api/src/workload-abuse/lib/behavioural-signals/agreement.ts diff --git a/apps/api/drizzle/0059_workload_abuse_behavioural_verdict.sql b/apps/api/drizzle/0059_workload_abuse_behavioural_verdict.sql new file mode 100644 index 0000000000..ca019dbb0c --- /dev/null +++ b/apps/api/drizzle/0059_workload_abuse_behavioural_verdict.sql @@ -0,0 +1 @@ +ALTER TYPE "public"."workload_abuse_verdict" ADD VALUE 'behavioural'; \ No newline at end of file diff --git a/apps/api/drizzle/meta/0059_snapshot.json b/apps/api/drizzle/meta/0059_snapshot.json new file mode 100644 index 0000000000..944a8a7ca7 --- /dev/null +++ b/apps/api/drizzle/meta/0059_snapshot.json @@ -0,0 +1,2054 @@ +{ + "id": "15591fc4-d0c0-4a54-8ea0-c72e70babfcc", + "prevId": "63bba39e-51aa-48a3-a83b-b019a98795f2", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.user_wallets": { + "name": "user_wallets", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "varchar", + "primaryKey": false, + "notNull": false + }, + "deployment_allowance": { + "name": "deployment_allowance", + "type": "numeric(20, 2)", + "primaryKey": false, + "notNull": true, + "default": "'0.00'" + }, + "fee_allowance": { + "name": "fee_allowance", + "type": "numeric(20, 2)", + "primaryKey": false, + "notNull": true, + "default": "'0.00'" + }, + "trial": { + "name": "trial", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": true + }, + "activated_at": { + "name": "activated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "credits_low_notified_at": { + "name": "credits_low_notified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "credits_sufficient_since": { + "name": "credits_sufficient_since", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "credits_low_since": { + "name": "credits_low_since", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "abuse_locked_at": { + "name": "abuse_locked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "abuse_locked_reason": { + "name": "abuse_locked_reason", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "user_wallets_user_id_userSetting_id_fk": { + "name": "user_wallets_user_id_userSetting_id_fk", + "tableFrom": "user_wallets", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_wallets_user_id_unique": { + "name": "user_wallets_user_id_unique", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + }, + "user_wallets_address_unique": { + "name": "user_wallets_address_unique", + "nullsNotDistinct": false, + "columns": [ + "address" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.payment_methods": { + "name": "payment_methods", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "fingerprint": { + "name": "fingerprint", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "payment_method_id": { + "name": "payment_method_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "is_validated": { + "name": "is_validated", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "payment_methods_fingerprint_payment_method_id_unique": { + "name": "payment_methods_fingerprint_payment_method_id_unique", + "columns": [ + { + "expression": "fingerprint", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "payment_method_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "payment_methods_user_id_is_default_unique": { + "name": "payment_methods_user_id_is_default_unique", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_default", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"payment_methods\".\"is_default\" = true", + "concurrently": false, + "method": "btree", + "with": {} + }, + "payment_methods_fingerprint_idx": { + "name": "payment_methods_fingerprint_idx", + "columns": [ + { + "expression": "fingerprint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "payment_methods_user_id_idx": { + "name": "payment_methods_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "payment_methods_user_id_is_validated_idx": { + "name": "payment_methods_user_id_is_validated_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_validated", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "payment_methods_user_id_fingerprint_payment_method_id_idx": { + "name": "payment_methods_user_id_fingerprint_payment_method_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "fingerprint", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "payment_method_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "payment_methods_user_id_userSetting_id_fk": { + "name": "payment_methods_user_id_userSetting_id_fk", + "tableFrom": "payment_methods", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.stripe_transactions": { + "name": "stripe_transactions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "stripe_transaction_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "stripe_transaction_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'created'" + }, + "amount": { + "name": "amount", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "amount_refunded": { + "name": "amount_refunded", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "bonus_amount": { + "name": "bonus_amount", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "currency": { + "name": "currency", + "type": "varchar(3)", + "primaryKey": false, + "notNull": true, + "default": "'usd'" + }, + "stripe_payment_intent_id": { + "name": "stripe_payment_intent_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "stripe_charge_id": { + "name": "stripe_charge_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "stripe_coupon_id": { + "name": "stripe_coupon_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "stripe_promotion_code_id": { + "name": "stripe_promotion_code_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "stripe_invoice_id": { + "name": "stripe_invoice_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "stripe_idempotency_key": { + "name": "stripe_idempotency_key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "payment_method_type": { + "name": "payment_method_type", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "card_brand": { + "name": "card_brand", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "card_last4": { + "name": "card_last4", + "type": "varchar(4)", + "primaryKey": false, + "notNull": false + }, + "receipt_url": { + "name": "receipt_url", + "type": "varchar(2048)", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "varchar(500)", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "varchar(1000)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "stripe_transactions_stripe_invoice_id_unique": { + "name": "stripe_transactions_stripe_invoice_id_unique", + "columns": [ + { + "expression": "stripe_invoice_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"stripe_transactions\".\"stripe_invoice_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_stripe_idempotency_key_unique": { + "name": "stripe_transactions_stripe_idempotency_key_unique", + "columns": [ + { + "expression": "stripe_idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"stripe_transactions\".\"stripe_idempotency_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_user_id_idx": { + "name": "stripe_transactions_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_stripe_payment_intent_id_idx": { + "name": "stripe_transactions_stripe_payment_intent_id_idx", + "columns": [ + { + "expression": "stripe_payment_intent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_stripe_charge_id_idx": { + "name": "stripe_transactions_stripe_charge_id_idx", + "columns": [ + { + "expression": "stripe_charge_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_stripe_coupon_id_idx": { + "name": "stripe_transactions_stripe_coupon_id_idx", + "columns": [ + { + "expression": "stripe_coupon_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_stripe_promotion_code_id_idx": { + "name": "stripe_transactions_stripe_promotion_code_id_idx", + "columns": [ + { + "expression": "stripe_promotion_code_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_status_idx": { + "name": "stripe_transactions_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_created_at_idx": { + "name": "stripe_transactions_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_user_id_created_at_idx": { + "name": "stripe_transactions_user_id_created_at_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "stripe_transactions_user_id_userSetting_id_fk": { + "name": "stripe_transactions_user_id_userSetting_id_fk", + "tableFrom": "stripe_transactions", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.wallet_settings": { + "name": "wallet_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "wallet_id": { + "name": "wallet_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "auto_reload_enabled": { + "name": "auto_reload_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "auto_reload_mode": { + "name": "auto_reload_mode", + "type": "auto_reload_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'prediction'" + }, + "auto_reload_threshold": { + "name": "auto_reload_threshold", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 2000 + }, + "auto_reload_amount": { + "name": "auto_reload_amount", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 10000 + }, + "last_auto_charge_at": { + "name": "last_auto_charge_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "auto_reload_failure_count": { + "name": "auto_reload_failure_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "auto_reload_paused_at": { + "name": "auto_reload_paused_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "wallet_settings_user_id_idx": { + "name": "wallet_settings_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "wallet_settings_wallet_id_user_wallets_id_fk": { + "name": "wallet_settings_wallet_id_user_wallets_id_fk", + "tableFrom": "wallet_settings", + "tableTo": "user_wallets", + "columnsFrom": [ + "wallet_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "wallet_settings_user_id_userSetting_id_fk": { + "name": "wallet_settings_user_id_userSetting_id_fk", + "tableFrom": "wallet_settings", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "wallet_settings_wallet_id_unique": { + "name": "wallet_settings_wallet_id_unique", + "nullsNotDistinct": false, + "columns": [ + "wallet_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.userSetting": { + "name": "userSetting", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "userId": { + "name": "userId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "username": { + "name": "username", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "emailVerified": { + "name": "emailVerified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "stripeCustomerId": { + "name": "stripeCustomerId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "bio": { + "name": "bio", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "subscribedToNewsletter": { + "name": "subscribedToNewsletter", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "youtubeUsername": { + "name": "youtubeUsername", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "twitterUsername": { + "name": "twitterUsername", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "githubUsername": { + "name": "githubUsername", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "last_active_at": { + "name": "last_active_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "last_ip": { + "name": "last_ip", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "last_user_agent": { + "name": "last_user_agent", + "type": "varchar(500)", + "primaryKey": false, + "notNull": false + }, + "last_fingerprint": { + "name": "last_fingerprint", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "onboardingSkippedAt": { + "name": "onboardingSkippedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "fair_use_policy_accepted_at": { + "name": "fair_use_policy_accepted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "userSetting_userId_unique": { + "name": "userSetting_userId_unique", + "nullsNotDistinct": false, + "columns": [ + "userId" + ] + }, + "userSetting_username_unique": { + "name": "userSetting_username_unique", + "nullsNotDistinct": false, + "columns": [ + "username" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.template": { + "name": "template", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "userId": { + "name": "userId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "copiedFromId": { + "name": "copiedFromId", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "isPublic": { + "name": "isPublic", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "cpu": { + "name": "cpu", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "ram": { + "name": "ram", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "storage": { + "name": "storage", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "sdl": { + "name": "sdl", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "template_userId_idx": { + "name": "template_userId_idx", + "columns": [ + { + "expression": "userId", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.templateFavorite": { + "name": "templateFavorite", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "userId": { + "name": "userId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "templateId": { + "name": "templateId", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "addedDate": { + "name": "addedDate", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "templateFavorite_userId_templateId_unique": { + "name": "templateFavorite_userId_templateId_unique", + "columns": [ + { + "expression": "userId", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "templateId", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "templateFavorite_templateId_template_id_fk": { + "name": "templateFavorite_templateId_template_id_fk", + "tableFrom": "templateFavorite", + "tableTo": "template", + "columnsFrom": [ + "templateId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.deployment_settings": { + "name": "deployment_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "dseq": { + "name": "dseq", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "auto_top_up_enabled": { + "name": "auto_top_up_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "closed": { + "name": "closed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "last_funded_at": { + "name": "last_funded_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "runtime_limit_hours": { + "name": "runtime_limit_hours", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "sdl": { + "name": "sdl", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sealed_secrets": { + "name": "sealed_secrets", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "manifest_version": { + "name": "manifest_version", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "runtime_ends_at": { + "name": "runtime_ends_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "runtime_ending_notified_for": { + "name": "runtime_ending_notified_for", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "provider_unreachable_notified_for": { + "name": "provider_unreachable_notified_for", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "id_auto_top_up_enabled_closed_idx": { + "name": "id_auto_top_up_enabled_closed_idx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "auto_top_up_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "closed", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_id_id_sealed_secrets_idx": { + "name": "user_id_id_sealed_secrets_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"deployment_settings\".\"sealed_secrets\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "deployment_settings_user_id_userSetting_id_fk": { + "name": "deployment_settings_user_id_userSetting_id_fk", + "tableFrom": "deployment_settings", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "dseq_user_id_idx": { + "name": "dseq_user_id_idx", + "nullsNotDistinct": false, + "columns": [ + "dseq", + "user_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.api_keys": { + "name": "api_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "hashed_key": { + "name": "hashed_key", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "key_format": { + "name": "key_format", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "api_keys_user_id_userSetting_id_fk": { + "name": "api_keys_user_id_userSetting_id_fk", + "tableFrom": "api_keys", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "api_keys_hashed_key_unique": { + "name": "api_keys_hashed_key_unique", + "nullsNotDistinct": false, + "columns": [ + "hashed_key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.email_verification_codes": { + "name": "email_verification_codes", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "code": { + "name": "code", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "email_verification_codes_user_id_idx": { + "name": "email_verification_codes_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "email_verification_codes_user_id_userSetting_id_fk": { + "name": "email_verification_codes_user_id_userSetting_id_fk", + "tableFrom": "email_verification_codes", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_keys": { + "name": "data_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "wrapped_key": { + "name": "wrapped_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "wrapped_by_kid": { + "name": "wrapped_by_kid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "retired_at": { + "name": "retired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_keys_active_user_id_idx": { + "name": "data_keys_active_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_keys\".\"retired_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_keys_wrapped_by_kid_idx": { + "name": "data_keys_wrapped_by_kid_idx", + "columns": [ + { + "expression": "wrapped_by_kid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_keys_user_id_userSetting_id_fk": { + "name": "data_keys_user_id_userSetting_id_fk", + "tableFrom": "data_keys", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.blocked_email_domains": { + "name": "blocked_email_domains", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "domain": { + "name": "domain", + "type": "varchar(253)", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "blocked_email_domain_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'blocked'" + }, + "source": { + "name": "source", + "type": "blocked_email_domain_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'manual'" + }, + "reason": { + "name": "reason", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "triggered_by_user_id": { + "name": "triggered_by_user_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "blocked_email_domains_domain_unique": { + "name": "blocked_email_domains_domain_unique", + "columns": [ + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "blocked_email_domains_triggered_by_user_id_userSetting_id_fk": { + "name": "blocked_email_domains_triggered_by_user_id_userSetting_id_fk", + "tableFrom": "blocked_email_domains", + "tableTo": "userSetting", + "columnsFrom": [ + "triggered_by_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "blocked_email_domains_domain_normalized": { + "name": "blocked_email_domains_domain_normalized", + "value": "\"blocked_email_domains\".\"domain\" ~ '^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?(?:\\.[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)+$'" + } + }, + "isRLSEnabled": false + }, + "public.workload_abuse_detections": { + "name": "workload_abuse_detections", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "wallet_id": { + "name": "wallet_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "dseq": { + "name": "dseq", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "verdict": { + "name": "verdict", + "type": "workload_abuse_verdict", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "probe_status": { + "name": "probe_status", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "signals": { + "name": "signals", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "evidence_excerpt": { + "name": "evidence_excerpt", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "workload_abuse_action", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'detected'" + }, + "enforcement_error": { + "name": "enforcement_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workload_abuse_detections_user_id_idx": { + "name": "workload_abuse_detections_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workload_abuse_detections_dseq_idx": { + "name": "workload_abuse_detections_dseq_idx", + "columns": [ + { + "expression": "dseq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workload_abuse_detections_user_id_userSetting_id_fk": { + "name": "workload_abuse_detections_user_id_userSetting_id_fk", + "tableFrom": "workload_abuse_detections", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workload_probe_evidence": { + "name": "workload_probe_evidence", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "wallet_id": { + "name": "wallet_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "dseq": { + "name": "dseq", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service": { + "name": "service", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "probe_status": { + "name": "probe_status", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "verdict": { + "name": "verdict", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true + }, + "detection_id": { + "name": "detection_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "accelerator": { + "name": "accelerator", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "process_origins": { + "name": "process_origins", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "net_shape": { + "name": "net_shape", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "behavioural_findings": { + "name": "behavioural_findings", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workload_probe_evidence_wallet_dseq_created_idx": { + "name": "workload_probe_evidence_wallet_dseq_created_idx", + "columns": [ + { + "expression": "wallet_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dseq", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workload_probe_evidence_created_idx": { + "name": "workload_probe_evidence_created_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.stripe_transaction_status": { + "name": "stripe_transaction_status", + "schema": "public", + "values": [ + "created", + "pending", + "requires_action", + "succeeded", + "failed", + "refunded", + "canceled" + ] + }, + "public.stripe_transaction_type": { + "name": "stripe_transaction_type", + "schema": "public", + "values": [ + "payment_intent", + "coupon_claim", + "manual_credit" + ] + }, + "public.auto_reload_mode": { + "name": "auto_reload_mode", + "schema": "public", + "values": [ + "prediction", + "threshold" + ] + }, + "public.blocked_email_domain_source": { + "name": "blocked_email_domain_source", + "schema": "public", + "values": [ + "auto", + "manual" + ] + }, + "public.blocked_email_domain_status": { + "name": "blocked_email_domain_status", + "schema": "public", + "values": [ + "blocked", + "allowed" + ] + }, + "public.workload_abuse_action": { + "name": "workload_abuse_action", + "schema": "public", + "values": [ + "detected", + "enforcing", + "enforced", + "enforcement_failed" + ] + }, + "public.workload_abuse_verdict": { + "name": "workload_abuse_verdict", + "schema": "public", + "values": [ + "hard", + "soft", + "proxy", + "behavioural" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/apps/api/drizzle/meta/_journal.json b/apps/api/drizzle/meta/_journal.json index 4eb242cd4a..67948e3926 100644 --- a/apps/api/drizzle/meta/_journal.json +++ b/apps/api/drizzle/meta/_journal.json @@ -414,6 +414,13 @@ "when": 1789839246549, "tag": "0058_workload_probe_evidence", "breakpoints": true + }, + { + "idx": 59, + "version": "7", + "when": 1789901600865, + "tag": "0059_workload_abuse_behavioural_verdict", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/api/src/workload-abuse/config/env.config.spec.ts b/apps/api/src/workload-abuse/config/env.config.spec.ts index c4858948a9..99dc04075b 100644 --- a/apps/api/src/workload-abuse/config/env.config.spec.ts +++ b/apps/api/src/workload-abuse/config/env.config.spec.ts @@ -36,6 +36,14 @@ describe("workload abuse env config", () => { expect(config.WORKLOAD_ABUSE_SIGNAL_ARTIFACT_MIN_MB).toBe(256); }); + it("defaults behavioural enforcement to off behind a multi-probe agreement", () => { + const config = envSchema.parse({}); + + expect(config.WORKLOAD_ABUSE_BEHAVIOURAL_ENFORCEMENT_ENABLED).toBe(false); + expect(config.WORKLOAD_ABUSE_BEHAVIOURAL_AGREEMENT_PROBES).toBe(3); + expect(config.WORKLOAD_ABUSE_BEHAVIOURAL_MIN_WINDOW_MINUTES).toBe(120); + }); + it("parses the relay endpoint list", () => { const config = envSchema.parse({ WORKLOAD_ABUSE_SIGNAL_RELAY_ENDPOINTS: JSON.stringify(["10.0.0.7", "10.0.0.8:443"]) }); diff --git a/apps/api/src/workload-abuse/config/env.config.ts b/apps/api/src/workload-abuse/config/env.config.ts index 6d6fd14e7f..c4df360e16 100644 --- a/apps/api/src/workload-abuse/config/env.config.ts +++ b/apps/api/src/workload-abuse/config/env.config.ts @@ -121,7 +121,14 @@ export const envSchema = z.object({ WORKLOAD_ABUSE_SIGNAL_ACCEL_MIN_VRAM_MB: z.preprocess(blankToUndefined, z.number({ coerce: true }).int().positive().default(1_024)), WORKLOAD_ABUSE_SIGNAL_ARTIFACT_MIN_MB: z.preprocess(blankToUndefined, z.number({ coerce: true }).int().positive().default(256)), /** The list itself lives in Doppler: these are our own endpoints, so a reader learns how the exclusion works but not what it covers. */ - WORKLOAD_ABUSE_SIGNAL_RELAY_ENDPOINTS: z.preprocess(blankToUndefined, z.string().default("[]").transform(parseRelayEndpoints)) + WORKLOAD_ABUSE_SIGNAL_RELAY_ENDPOINTS: z.preprocess(blankToUndefined, z.string().default("[]").transform(parseRelayEndpoints)), + WORKLOAD_ABUSE_BEHAVIOURAL_ENFORCEMENT_ENABLED: z + .enum(["true", "false"]) + .default("false") + .transform(value => value === "true"), + WORKLOAD_ABUSE_BEHAVIOURAL_AGREEMENT_PROBES: z.number({ coerce: true }).int().positive().default(3), + /** Back-to-back probes can land minutes apart, so a streak also has to span real time before it counts. */ + WORKLOAD_ABUSE_BEHAVIOURAL_MIN_WINDOW_MINUTES: z.number({ coerce: true }).int().positive().default(120) }); export type WorkloadAbuseConfig = z.infer; diff --git a/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.spec.ts b/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.spec.ts new file mode 100644 index 0000000000..865ce3dc9c --- /dev/null +++ b/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.spec.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from "vitest"; + +import { type AgreementRow, findBehaviouralAgreement } from "./agreement"; +import type { BehaviouralFinding } from "./types"; + +const NOW = new Date("2026-09-20T12:00:00.000Z"); + +const BOTH_SIGNALS: BehaviouralFinding[] = [ + { signal: "accel_without_artifacts", detail: {} }, + { signal: "network_isolated", detail: {} } +]; + +describe("findBehaviouralAgreement", () => { + it("agrees once enough consecutive probes see the same shape over a long enough window", () => { + const { rows, params } = setup({ minutesAgo: [10, 130, 250] }); + + expect(findBehaviouralAgreement(rows, params)).toEqual({ agreed: true, service: "web", streak: 3, spanMinutes: 250 }); + }); + + it("holds off while the streak is one probe short", () => { + const { rows, params } = setup({ minutesAgo: [10, 130] }); + + expect(findBehaviouralAgreement(rows, params)).toMatchObject({ agreed: false, streak: 2 }); + }); + + it("holds off while the streak is younger than the minimum window", () => { + const { rows, params } = setup({ minutesAgo: [1, 3, 5] }); + + expect(findBehaviouralAgreement(rows, params)).toMatchObject({ agreed: false, streak: 3, spanMinutes: 5 }); + }); + + it("breaks the streak on the first probe that saw a different shape", () => { + const { rows, params } = setup({ minutesAgo: [10, 130, 250], findingsByIndex: { 1: [{ signal: "network_isolated", detail: {} }] } }); + + expect(findBehaviouralAgreement(rows, params)).toMatchObject({ agreed: false, streak: 1 }); + }); + + it("steps over a probe that collected nothing without breaking the streak", () => { + const { rows, params } = setup({ minutesAgo: [10, 130, 250, 320], statusByIndex: { 1: "shell_unavailable" } }); + + expect(findBehaviouralAgreement(rows, params)).toMatchObject({ agreed: true, streak: 3, spanMinutes: 320 }); + }); + + it("agrees on the service that holds the streak rather than on the deployment as a whole", () => { + const { rows, params } = setup({ minutesAgo: [10, 130, 250] }); + const sidecar = rows.map(row => ({ ...row, service: "sidecar", behaviouralFindings: null })); + + expect(findBehaviouralAgreement([...rows, ...sidecar], params)).toMatchObject({ agreed: true, service: "web", streak: 3 }); + }); + + it("finds no agreement in an empty history", () => { + const { params } = setup({ minutesAgo: [] }); + + expect(findBehaviouralAgreement([], params)).toEqual({ agreed: false, service: null, streak: 0, spanMinutes: 0 }); + }); + + function setup(input: { + minutesAgo: number[]; + agreementProbes?: number; + minWindowMinutes?: number; + findingsByIndex?: Record; + statusByIndex?: Record; + }) { + const rows: AgreementRow[] = input.minutesAgo.map((minutes, index) => ({ + service: "web", + createdAt: new Date(NOW.getTime() - minutes * 60_000), + probeStatus: input.statusByIndex?.[index] ?? "probed", + behaviouralFindings: input.findingsByIndex?.[index] ?? BOTH_SIGNALS + })); + + return { + rows, + params: { agreementProbes: input.agreementProbes ?? 3, minWindowMinutes: input.minWindowMinutes ?? 120, now: NOW } + }; + } +}); diff --git a/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.ts b/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.ts new file mode 100644 index 0000000000..7f1e37b362 --- /dev/null +++ b/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.ts @@ -0,0 +1,72 @@ +import { isBehaviouralCandidate } from "./evaluate-behavioural-signals"; +import type { BehaviouralFinding } from "./types"; + +const COLLECTED_PROBE_STATUS = "probed"; +const MS_PER_MINUTE = 60_000; + +export type AgreementRow = { + service: string; + createdAt: Date; + probeStatus: string; + behaviouralFindings: BehaviouralFinding[] | null; +}; + +export type BehaviouralAgreement = { + agreed: boolean; + service: string | null; + streak: number; + spanMinutes: number; +}; + +export type AgreementParams = { + agreementProbes: number; + minWindowMinutes: number; + now?: Date; +}; + +/** A probe that could not collect anything neither advances nor breaks the streak, because a flaking provider would otherwise put agreement out of reach. */ +export function findBehaviouralAgreement(rows: AgreementRow[], params: AgreementParams): BehaviouralAgreement { + const now = params.now ?? new Date(); + const perService = [...groupByService(rows).values()].map(serviceRows => measureService(serviceRows, params, now)); + + return perService.reduce(pickStrongest, { agreed: false, service: null, streak: 0, spanMinutes: 0 }); +} + +function groupByService(rows: AgreementRow[]): Map { + const grouped = new Map(); + + for (const row of rows) { + const serviceRows = grouped.get(row.service) ?? []; + serviceRows.push(row); + grouped.set(row.service, serviceRows); + } + + return grouped; +} + +function measureService(rows: AgreementRow[], params: AgreementParams, now: Date): BehaviouralAgreement { + const newestFirst = [...rows].sort((left, right) => right.createdAt.getTime() - left.createdAt.getTime()); + const streak: AgreementRow[] = []; + + for (const row of newestFirst) { + if (row.probeStatus !== COLLECTED_PROBE_STATUS) continue; + if (!isBehaviouralCandidate(row.behaviouralFindings ?? [])) break; + streak.push(row); + } + + const oldest = streak.at(-1); + const spanMinutes = oldest ? Math.floor((now.getTime() - oldest.createdAt.getTime()) / MS_PER_MINUTE) : 0; + + return { + agreed: streak.length >= params.agreementProbes && spanMinutes >= params.minWindowMinutes, + service: newestFirst[0]?.service ?? null, + streak: streak.length, + spanMinutes + }; +} + +function pickStrongest(strongest: BehaviouralAgreement, candidate: BehaviouralAgreement): BehaviouralAgreement { + if (candidate.agreed !== strongest.agreed) return candidate.agreed ? candidate : strongest; + + return candidate.streak > strongest.streak ? candidate : strongest; +} diff --git a/apps/api/src/workload-abuse/model-schemas/workload-abuse-detection/workload-abuse-detection.schema.ts b/apps/api/src/workload-abuse/model-schemas/workload-abuse-detection/workload-abuse-detection.schema.ts index 208008a367..bb34a9ad08 100644 --- a/apps/api/src/workload-abuse/model-schemas/workload-abuse-detection/workload-abuse-detection.schema.ts +++ b/apps/api/src/workload-abuse/model-schemas/workload-abuse-detection/workload-abuse-detection.schema.ts @@ -4,7 +4,7 @@ import { index, integer, jsonb, pgEnum, pgTable, text, timestamp, uuid, varchar import { Users } from "@src/user/model-schemas"; import type { DetectionSignal } from "@src/workload-abuse/lib/evidence-scanner/evidence-scanner"; -export const workloadAbuseVerdictEnum = pgEnum("workload_abuse_verdict", ["hard", "soft", "proxy"]); +export const workloadAbuseVerdictEnum = pgEnum("workload_abuse_verdict", ["hard", "soft", "proxy", "behavioural"]); export const workloadAbuseActionEnum = pgEnum("workload_abuse_action", ["detected", "enforcing", "enforced", "enforcement_failed"]); diff --git a/apps/api/src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository.integration.ts b/apps/api/src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository.integration.ts index 42931d9348..307fee154e 100644 --- a/apps/api/src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository.integration.ts +++ b/apps/api/src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository.integration.ts @@ -7,7 +7,7 @@ import { UserRepository } from "@src/user/repositories"; import { WorkloadAbuseDetectionRepository } from "./workload-abuse-detection.repository"; describe(WorkloadAbuseDetectionRepository.name, () => { - describe("findRecentHardTargets", () => { + describe("findRecentDetectedTargets", () => { it("lists each deployment with a confirmed detection inside the window once", async () => { const { repository, walletId, createDetection } = await setup(); await createDetection({ dseq: "1", verdict: "hard" }); @@ -15,7 +15,16 @@ describe(WorkloadAbuseDetectionRepository.name, () => { await createDetection({ dseq: "2", verdict: "soft" }); await createDetection({ dseq: "3", verdict: "hard", createdAt: subHours(new Date(), 30) }); - const targets = await repository.findRecentHardTargets({ since: subHours(new Date(), 26) }); + const targets = await repository.findRecentDetectedTargets({ since: subHours(new Date(), 26) }); + + expect(targets.filter(target => target.walletId === walletId)).toEqual([{ walletId, dseq: "1" }]); + }); + + it("lists a deployment confirmed by repeated behaviour alongside the rest", async () => { + const { repository, walletId, createDetection } = await setup(); + await createDetection({ dseq: "1", verdict: "behavioural" }); + + const targets = await repository.findRecentDetectedTargets({ since: subHours(new Date(), 26) }); expect(targets.filter(target => target.walletId === walletId)).toEqual([{ walletId, dseq: "1" }]); }); @@ -46,7 +55,7 @@ describe(WorkloadAbuseDetectionRepository.name, () => { async function createDetection(input: { dseq: string; - verdict: "hard" | "soft" | "proxy"; + verdict: "hard" | "soft" | "proxy" | "behavioural"; action?: "detected" | "enforcing" | "enforced" | "enforcement_failed"; createdAt?: Date; }) { diff --git a/apps/api/src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository.ts b/apps/api/src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository.ts index 180f201201..0a5a8180e3 100644 --- a/apps/api/src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository.ts +++ b/apps/api/src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository.ts @@ -24,11 +24,11 @@ export class WorkloadAbuseDetectionRepository extends BaseRepository> { + async findRecentDetectedTargets({ since }: { since: Date }): Promise> { return await this.cursor .selectDistinct({ walletId: this.table.walletId, dseq: this.table.dseq }) .from(this.table) - .where(and(eq(this.table.verdict, "hard"), gt(this.table.createdAt, since))); + .where(and(inArray(this.table.verdict, ["hard", "behavioural"]), gt(this.table.createdAt, since))); } /** One detection per wallet, since the wipe covers the whole wallet whichever of its detections re-queues it. */ diff --git a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.integration.ts b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.integration.ts index dd46ba05e8..05f92a31d4 100644 --- a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.integration.ts +++ b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.integration.ts @@ -142,6 +142,36 @@ describe(ProbeTrialDeploymentHandler.name, () => { expect(evidence.behaviouralFindings).toBeNull(); }); + it("confirms the deployment and stops probing once the recorded shape agrees across probes", async () => { + const { probeDeployment, findDetections, findNextProbe, seedShapeHistory } = await setup({ + verdict: "clean", + behaviouralSignalsEnabled: true, + behaviouralEnforcementEnabled: true, + shellOutputs: [{ service: "ssh", provider: "akash1provider", output: ISOLATED_ACCELERATED_SHELL_OUTPUT }] + }); + await seedShapeHistory([130, 250]); + + await probeDeployment(); + + expect(await findDetections()).toMatchObject([{ verdict: "behavioural", probeStatus: "probed" }]); + expect(await findNextProbe()).toBeUndefined(); + }); + + it("keeps probing while the recorded shape is one probe short of agreement", async () => { + const { probeDeployment, findDetections, findNextProbe, seedShapeHistory } = await setup({ + verdict: "clean", + behaviouralSignalsEnabled: true, + behaviouralEnforcementEnabled: true, + shellOutputs: [{ service: "ssh", provider: "akash1provider", output: ISOLATED_ACCELERATED_SHELL_OUTPUT }] + }); + await seedShapeHistory([130]); + + await probeDeployment(); + + expect(await findDetections()).toHaveLength(0); + expect((await findNextProbe())?.data).toMatchObject({ attempt: 2 }); + }); + it("records nothing more for a deployment already judged abusive", async () => { const { probeDeployment, findDetections, probe, seedExistingDetection } = await setup({ verdict: "hard" }); await seedExistingDetection(); @@ -159,6 +189,7 @@ describe(ProbeTrialDeploymentHandler.name, () => { probeStatus?: ProbeReport["probeStatus"]; shellEvidence?: ProbeReport["shellEvidence"]; behaviouralSignalsEnabled?: boolean; + behaviouralEnforcementEnabled?: boolean; }) { const { enqueue, startWorkers } = await jobWorkers(); const detectionRepository = container.resolve(WorkloadAbuseDetectionRepository); @@ -170,7 +201,8 @@ describe(ProbeTrialDeploymentHandler.name, () => { const readConfig = config.get.bind(config); const overrides: Record = { WORKLOAD_ABUSE_PROBE_ENABLED: true, - WORKLOAD_ABUSE_BEHAVIOURAL_SIGNALS_ENABLED: input.behaviouralSignalsEnabled ?? false + WORKLOAD_ABUSE_BEHAVIOURAL_SIGNALS_ENABLED: input.behaviouralSignalsEnabled ?? false, + WORKLOAD_ABUSE_BEHAVIOURAL_ENFORCEMENT_ENABLED: input.behaviouralEnforcementEnabled ?? false }; vi.spyOn(config, "get").mockImplementation((key => (key in overrides ? overrides[key] : readConfig(key))) as typeof config.get); @@ -190,6 +222,22 @@ describe(ProbeTrialDeploymentHandler.name, () => { probe, findDetections: () => detectionRepository.find({ walletId: wallet.id, dseq }), findEvidence: () => evidenceRepository.find({ walletId: wallet.id, dseq }), + seedShapeHistory: (minutesAgo: number[]) => + evidenceRepository.insertMany( + minutesAgo.map(minutes => ({ + walletId: wallet.id, + dseq, + provider: "akash1provider", + service: "ssh", + probeStatus: "probed", + verdict: "clean", + createdAt: new Date(Date.now() - minutes * 60_000), + behaviouralFindings: [ + { signal: "accel_without_artifacts", detail: {} }, + { signal: "network_isolated", detail: {} } + ] + })) + ), seedExistingDetection: () => detectionRepository.create({ userId: user.id, diff --git a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts index e8278f01b1..933375f4ed 100644 --- a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts +++ b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts @@ -7,7 +7,10 @@ import type { WorkloadAbuseDetectionOutput, WorkloadAbuseDetectionRepository } from "@src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository"; -import type { WorkloadProbeEvidenceOutput } from "@src/workload-abuse/repositories/workload-probe-evidence/workload-probe-evidence.repository"; +import type { + WorkloadProbeEvidenceOutput, + WorkloadProbeEvidenceRepository +} from "@src/workload-abuse/repositories/workload-probe-evidence/workload-probe-evidence.repository"; import { EnforceTrialAbuse } from "@src/workload-abuse/services/enforce-trial-abuse/enforce-trial-abuse.handler"; import type { ProbeEvidenceService, RecordedBehaviouralFindings } from "@src/workload-abuse/services/probe-evidence/probe-evidence.service"; import type { ProbeReport, TrialWorkloadProbeService } from "@src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service"; @@ -228,6 +231,61 @@ describe(ProbeTrialDeploymentHandler.name, () => { }); }); + it("records a detection and queues the wipe once the same shape holds across probes", async () => { + const { handler, wallet, detectionRepository, jobQueueService, instrumentation } = setup({ + report: createReport({ verdict: "clean" }), + behaviouralEnforcement: true, + enforcementMode: "enforce", + deploymentEvidence: createShapeHistory([10, 130, 250]) + }); + + await handler.handle(PAYLOAD); + + expect(detectionRepository.create).toHaveBeenCalledWith(expect.objectContaining({ dseq: PAYLOAD.dseq, verdict: "behavioural" })); + expect(instrumentation.recordDetection).toHaveBeenCalledWith("behavioural"); + expect(jobQueueService.enqueue).toHaveBeenCalledWith(new EnforceTrialAbuse({ walletId: wallet.id, detectionId: "detection-1" }), { + singletonKey: `enforceTrialAbuse.${wallet.id}` + }); + }); + + it("keeps probing while the shape has held for fewer probes than required", async () => { + const { handler, detectionRepository, probeJobService } = setup({ + report: createReport({ verdict: "clean" }), + behaviouralEnforcement: true, + deploymentEvidence: createShapeHistory([10, 130]) + }); + + await handler.handle(PAYLOAD); + + expect(detectionRepository.create).not.toHaveBeenCalled(); + expect(probeJobService.scheduleNext).toHaveBeenCalledWith(PAYLOAD); + }); + + it("keeps probing while the shape is younger than the minimum window", async () => { + const { handler, detectionRepository, probeJobService } = setup({ + report: createReport({ verdict: "clean" }), + behaviouralEnforcement: true, + deploymentEvidence: createShapeHistory([1, 3, 5]) + }); + + await handler.handle(PAYLOAD); + + expect(detectionRepository.create).not.toHaveBeenCalled(); + expect(probeJobService.scheduleNext).toHaveBeenCalledWith(PAYLOAD); + }); + + it("records nothing from a repeated shape while behavioural enforcement is off", async () => { + const { handler, detectionRepository, evidenceRepository } = setup({ + report: createReport({ verdict: "clean" }), + deploymentEvidence: createShapeHistory([10, 130, 250]) + }); + + await handler.handle(PAYLOAD); + + expect(evidenceRepository.findRecentForDeployment).not.toHaveBeenCalled(); + expect(detectionRepository.create).not.toHaveBeenCalled(); + }); + it("stops after the last allowed attempt", async () => { const { handler, probeJobService } = setup({ report: createReport({ verdict: "clean" }), maxAttempts: 2 }); @@ -287,6 +345,20 @@ describe(ProbeTrialDeploymentHandler.name, () => { }); }); + function createShapeHistory(minutesAgo: number[]) { + return minutesAgo.map(minutes => + mock({ + service: "web", + probeStatus: "probed", + createdAt: new Date(Date.now() - minutes * 60_000), + behaviouralFindings: [ + { signal: "accel_without_artifacts", detail: {} }, + { signal: "network_isolated", detail: {} } + ] + }) + ); + } + function setup(input: { enabled?: boolean; wallet?: ReturnType | null; @@ -296,6 +368,10 @@ describe(ProbeTrialDeploymentHandler.name, () => { enforcementMode?: "detect" | "enforce"; evidenceRows?: WorkloadProbeEvidenceOutput[]; behaviouralFindings?: RecordedBehaviouralFindings[]; + deploymentEvidence?: WorkloadProbeEvidenceOutput[]; + behaviouralEnforcement?: boolean; + agreementProbes?: number; + minWindowMinutes?: number; }) { const wallet = input.wallet === undefined ? createUserWallet({ isTrialing: true }) : input.wallet; const userWalletRepository = mock(); @@ -306,6 +382,8 @@ describe(ProbeTrialDeploymentHandler.name, () => { const detectionRepository = mock(); detectionRepository.create.mockResolvedValue(mock({ id: "detection-1" })); detectionRepository.findOneBy.mockResolvedValue(input.existingDetection ? mock({ id: "detection-0" }) : undefined); + const evidenceRepository = mock(); + evidenceRepository.findRecentForDeployment.mockResolvedValue(input.deploymentEvidence ?? []); const probeEvidenceService = mock(); probeEvidenceService.recordEvidence.mockImplementation( async ({ shellEvidence }) => input.evidenceRows ?? shellEvidence.map(entry => mock({ service: entry.service })) @@ -315,7 +393,11 @@ describe(ProbeTrialDeploymentHandler.name, () => { const config = mockConfigService({ WORKLOAD_ABUSE_PROBE_ENABLED: input.enabled ?? true, WORKLOAD_ABUSE_PROBE_MAX_PER_DEPLOYMENT: input.maxAttempts ?? 30, - WORKLOAD_ABUSE_ENFORCEMENT_MODE: input.enforcementMode ?? "detect" + WORKLOAD_ABUSE_ENFORCEMENT_MODE: input.enforcementMode ?? "detect", + WORKLOAD_ABUSE_BEHAVIOURAL_ENFORCEMENT_ENABLED: input.behaviouralEnforcement ?? false, + WORKLOAD_ABUSE_BEHAVIOURAL_AGREEMENT_PROBES: input.agreementProbes ?? 3, + WORKLOAD_ABUSE_BEHAVIOURAL_MIN_WINDOW_MINUTES: input.minWindowMinutes ?? 120, + WORKLOAD_ABUSE_PROBE_INTERVAL_MIN: 60 }); const jobQueueService = mock(); const logger = mock>(); @@ -326,6 +408,7 @@ describe(ProbeTrialDeploymentHandler.name, () => { probeService, probeJobService, detectionRepository, + evidenceRepository, probeEvidenceService, instrumentation, config, @@ -340,6 +423,7 @@ describe(ProbeTrialDeploymentHandler.name, () => { probeService, probeJobService, detectionRepository, + evidenceRepository, probeEvidenceService, instrumentation, jobQueueService, diff --git a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts index 51cea2ad86..a54a777d4a 100644 --- a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts +++ b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts @@ -2,9 +2,11 @@ import { inject, singleton } from "tsyringe"; import { isWalletInitialized, UserWalletRepository } from "@src/billing/repositories"; import { type CreateLogger, JOB_NAME, type JobHandler, type JobPayload, type JobPermissions, JobQueueService, LOGGER_FACTORY } from "@src/core"; +import { findBehaviouralAgreement } from "@src/workload-abuse/lib/behavioural-signals/agreement"; import { withoutFileContents } from "@src/workload-abuse/lib/evidence-scanner/evidence-scanner"; import { truncateToUtf8Bytes } from "@src/workload-abuse/lib/utf8-text/utf8-text"; import { WorkloadAbuseDetectionRepository } from "@src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository"; +import { WorkloadProbeEvidenceRepository } from "@src/workload-abuse/repositories/workload-probe-evidence/workload-probe-evidence.repository"; import { EnforceTrialAbuse, enforceTrialAbuseKeyFor } from "@src/workload-abuse/services/enforce-trial-abuse/enforce-trial-abuse.handler"; import { ProbeEvidenceService } from "@src/workload-abuse/services/probe-evidence/probe-evidence.service"; import { type ProbeReport, TrialWorkloadProbeService } from "@src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service"; @@ -32,6 +34,7 @@ export class ProbeTrialDeploymentHandler implements JobHandler= this.config.get("WORKLOAD_ABUSE_PROBE_MAX_PER_DEPLOYMENT")) { this.logger.info({ event: "TRIAL_WORKLOAD_PROBE_FINISHED", reason: "MAX_ATTEMPTS", ...context, userId: wallet.userId }); return; @@ -179,6 +184,63 @@ export class ProbeTrialDeploymentHandler implements JobHandler + ): Promise { + if (!this.config.get("WORKLOAD_ABUSE_BEHAVIOURAL_ENFORCEMENT_ENABLED")) return false; + + const rows = await this.evidenceRepository.findRecentForDeployment({ walletId: wallet.id, dseq, since: this.#evidenceLookbackStart() }); + const agreement = findBehaviouralAgreement(rows, { + agreementProbes: this.config.get("WORKLOAD_ABUSE_BEHAVIOURAL_AGREEMENT_PROBES"), + minWindowMinutes: this.config.get("WORKLOAD_ABUSE_BEHAVIOURAL_MIN_WINDOW_MINUTES") + }); + + if (!agreement.agreed) return false; + + const existing = await this.detectionRepository.findOneBy({ walletId: wallet.id, dseq, verdict: "behavioural" }); + + if (existing) { + await this.#enforce(wallet, existing.id, context); + return true; + } + + const detection = await this.detectionRepository.create({ + userId: wallet.userId, + walletId: wallet.id, + dseq, + provider: report.leases.map(lease => lease.provider).join(","), + verdict: "behavioural", + probeStatus: report.probeStatus, + signals: report.signals, + evidenceExcerpt: withoutFileContents(report.excerpt) + }); + this.instrumentation.recordDetection("behavioural"); + + this.logger.warn({ + event: "TRIAL_WORKLOAD_SHAPE_CONFIRMED", + ...context, + userId: wallet.userId, + detectionId: detection.id, + service: agreement.service, + probes: agreement.streak, + spanMinutes: agreement.spanMinutes + }); + + await this.#enforce(wallet, detection.id, context); + + return true; + } + + #evidenceLookbackStart(): Date { + const probeSpanMinutes = this.config.get("WORKLOAD_ABUSE_PROBE_INTERVAL_MIN") * this.config.get("WORKLOAD_ABUSE_PROBE_MAX_PER_DEPLOYMENT"); + + return new Date(Date.now() - probeSpanMinutes * 60_000); + } + /** Detect mode records the verdict and stops there, so a rollout can be compared against manual review before anything is wiped. */ async #enforce(wallet: { id: number; userId: string }, detectionId: string, context: Record): Promise { if (this.config.get("WORKLOAD_ABUSE_ENFORCEMENT_MODE") !== "enforce") { diff --git a/apps/api/src/workload-abuse/services/trial-workload-probe-job/trial-workload-probe-job.service.spec.ts b/apps/api/src/workload-abuse/services/trial-workload-probe-job/trial-workload-probe-job.service.spec.ts index a159b0fb2e..e9439f7fee 100644 --- a/apps/api/src/workload-abuse/services/trial-workload-probe-job/trial-workload-probe-job.service.spec.ts +++ b/apps/api/src/workload-abuse/services/trial-workload-probe-job/trial-workload-probe-job.service.spec.ts @@ -194,7 +194,7 @@ describe(TrialWorkloadProbeJobService.name, () => { await service.reconcile({ dryRun: false }); - expect(detectionRepository.findRecentHardTargets).toHaveBeenCalledWith({ since: subHours(LEASE_CREATED_AT, 26) }); + expect(detectionRepository.findRecentDetectedTargets).toHaveBeenCalledWith({ since: subHours(LEASE_CREATED_AT, 26) }); expect(jobQueueService.enqueue).toHaveBeenCalledTimes(1); expect(jobQueueService.enqueue).toHaveBeenCalledWith( expect.objectContaining({ data: expect.objectContaining({ walletId: 2, dseq: "2" }) }), @@ -232,7 +232,7 @@ describe(TrialWorkloadProbeJobService.name, () => { const deploymentSettingRepository = mock(); deploymentSettingRepository.findLiveTrialDeployments.mockResolvedValue(input.live ?? []); const detectionRepository = mock(); - detectionRepository.findRecentHardTargets.mockResolvedValue(input.detected ?? []); + detectionRepository.findRecentDetectedTargets.mockResolvedValue(input.detected ?? []); const config = mockConfigService({ WORKLOAD_ABUSE_PROBE_ENABLED: input.enabled ?? true, WORKLOAD_ABUSE_PROBE_INITIAL_DELAYS_MIN: [5, 20, 60], diff --git a/apps/api/src/workload-abuse/services/trial-workload-probe-job/trial-workload-probe-job.service.ts b/apps/api/src/workload-abuse/services/trial-workload-probe-job/trial-workload-probe-job.service.ts index afdce86528..32ed5fe7a3 100644 --- a/apps/api/src/workload-abuse/services/trial-workload-probe-job/trial-workload-probe-job.service.ts +++ b/apps/api/src/workload-abuse/services/trial-workload-probe-job/trial-workload-probe-job.service.ts @@ -107,7 +107,7 @@ export class TrialWorkloadProbeJobService { if (dryRun) return; const pendingKeys = await this.jobQueueService.findPendingSingletonKeys(ProbeTrialDeployment[JOB_NAME]); - const detectedTargets = await this.detectionRepository.findRecentHardTargets({ since: subHours(new Date(), maxAgeHours) }); + const detectedTargets = await this.detectionRepository.findRecentDetectedTargets({ since: subHours(new Date(), maxAgeHours) }); const detectedKeys = new Set(detectedTargets.map(probeTrialDeploymentKeyFor)); let scheduled = 0; let alreadyScheduled = 0; diff --git a/apps/api/src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service.ts b/apps/api/src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service.ts index 7729d65b25..f43f148720 100644 --- a/apps/api/src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service.ts +++ b/apps/api/src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service.ts @@ -8,6 +8,8 @@ export type DomainBlockResult = "blocked" | "raced" | "skipped" | "dry_run" | "f export type EvidenceWriteOperation = "insert" | "findings" | "purge"; +export type DetectionVerdict = WorkloadVerdict | "behavioural"; + @singleton() export class WorkloadAbuseInstrumentationService { private readonly meter: Meter; @@ -48,7 +50,7 @@ export class WorkloadAbuseInstrumentationService { this.probes.add(1, { verdict: input.verdict, probe_status: input.probeStatus }); } - recordDetection(verdict: WorkloadVerdict): void { + recordDetection(verdict: DetectionVerdict): void { this.detections.add(1, { verdict }); } From 9ada4dba4bb828fb1e4c97a5ad99bd7edb9dd81a Mon Sep 17 00:00:00 2001 From: Maxime Beauchamp <15185355+baktun14@users.noreply.github.com> Date: Sun, 20 Sep 2026 16:06:14 +0400 Subject: [PATCH 2/6] fix(deployment): read back far enough for the agreement window to be reachable Agreement needs a streak that spans a configured window, but the evidence query only looked back as far as the probe cadence allows. Shorten the cadence and the oldest row it can return is younger than the span it has to prove, so agreement never lands however long the shape holds. The lookback now covers whichever of the two is longer. Agreement also reads the status of the shell that collected each row, matching where the signals are now scored. --- .../lib/behavioural-signals/agreement.spec.ts | 2 +- .../lib/behavioural-signals/agreement.ts | 7 +++---- ...be-trial-deployment.handler.integration.ts | 6 +++--- .../probe-trial-deployment.handler.spec.ts | 20 +++++++++++++++++-- .../probe-trial-deployment.handler.ts | 4 +++- 5 files changed, 28 insertions(+), 11 deletions(-) diff --git a/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.spec.ts b/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.spec.ts index 865ce3dc9c..7f908ba55c 100644 --- a/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.spec.ts +++ b/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.spec.ts @@ -64,7 +64,7 @@ describe("findBehaviouralAgreement", () => { const rows: AgreementRow[] = input.minutesAgo.map((minutes, index) => ({ service: "web", createdAt: new Date(NOW.getTime() - minutes * 60_000), - probeStatus: input.statusByIndex?.[index] ?? "probed", + shellStatus: input.statusByIndex?.[index] ?? "completed", behaviouralFindings: input.findingsByIndex?.[index] ?? BOTH_SIGNALS })); diff --git a/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.ts b/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.ts index 7f1e37b362..9bf5e6f0a4 100644 --- a/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.ts +++ b/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.ts @@ -1,13 +1,12 @@ import { isBehaviouralCandidate } from "./evaluate-behavioural-signals"; -import type { BehaviouralFinding } from "./types"; +import { type BehaviouralFinding, COMPLETE_SHELL_STATUS } from "./types"; -const COLLECTED_PROBE_STATUS = "probed"; const MS_PER_MINUTE = 60_000; export type AgreementRow = { service: string; createdAt: Date; - probeStatus: string; + shellStatus: string; behaviouralFindings: BehaviouralFinding[] | null; }; @@ -49,7 +48,7 @@ function measureService(rows: AgreementRow[], params: AgreementParams, now: Date const streak: AgreementRow[] = []; for (const row of newestFirst) { - if (row.probeStatus !== COLLECTED_PROBE_STATUS) continue; + if (row.shellStatus !== COMPLETE_SHELL_STATUS) continue; if (!isBehaviouralCandidate(row.behaviouralFindings ?? [])) break; streak.push(row); } diff --git a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.integration.ts b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.integration.ts index 05f92a31d4..735b9b1e0d 100644 --- a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.integration.ts +++ b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.integration.ts @@ -147,7 +147,7 @@ describe(ProbeTrialDeploymentHandler.name, () => { verdict: "clean", behaviouralSignalsEnabled: true, behaviouralEnforcementEnabled: true, - shellOutputs: [{ service: "ssh", provider: "akash1provider", output: ISOLATED_ACCELERATED_SHELL_OUTPUT }] + shellEvidence: [{ service: "ssh", provider: "akash1provider", status: "completed", evidence: ISOLATED_ACCELERATED_EVIDENCE }] }); await seedShapeHistory([130, 250]); @@ -162,7 +162,7 @@ describe(ProbeTrialDeploymentHandler.name, () => { verdict: "clean", behaviouralSignalsEnabled: true, behaviouralEnforcementEnabled: true, - shellOutputs: [{ service: "ssh", provider: "akash1provider", output: ISOLATED_ACCELERATED_SHELL_OUTPUT }] + shellEvidence: [{ service: "ssh", provider: "akash1provider", status: "completed", evidence: ISOLATED_ACCELERATED_EVIDENCE }] }); await seedShapeHistory([130]); @@ -229,7 +229,7 @@ describe(ProbeTrialDeploymentHandler.name, () => { dseq, provider: "akash1provider", service: "ssh", - probeStatus: "probed", + shellStatus: "completed", verdict: "clean", createdAt: new Date(Date.now() - minutes * 60_000), behaviouralFindings: [ diff --git a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts index 933375f4ed..2849a4b6f2 100644 --- a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts +++ b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts @@ -248,6 +248,21 @@ describe(ProbeTrialDeploymentHandler.name, () => { }); }); + it("reads back at least as far as the window the shape has to span, however the probe cadence is set", async () => { + const { handler, detectionRepository } = setup({ + report: createReport({ verdict: "clean" }), + behaviouralEnforcement: true, + probeIntervalMinutes: 10, + maxAttempts: 5, + minWindowMinutes: 120, + deploymentEvidence: createShapeHistory([10, 130, 250]) + }); + + await handler.handle(PAYLOAD); + + expect(detectionRepository.create).toHaveBeenCalledWith(expect.objectContaining({ verdict: "behavioural" })); + }); + it("keeps probing while the shape has held for fewer probes than required", async () => { const { handler, detectionRepository, probeJobService } = setup({ report: createReport({ verdict: "clean" }), @@ -349,7 +364,7 @@ describe(ProbeTrialDeploymentHandler.name, () => { return minutesAgo.map(minutes => mock({ service: "web", - probeStatus: "probed", + shellStatus: "completed", createdAt: new Date(Date.now() - minutes * 60_000), behaviouralFindings: [ { signal: "accel_without_artifacts", detail: {} }, @@ -372,6 +387,7 @@ describe(ProbeTrialDeploymentHandler.name, () => { behaviouralEnforcement?: boolean; agreementProbes?: number; minWindowMinutes?: number; + probeIntervalMinutes?: number; }) { const wallet = input.wallet === undefined ? createUserWallet({ isTrialing: true }) : input.wallet; const userWalletRepository = mock(); @@ -397,7 +413,7 @@ describe(ProbeTrialDeploymentHandler.name, () => { WORKLOAD_ABUSE_BEHAVIOURAL_ENFORCEMENT_ENABLED: input.behaviouralEnforcement ?? false, WORKLOAD_ABUSE_BEHAVIOURAL_AGREEMENT_PROBES: input.agreementProbes ?? 3, WORKLOAD_ABUSE_BEHAVIOURAL_MIN_WINDOW_MINUTES: input.minWindowMinutes ?? 120, - WORKLOAD_ABUSE_PROBE_INTERVAL_MIN: 60 + WORKLOAD_ABUSE_PROBE_INTERVAL_MIN: input.probeIntervalMinutes ?? 60 }); const jobQueueService = mock(); const logger = mock>(); diff --git a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts index a54a777d4a..de93aa8777 100644 --- a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts +++ b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts @@ -235,10 +235,12 @@ export class ProbeTrialDeploymentHandler implements JobHandler Date: Sun, 20 Sep 2026 16:59:26 +0400 Subject: [PATCH 3/6] fix(deployment): record what confirmed a shape on the detection row The probe that tips a deployment over can itself be a clean one, so the row was being written with an empty signal list and a benign excerpt. It now carries the service, the number of agreeing probes and the span they cover, which is what a reviewer needs while enforcement is still off. --- .../probe-trial-deployment.handler.spec.ts | 8 +++++++- .../probe-trial-deployment.handler.ts | 9 +++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts index 2849a4b6f2..d8e432e3e1 100644 --- a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts +++ b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts @@ -241,7 +241,13 @@ describe(ProbeTrialDeploymentHandler.name, () => { await handler.handle(PAYLOAD); - expect(detectionRepository.create).toHaveBeenCalledWith(expect.objectContaining({ dseq: PAYLOAD.dseq, verdict: "behavioural" })); + expect(detectionRepository.create).toHaveBeenCalledWith( + expect.objectContaining({ + dseq: PAYLOAD.dseq, + verdict: "behavioural", + evidenceExcerpt: expect.stringContaining("service:web probes:3 spanMinutes:") + }) + ); expect(instrumentation.recordDetection).toHaveBeenCalledWith("behavioural"); expect(jobQueueService.enqueue).toHaveBeenCalledWith(new EnforceTrialAbuse({ walletId: wallet.id, detectionId: "detection-1" }), { singletonKey: `enforceTrialAbuse.${wallet.id}` diff --git a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts index de93aa8777..e68fbf4064 100644 --- a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts +++ b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts @@ -2,7 +2,7 @@ import { inject, singleton } from "tsyringe"; import { isWalletInitialized, UserWalletRepository } from "@src/billing/repositories"; import { type CreateLogger, JOB_NAME, type JobHandler, type JobPayload, type JobPermissions, JobQueueService, LOGGER_FACTORY } from "@src/core"; -import { findBehaviouralAgreement } from "@src/workload-abuse/lib/behavioural-signals/agreement"; +import { type BehaviouralAgreement, findBehaviouralAgreement } from "@src/workload-abuse/lib/behavioural-signals/agreement"; import { withoutFileContents } from "@src/workload-abuse/lib/evidence-scanner/evidence-scanner"; import { truncateToUtf8Bytes } from "@src/workload-abuse/lib/utf8-text/utf8-text"; import { WorkloadAbuseDetectionRepository } from "@src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository"; @@ -17,6 +17,11 @@ import { WorkloadAbuseInstrumentationService } from "@src/workload-abuse/service /** Loki splits a line past 16 KiB into unparseable partials, and a clean verdict has nowhere else to keep what the shell saw. */ const MAX_LOGGED_EXCERPT_BYTES = 4_096; +/** The probe that confirms a shape can be a clean one, so the row has to carry the streak that confirmed it rather than what that last probe saw. */ +function toShapeExcerpt(agreement: BehaviouralAgreement, excerpt: string): string { + return [`[behavioural] service:${agreement.service} probes:${agreement.streak} spanMinutes:${agreement.spanMinutes}`, excerpt].join("\n"); +} + /** Re-reads the wallet and the chain on every run, so a probe that waited an hour decides on what is true when it runs, not when it was queued. */ @singleton() export class ProbeTrialDeploymentHandler implements JobHandler { @@ -216,7 +221,7 @@ export class ProbeTrialDeploymentHandler implements JobHandler Date: Sun, 20 Sep 2026 21:30:15 +0400 Subject: [PATCH 4/6] refactor(deployment): handle an empty probe where it happens and cover the window The mutation gate flagged the lookback math, the ordering of a history that arrives unsorted, the window boundary and the service the report names as untested. Each now has a case that fails when the behaviour changes. A probe that recorded nothing is handled where it happens instead of standing in as an empty finding list, and reading the first service of a group no longer guards against an empty one, since a group only exists because a row put it there. --- .../lib/behavioural-signals/agreement.spec.ts | 42 ++++++++- .../lib/behavioural-signals/agreement.ts | 4 +- .../probe-trial-deployment.handler.spec.ts | 91 ++++++++++++++++++- 3 files changed, 133 insertions(+), 4 deletions(-) diff --git a/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.spec.ts b/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.spec.ts index 7f908ba55c..d48f5caddd 100644 --- a/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.spec.ts +++ b/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.spec.ts @@ -48,6 +48,45 @@ describe("findBehaviouralAgreement", () => { expect(findBehaviouralAgreement([...rows, ...sidecar], params)).toMatchObject({ agreed: true, service: "web", streak: 3 }); }); + it("orders a history that arrives out of order before counting the streak", () => { + const { rows, params } = setup({ minutesAgo: [250, 10, 130], findingsByIndex: { 0: [{ signal: "network_isolated", detail: {} }] } }); + + expect(findBehaviouralAgreement(rows, params)).toMatchObject({ agreed: false, streak: 2, spanMinutes: 130 }); + }); + + it("breaks the streak on a probe that recorded no findings at all", () => { + const { rows, params } = setup({ minutesAgo: [10, 130, 250], nullFindingsAt: 1 }); + + expect(findBehaviouralAgreement(rows, params)).toMatchObject({ agreed: false, streak: 1 }); + }); + + it("agrees once the streak is exactly as old as the minimum window", () => { + const { rows, params } = setup({ minutesAgo: [10, 60, 120], minWindowMinutes: 120 }); + + expect(findBehaviouralAgreement(rows, params)).toMatchObject({ agreed: true, streak: 3, spanMinutes: 120 }); + }); + + it("prefers the service that agreed over one carrying a longer streak that has not", () => { + const { rows, params } = setup({ minutesAgo: [10, 130, 250] }); + const sidecar = setup({ minutesAgo: [1, 2, 3, 4] }).rows.map(row => ({ ...row, service: "sidecar" })); + + expect(findBehaviouralAgreement([...rows, ...sidecar], params)).toMatchObject({ agreed: true, service: "web", streak: 3 }); + }); + + it("reports the longest streak while no service has agreed", () => { + const { rows, params } = setup({ minutesAgo: [10, 130] }); + const sidecar = setup({ minutesAgo: [1, 2, 3, 4] }).rows.map(row => ({ ...row, service: "sidecar" })); + + expect(findBehaviouralAgreement([...rows, ...sidecar], params)).toMatchObject({ agreed: false, service: "sidecar", streak: 4 }); + }); + + it("keeps the first service when two are equally short of agreement", () => { + const { rows, params } = setup({ minutesAgo: [10, 130] }); + const sidecar = rows.map(row => ({ ...row, service: "sidecar" })); + + expect(findBehaviouralAgreement([...rows, ...sidecar], params)).toMatchObject({ agreed: false, service: "web", streak: 2 }); + }); + it("finds no agreement in an empty history", () => { const { params } = setup({ minutesAgo: [] }); @@ -60,12 +99,13 @@ describe("findBehaviouralAgreement", () => { minWindowMinutes?: number; findingsByIndex?: Record; statusByIndex?: Record; + nullFindingsAt?: number; }) { const rows: AgreementRow[] = input.minutesAgo.map((minutes, index) => ({ service: "web", createdAt: new Date(NOW.getTime() - minutes * 60_000), shellStatus: input.statusByIndex?.[index] ?? "completed", - behaviouralFindings: input.findingsByIndex?.[index] ?? BOTH_SIGNALS + behaviouralFindings: index === input.nullFindingsAt ? null : input.findingsByIndex?.[index] ?? BOTH_SIGNALS })); return { diff --git a/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.ts b/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.ts index 9bf5e6f0a4..db2ae1e8aa 100644 --- a/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.ts +++ b/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.ts @@ -49,7 +49,7 @@ function measureService(rows: AgreementRow[], params: AgreementParams, now: Date for (const row of newestFirst) { if (row.shellStatus !== COMPLETE_SHELL_STATUS) continue; - if (!isBehaviouralCandidate(row.behaviouralFindings ?? [])) break; + if (!row.behaviouralFindings || !isBehaviouralCandidate(row.behaviouralFindings)) break; streak.push(row); } @@ -58,7 +58,7 @@ function measureService(rows: AgreementRow[], params: AgreementParams, now: Date return { agreed: streak.length >= params.agreementProbes && spanMinutes >= params.minWindowMinutes, - service: newestFirst[0]?.service ?? null, + service: newestFirst[0].service, streak: streak.length, spanMinutes }; diff --git a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts index d8e432e3e1..bd39a51512 100644 --- a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts +++ b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts @@ -269,6 +269,84 @@ describe(ProbeTrialDeploymentHandler.name, () => { expect(detectionRepository.create).toHaveBeenCalledWith(expect.objectContaining({ verdict: "behavioural" })); }); + it("reads evidence back across the whole probe schedule when it outlasts the minimum window", async () => { + const { handler, wallet, evidenceRepository } = setup({ + report: createReport({ verdict: "clean" }), + behaviouralEnforcement: true, + probeIntervalMinutes: 60, + maxAttempts: 5, + minWindowMinutes: 120, + now: new Date("2026-09-20T12:00:00.000Z") + }); + + await handler.handle(PAYLOAD); + + expect(evidenceRepository.findRecentForDeployment).toHaveBeenCalledWith({ + walletId: wallet.id, + dseq: PAYLOAD.dseq, + since: new Date("2026-09-20T07:00:00.000Z") + }); + }); + + it("queues the wipe against the detection a confirmed shape already has instead of recording a second one", async () => { + const { handler, wallet, detectionRepository, jobQueueService, probeJobService } = setup({ + report: createReport({ verdict: "clean" }), + behaviouralEnforcement: true, + enforcementMode: "enforce", + existingBehaviouralDetection: true, + deploymentEvidence: createShapeHistory([10, 130, 250]) + }); + + await handler.handle(PAYLOAD); + + expect(detectionRepository.findOneBy).toHaveBeenCalledWith({ walletId: wallet.id, dseq: PAYLOAD.dseq, verdict: "behavioural" }); + expect(detectionRepository.create).not.toHaveBeenCalled(); + expect(probeJobService.scheduleNext).not.toHaveBeenCalled(); + expect(jobQueueService.enqueue).toHaveBeenCalledWith(new EnforceTrialAbuse({ walletId: wallet.id, detectionId: "detection-2" }), { + singletonKey: `enforceTrialAbuse.${wallet.id}` + }); + }); + + it("names every provider the shape was seen on and stops probing once it is recorded", async () => { + const { handler, detectionRepository, probeJobService } = setup({ + report: createReport({ + verdict: "clean", + leases: [ + { + provider: "akash1first", + hostUri: "https://first.example", + gseq: 1, + oseq: 1, + services: ["web"], + shellStatuses: ["completed"], + logStatus: "completed" + }, + { + provider: "akash1second", + hostUri: "https://second.example", + gseq: 2, + oseq: 1, + services: ["web"], + shellStatuses: ["completed"], + logStatus: "completed" + } + ] + }), + behaviouralEnforcement: true, + deploymentEvidence: createShapeHistory([10, 130, 250]) + }); + + await handler.handle(PAYLOAD); + + expect(detectionRepository.create).toHaveBeenCalledWith( + expect.objectContaining({ + provider: "akash1first,akash1second", + evidenceExcerpt: expect.stringMatching(/^\[behavioural\] service:web probes:3 spanMinutes:\d+\n\[soft\/pool-port\] shell:ssh: pool:3333$/) + }) + ); + expect(probeJobService.scheduleNext).not.toHaveBeenCalled(); + }); + it("keeps probing while the shape has held for fewer probes than required", async () => { const { handler, detectionRepository, probeJobService } = setup({ report: createReport({ verdict: "clean" }), @@ -386,6 +464,8 @@ describe(ProbeTrialDeploymentHandler.name, () => { report?: ProbeReport; maxAttempts?: number; existingDetection?: boolean; + existingBehaviouralDetection?: boolean; + now?: Date; enforcementMode?: "detect" | "enforce"; evidenceRows?: WorkloadProbeEvidenceOutput[]; behaviouralFindings?: RecordedBehaviouralFindings[]; @@ -395,6 +475,9 @@ describe(ProbeTrialDeploymentHandler.name, () => { minWindowMinutes?: number; probeIntervalMinutes?: number; }) { + if (input.now) vi.useFakeTimers({ now: input.now, toFake: ["Date"] }); + else vi.useRealTimers(); + const wallet = input.wallet === undefined ? createUserWallet({ isTrialing: true }) : input.wallet; const userWalletRepository = mock(); userWalletRepository.findById.mockResolvedValue(wallet ?? undefined); @@ -403,7 +486,13 @@ describe(ProbeTrialDeploymentHandler.name, () => { const probeJobService = mock(); const detectionRepository = mock(); detectionRepository.create.mockResolvedValue(mock({ id: "detection-1" })); - detectionRepository.findOneBy.mockResolvedValue(input.existingDetection ? mock({ id: "detection-0" }) : undefined); + const hardDetection = mock({ id: "detection-0" }); + const behaviouralDetection = mock({ id: "detection-2" }); + detectionRepository.findOneBy.mockImplementation(async query => { + if (query?.verdict === "hard") return input.existingDetection ? hardDetection : undefined; + if (query?.verdict === "behavioural") return input.existingBehaviouralDetection ? behaviouralDetection : undefined; + return undefined; + }); const evidenceRepository = mock(); evidenceRepository.findRecentForDeployment.mockResolvedValue(input.deploymentEvidence ?? []); const probeEvidenceService = mock(); From c75da4b567b385df273ee33b35fbc3a80e53b50e Mon Sep 17 00:00:00 2001 From: Maxime Beauchamp <15185355+baktun14@users.noreply.github.com> Date: Sun, 20 Sep 2026 22:07:04 +0400 Subject: [PATCH 5/6] fix(deployment): keep blank enforcement settings from failing config A declared but empty environment variable never reaches a Zod default, so a blank value for any of the three enforcement settings failed config resolution for the whole module at boot. They now go through the same blank-to-undefined preprocessor their siblings already use. --- .../workload-abuse/config/env.config.spec.ts | 24 +++++++++++++++++++ .../src/workload-abuse/config/env.config.ts | 15 +++++++----- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/apps/api/src/workload-abuse/config/env.config.spec.ts b/apps/api/src/workload-abuse/config/env.config.spec.ts index 99dc04075b..01738260cb 100644 --- a/apps/api/src/workload-abuse/config/env.config.spec.ts +++ b/apps/api/src/workload-abuse/config/env.config.spec.ts @@ -93,6 +93,30 @@ describe("workload abuse env config", () => { expect(config.WORKLOAD_ABUSE_SIGNAL_ARTIFACT_MIN_MB).toBe(256); }); + it("falls back to the enforcement defaults when those variables are set but blank", () => { + const config = envSchema.parse({ + WORKLOAD_ABUSE_BEHAVIOURAL_ENFORCEMENT_ENABLED: "", + WORKLOAD_ABUSE_BEHAVIOURAL_AGREEMENT_PROBES: "", + WORKLOAD_ABUSE_BEHAVIOURAL_MIN_WINDOW_MINUTES: " " + }); + + expect(config.WORKLOAD_ABUSE_BEHAVIOURAL_ENFORCEMENT_ENABLED).toBe(false); + expect(config.WORKLOAD_ABUSE_BEHAVIOURAL_AGREEMENT_PROBES).toBe(3); + expect(config.WORKLOAD_ABUSE_BEHAVIOURAL_MIN_WINDOW_MINUTES).toBe(120); + }); + + it("still reads the enforcement variables when they carry a value", () => { + const config = envSchema.parse({ + WORKLOAD_ABUSE_BEHAVIOURAL_ENFORCEMENT_ENABLED: "true", + WORKLOAD_ABUSE_BEHAVIOURAL_AGREEMENT_PROBES: "5", + WORKLOAD_ABUSE_BEHAVIOURAL_MIN_WINDOW_MINUTES: "240" + }); + + expect(config.WORKLOAD_ABUSE_BEHAVIOURAL_ENFORCEMENT_ENABLED).toBe(true); + expect(config.WORKLOAD_ABUSE_BEHAVIOURAL_AGREEMENT_PROBES).toBe(5); + expect(config.WORKLOAD_ABUSE_BEHAVIOURAL_MIN_WINDOW_MINUTES).toBe(240); + }); + it("still reads the behavioural signal variables when they carry a value", () => { const config = envSchema.parse({ WORKLOAD_ABUSE_BEHAVIOURAL_SIGNALS_ENABLED: "true", diff --git a/apps/api/src/workload-abuse/config/env.config.ts b/apps/api/src/workload-abuse/config/env.config.ts index c4df360e16..8fb9c1e480 100644 --- a/apps/api/src/workload-abuse/config/env.config.ts +++ b/apps/api/src/workload-abuse/config/env.config.ts @@ -122,13 +122,16 @@ export const envSchema = z.object({ WORKLOAD_ABUSE_SIGNAL_ARTIFACT_MIN_MB: z.preprocess(blankToUndefined, z.number({ coerce: true }).int().positive().default(256)), /** The list itself lives in Doppler: these are our own endpoints, so a reader learns how the exclusion works but not what it covers. */ WORKLOAD_ABUSE_SIGNAL_RELAY_ENDPOINTS: z.preprocess(blankToUndefined, z.string().default("[]").transform(parseRelayEndpoints)), - WORKLOAD_ABUSE_BEHAVIOURAL_ENFORCEMENT_ENABLED: z - .enum(["true", "false"]) - .default("false") - .transform(value => value === "true"), - WORKLOAD_ABUSE_BEHAVIOURAL_AGREEMENT_PROBES: z.number({ coerce: true }).int().positive().default(3), + WORKLOAD_ABUSE_BEHAVIOURAL_ENFORCEMENT_ENABLED: z.preprocess( + blankToUndefined, + z + .enum(["true", "false"]) + .default("false") + .transform(value => value === "true") + ), + WORKLOAD_ABUSE_BEHAVIOURAL_AGREEMENT_PROBES: z.preprocess(blankToUndefined, z.number({ coerce: true }).int().positive().default(3)), /** Back-to-back probes can land minutes apart, so a streak also has to span real time before it counts. */ - WORKLOAD_ABUSE_BEHAVIOURAL_MIN_WINDOW_MINUTES: z.number({ coerce: true }).int().positive().default(120) + WORKLOAD_ABUSE_BEHAVIOURAL_MIN_WINDOW_MINUTES: z.preprocess(blankToUndefined, z.number({ coerce: true }).int().positive().default(120)) }); export type WorkloadAbuseConfig = z.infer; From 65c06f8ae4949f4d8869140fadad815ce9020c5e Mon Sep 17 00:00:00 2001 From: Maxime Beauchamp <15185355+baktun14@users.noreply.github.com> Date: Sun, 20 Sep 2026 22:26:38 +0400 Subject: [PATCH 6/6] fix(deployment): count a repeated shape per provider rather than per service A lease being reclaimed stays live alongside the one replacing it, so a single run can write two rows for the same service. Counting those rows as one series let a streak reach the required length in fewer runs than the setting asks for, which is the one direction that matters on a path that wipes an account. The series key now carries the provider, so each observation point counts once and a migration restarts the count instead of compounding it. The replay groups the same way so its report still predicts what the gate does. --- .../lib/behavioural-signals/agreement.spec.ts | 9 +++++++++ .../workload-abuse/lib/behavioural-signals/agreement.ts | 7 +++++-- .../behavioural-signal-replay.service.spec.ts | 7 ++++--- .../behavioural-signal-replay.service.ts | 2 +- .../probe-trial-deployment.handler.spec.ts | 1 + 5 files changed, 20 insertions(+), 6 deletions(-) diff --git a/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.spec.ts b/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.spec.ts index d48f5caddd..72fd1b1a51 100644 --- a/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.spec.ts +++ b/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.spec.ts @@ -87,6 +87,13 @@ describe("findBehaviouralAgreement", () => { expect(findBehaviouralAgreement([...rows, ...sidecar], params)).toMatchObject({ agreed: false, service: "web", streak: 2 }); }); + it("counts a service seen on two providers at once as two series rather than one longer streak", () => { + const { rows, params } = setup({ minutesAgo: [10, 130] }); + const reclaiming = setup({ minutesAgo: [11, 131], provider: "akash1reclaiming" }).rows; + + expect(findBehaviouralAgreement([...rows, ...reclaiming], params)).toMatchObject({ agreed: false, streak: 2 }); + }); + it("finds no agreement in an empty history", () => { const { params } = setup({ minutesAgo: [] }); @@ -100,9 +107,11 @@ describe("findBehaviouralAgreement", () => { findingsByIndex?: Record; statusByIndex?: Record; nullFindingsAt?: number; + provider?: string; }) { const rows: AgreementRow[] = input.minutesAgo.map((minutes, index) => ({ service: "web", + provider: input.provider ?? "akash1provider", createdAt: new Date(NOW.getTime() - minutes * 60_000), shellStatus: input.statusByIndex?.[index] ?? "completed", behaviouralFindings: index === input.nullFindingsAt ? null : input.findingsByIndex?.[index] ?? BOTH_SIGNALS diff --git a/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.ts b/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.ts index db2ae1e8aa..774eb0cbfe 100644 --- a/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.ts +++ b/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.ts @@ -5,6 +5,7 @@ const MS_PER_MINUTE = 60_000; export type AgreementRow = { service: string; + provider: string; createdAt: Date; shellStatus: string; behaviouralFindings: BehaviouralFinding[] | null; @@ -31,13 +32,15 @@ export function findBehaviouralAgreement(rows: AgreementRow[], params: Agreement return perService.reduce(pickStrongest, { agreed: false, service: null, streak: 0, spanMinutes: 0 }); } +/** A lease under reclamation overlaps the one replacing it, so a single run can write two rows for one service and only a per provider series counts distinct probes. */ function groupByService(rows: AgreementRow[]): Map { const grouped = new Map(); for (const row of rows) { - const serviceRows = grouped.get(row.service) ?? []; + const key = `${row.service}/${row.provider}`; + const serviceRows = grouped.get(key) ?? []; serviceRows.push(row); - grouped.set(row.service, serviceRows); + grouped.set(key, serviceRows); } return grouped; diff --git a/apps/api/src/workload-abuse/services/behavioural-signal-replay/behavioural-signal-replay.service.spec.ts b/apps/api/src/workload-abuse/services/behavioural-signal-replay/behavioural-signal-replay.service.spec.ts index d00d726c31..6e67575bba 100644 --- a/apps/api/src/workload-abuse/services/behavioural-signal-replay/behavioural-signal-replay.service.spec.ts +++ b/apps/api/src/workload-abuse/services/behavioural-signal-replay/behavioural-signal-replay.service.spec.ts @@ -97,8 +97,8 @@ describe(BehaviouralSignalReplayService.name, () => { until: new Date("2026-09-01T00:00:00.000Z") }); expect(summary.deployments).toEqual([ - { source: "database", label: "42/1000001/web", probes: 2, accelFires: 2, networkFires: 2, candidateProbes: 2, longestAgreement: 2 }, - { source: "database", label: "42/1000001/sidecar", probes: 1, accelFires: 0, networkFires: 1, candidateProbes: 0, longestAgreement: 0 } + { source: "database", label: "42/1000001/web/akash1provider", probes: 2, accelFires: 2, networkFires: 2, candidateProbes: 2, longestAgreement: 2 }, + { source: "database", label: "42/1000001/sidecar/akash1provider", probes: 1, accelFires: 0, networkFires: 1, candidateProbes: 0, longestAgreement: 0 } ]); }); @@ -110,7 +110,7 @@ describe(BehaviouralSignalReplayService.name, () => { const summary = await service.replay({ since: new Date("2026-08-01T00:00:00.000Z") }); expect(summary.deployments).toEqual([ - { source: "database", label: "42/1000001/web", probes: 1, accelFires: 1, networkFires: 1, candidateProbes: 1, longestAgreement: 1 } + { source: "database", label: "42/1000001/web/akash1provider", probes: 1, accelFires: 1, networkFires: 1, candidateProbes: 1, longestAgreement: 1 } ]); }); @@ -185,6 +185,7 @@ describe(BehaviouralSignalReplayService.name, () => { return mock({ walletId: 42, dseq: "1000001", + provider: "akash1provider", shellStatus: "completed", accelerator: [{ name: "accelerator-0", utilPct: 99, memUsedMb: 20_480, memTotalMb: 24_576, processes: [{ pid: 1234, name: "worker", vramMb: 18_000 }] }], artifacts: [{ path: "/opt/worker", sizeBytes: 4_194_304 }], diff --git a/apps/api/src/workload-abuse/services/behavioural-signal-replay/behavioural-signal-replay.service.ts b/apps/api/src/workload-abuse/services/behavioural-signal-replay/behavioural-signal-replay.service.ts index a61c19a088..4ae9fc5829 100644 --- a/apps/api/src/workload-abuse/services/behavioural-signal-replay/behavioural-signal-replay.service.ts +++ b/apps/api/src/workload-abuse/services/behavioural-signal-replay/behavioural-signal-replay.service.ts @@ -124,7 +124,7 @@ export class BehaviouralSignalReplayService { for (const row of rows) { if (row.shellStatus !== COMPLETE_SHELL_STATUS) continue; - const label = `${row.walletId}/${row.dseq}/${row.service}`; + const label = `${row.walletId}/${row.dseq}/${row.service}/${row.provider}`; const series = byDeployment.get(label) ?? { source: "database" as const, label, snapshots: [] }; series.snapshots.push(row); byDeployment.set(label, series); diff --git a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts index bd39a51512..fd13db5da0 100644 --- a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts +++ b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts @@ -448,6 +448,7 @@ describe(ProbeTrialDeploymentHandler.name, () => { return minutesAgo.map(minutes => mock({ service: "web", + provider: "akash1provider", shellStatus: "completed", createdAt: new Date(Date.now() - minutes * 60_000), behaviouralFindings: [