diff --git a/apps/api/drizzle/0059_workload_abuse_behavioural_verdict.sql b/apps/api/drizzle/0059_workload_abuse_behavioural_verdict.sql new file mode 100644 index 0000000000..ca019dbb0c --- /dev/null +++ b/apps/api/drizzle/0059_workload_abuse_behavioural_verdict.sql @@ -0,0 +1 @@ +ALTER TYPE "public"."workload_abuse_verdict" ADD VALUE 'behavioural'; \ No newline at end of file diff --git a/apps/api/drizzle/meta/0059_snapshot.json b/apps/api/drizzle/meta/0059_snapshot.json new file mode 100644 index 0000000000..944a8a7ca7 --- /dev/null +++ b/apps/api/drizzle/meta/0059_snapshot.json @@ -0,0 +1,2054 @@ +{ + "id": "15591fc4-d0c0-4a54-8ea0-c72e70babfcc", + "prevId": "63bba39e-51aa-48a3-a83b-b019a98795f2", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.user_wallets": { + "name": "user_wallets", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "varchar", + "primaryKey": false, + "notNull": false + }, + "deployment_allowance": { + "name": "deployment_allowance", + "type": "numeric(20, 2)", + "primaryKey": false, + "notNull": true, + "default": "'0.00'" + }, + "fee_allowance": { + "name": "fee_allowance", + "type": "numeric(20, 2)", + "primaryKey": false, + "notNull": true, + "default": "'0.00'" + }, + "trial": { + "name": "trial", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": true + }, + "activated_at": { + "name": "activated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "credits_low_notified_at": { + "name": "credits_low_notified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "credits_sufficient_since": { + "name": "credits_sufficient_since", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "credits_low_since": { + "name": "credits_low_since", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "abuse_locked_at": { + "name": "abuse_locked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "abuse_locked_reason": { + "name": "abuse_locked_reason", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "user_wallets_user_id_userSetting_id_fk": { + "name": "user_wallets_user_id_userSetting_id_fk", + "tableFrom": "user_wallets", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_wallets_user_id_unique": { + "name": "user_wallets_user_id_unique", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + }, + "user_wallets_address_unique": { + "name": "user_wallets_address_unique", + "nullsNotDistinct": false, + "columns": [ + "address" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.payment_methods": { + "name": "payment_methods", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "fingerprint": { + "name": "fingerprint", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "payment_method_id": { + "name": "payment_method_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "is_validated": { + "name": "is_validated", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "payment_methods_fingerprint_payment_method_id_unique": { + "name": "payment_methods_fingerprint_payment_method_id_unique", + "columns": [ + { + "expression": "fingerprint", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "payment_method_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "payment_methods_user_id_is_default_unique": { + "name": "payment_methods_user_id_is_default_unique", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_default", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"payment_methods\".\"is_default\" = true", + "concurrently": false, + "method": "btree", + "with": {} + }, + "payment_methods_fingerprint_idx": { + "name": "payment_methods_fingerprint_idx", + "columns": [ + { + "expression": "fingerprint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "payment_methods_user_id_idx": { + "name": "payment_methods_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "payment_methods_user_id_is_validated_idx": { + "name": "payment_methods_user_id_is_validated_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_validated", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "payment_methods_user_id_fingerprint_payment_method_id_idx": { + "name": "payment_methods_user_id_fingerprint_payment_method_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "fingerprint", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "payment_method_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "payment_methods_user_id_userSetting_id_fk": { + "name": "payment_methods_user_id_userSetting_id_fk", + "tableFrom": "payment_methods", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.stripe_transactions": { + "name": "stripe_transactions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "stripe_transaction_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "stripe_transaction_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'created'" + }, + "amount": { + "name": "amount", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "amount_refunded": { + "name": "amount_refunded", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "bonus_amount": { + "name": "bonus_amount", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "currency": { + "name": "currency", + "type": "varchar(3)", + "primaryKey": false, + "notNull": true, + "default": "'usd'" + }, + "stripe_payment_intent_id": { + "name": "stripe_payment_intent_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "stripe_charge_id": { + "name": "stripe_charge_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "stripe_coupon_id": { + "name": "stripe_coupon_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "stripe_promotion_code_id": { + "name": "stripe_promotion_code_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "stripe_invoice_id": { + "name": "stripe_invoice_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "stripe_idempotency_key": { + "name": "stripe_idempotency_key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "payment_method_type": { + "name": "payment_method_type", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "card_brand": { + "name": "card_brand", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "card_last4": { + "name": "card_last4", + "type": "varchar(4)", + "primaryKey": false, + "notNull": false + }, + "receipt_url": { + "name": "receipt_url", + "type": "varchar(2048)", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "varchar(500)", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "varchar(1000)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "stripe_transactions_stripe_invoice_id_unique": { + "name": "stripe_transactions_stripe_invoice_id_unique", + "columns": [ + { + "expression": "stripe_invoice_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"stripe_transactions\".\"stripe_invoice_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_stripe_idempotency_key_unique": { + "name": "stripe_transactions_stripe_idempotency_key_unique", + "columns": [ + { + "expression": "stripe_idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"stripe_transactions\".\"stripe_idempotency_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_user_id_idx": { + "name": "stripe_transactions_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_stripe_payment_intent_id_idx": { + "name": "stripe_transactions_stripe_payment_intent_id_idx", + "columns": [ + { + "expression": "stripe_payment_intent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_stripe_charge_id_idx": { + "name": "stripe_transactions_stripe_charge_id_idx", + "columns": [ + { + "expression": "stripe_charge_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_stripe_coupon_id_idx": { + "name": "stripe_transactions_stripe_coupon_id_idx", + "columns": [ + { + "expression": "stripe_coupon_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_stripe_promotion_code_id_idx": { + "name": "stripe_transactions_stripe_promotion_code_id_idx", + "columns": [ + { + "expression": "stripe_promotion_code_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_status_idx": { + "name": "stripe_transactions_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_created_at_idx": { + "name": "stripe_transactions_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "stripe_transactions_user_id_created_at_idx": { + "name": "stripe_transactions_user_id_created_at_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "stripe_transactions_user_id_userSetting_id_fk": { + "name": "stripe_transactions_user_id_userSetting_id_fk", + "tableFrom": "stripe_transactions", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.wallet_settings": { + "name": "wallet_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "wallet_id": { + "name": "wallet_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "auto_reload_enabled": { + "name": "auto_reload_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "auto_reload_mode": { + "name": "auto_reload_mode", + "type": "auto_reload_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'prediction'" + }, + "auto_reload_threshold": { + "name": "auto_reload_threshold", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 2000 + }, + "auto_reload_amount": { + "name": "auto_reload_amount", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 10000 + }, + "last_auto_charge_at": { + "name": "last_auto_charge_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "auto_reload_failure_count": { + "name": "auto_reload_failure_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "auto_reload_paused_at": { + "name": "auto_reload_paused_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "wallet_settings_user_id_idx": { + "name": "wallet_settings_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "wallet_settings_wallet_id_user_wallets_id_fk": { + "name": "wallet_settings_wallet_id_user_wallets_id_fk", + "tableFrom": "wallet_settings", + "tableTo": "user_wallets", + "columnsFrom": [ + "wallet_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "wallet_settings_user_id_userSetting_id_fk": { + "name": "wallet_settings_user_id_userSetting_id_fk", + "tableFrom": "wallet_settings", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "wallet_settings_wallet_id_unique": { + "name": "wallet_settings_wallet_id_unique", + "nullsNotDistinct": false, + "columns": [ + "wallet_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.userSetting": { + "name": "userSetting", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "userId": { + "name": "userId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "username": { + "name": "username", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "emailVerified": { + "name": "emailVerified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "stripeCustomerId": { + "name": "stripeCustomerId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "bio": { + "name": "bio", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "subscribedToNewsletter": { + "name": "subscribedToNewsletter", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "youtubeUsername": { + "name": "youtubeUsername", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "twitterUsername": { + "name": "twitterUsername", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "githubUsername": { + "name": "githubUsername", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "last_active_at": { + "name": "last_active_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "last_ip": { + "name": "last_ip", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "last_user_agent": { + "name": "last_user_agent", + "type": "varchar(500)", + "primaryKey": false, + "notNull": false + }, + "last_fingerprint": { + "name": "last_fingerprint", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "onboardingSkippedAt": { + "name": "onboardingSkippedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "fair_use_policy_accepted_at": { + "name": "fair_use_policy_accepted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "userSetting_userId_unique": { + "name": "userSetting_userId_unique", + "nullsNotDistinct": false, + "columns": [ + "userId" + ] + }, + "userSetting_username_unique": { + "name": "userSetting_username_unique", + "nullsNotDistinct": false, + "columns": [ + "username" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.template": { + "name": "template", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "userId": { + "name": "userId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "copiedFromId": { + "name": "copiedFromId", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "isPublic": { + "name": "isPublic", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "cpu": { + "name": "cpu", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "ram": { + "name": "ram", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "storage": { + "name": "storage", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "sdl": { + "name": "sdl", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "template_userId_idx": { + "name": "template_userId_idx", + "columns": [ + { + "expression": "userId", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.templateFavorite": { + "name": "templateFavorite", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "userId": { + "name": "userId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "templateId": { + "name": "templateId", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "addedDate": { + "name": "addedDate", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "templateFavorite_userId_templateId_unique": { + "name": "templateFavorite_userId_templateId_unique", + "columns": [ + { + "expression": "userId", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "templateId", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "templateFavorite_templateId_template_id_fk": { + "name": "templateFavorite_templateId_template_id_fk", + "tableFrom": "templateFavorite", + "tableTo": "template", + "columnsFrom": [ + "templateId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.deployment_settings": { + "name": "deployment_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "dseq": { + "name": "dseq", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "auto_top_up_enabled": { + "name": "auto_top_up_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "closed": { + "name": "closed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "last_funded_at": { + "name": "last_funded_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "runtime_limit_hours": { + "name": "runtime_limit_hours", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "sdl": { + "name": "sdl", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sealed_secrets": { + "name": "sealed_secrets", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "manifest_version": { + "name": "manifest_version", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "runtime_ends_at": { + "name": "runtime_ends_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "runtime_ending_notified_for": { + "name": "runtime_ending_notified_for", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "provider_unreachable_notified_for": { + "name": "provider_unreachable_notified_for", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "id_auto_top_up_enabled_closed_idx": { + "name": "id_auto_top_up_enabled_closed_idx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "auto_top_up_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "closed", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_id_id_sealed_secrets_idx": { + "name": "user_id_id_sealed_secrets_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"deployment_settings\".\"sealed_secrets\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "deployment_settings_user_id_userSetting_id_fk": { + "name": "deployment_settings_user_id_userSetting_id_fk", + "tableFrom": "deployment_settings", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "dseq_user_id_idx": { + "name": "dseq_user_id_idx", + "nullsNotDistinct": false, + "columns": [ + "dseq", + "user_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.api_keys": { + "name": "api_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "hashed_key": { + "name": "hashed_key", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "key_format": { + "name": "key_format", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "api_keys_user_id_userSetting_id_fk": { + "name": "api_keys_user_id_userSetting_id_fk", + "tableFrom": "api_keys", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "api_keys_hashed_key_unique": { + "name": "api_keys_hashed_key_unique", + "nullsNotDistinct": false, + "columns": [ + "hashed_key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.email_verification_codes": { + "name": "email_verification_codes", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "code": { + "name": "code", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "email_verification_codes_user_id_idx": { + "name": "email_verification_codes_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "email_verification_codes_user_id_userSetting_id_fk": { + "name": "email_verification_codes_user_id_userSetting_id_fk", + "tableFrom": "email_verification_codes", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_keys": { + "name": "data_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "wrapped_key": { + "name": "wrapped_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "wrapped_by_kid": { + "name": "wrapped_by_kid", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "retired_at": { + "name": "retired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_keys_active_user_id_idx": { + "name": "data_keys_active_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_keys\".\"retired_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_keys_wrapped_by_kid_idx": { + "name": "data_keys_wrapped_by_kid_idx", + "columns": [ + { + "expression": "wrapped_by_kid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_keys_user_id_userSetting_id_fk": { + "name": "data_keys_user_id_userSetting_id_fk", + "tableFrom": "data_keys", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.blocked_email_domains": { + "name": "blocked_email_domains", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "domain": { + "name": "domain", + "type": "varchar(253)", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "blocked_email_domain_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'blocked'" + }, + "source": { + "name": "source", + "type": "blocked_email_domain_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'manual'" + }, + "reason": { + "name": "reason", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "triggered_by_user_id": { + "name": "triggered_by_user_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "blocked_email_domains_domain_unique": { + "name": "blocked_email_domains_domain_unique", + "columns": [ + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "blocked_email_domains_triggered_by_user_id_userSetting_id_fk": { + "name": "blocked_email_domains_triggered_by_user_id_userSetting_id_fk", + "tableFrom": "blocked_email_domains", + "tableTo": "userSetting", + "columnsFrom": [ + "triggered_by_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "blocked_email_domains_domain_normalized": { + "name": "blocked_email_domains_domain_normalized", + "value": "\"blocked_email_domains\".\"domain\" ~ '^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?(?:\\.[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)+$'" + } + }, + "isRLSEnabled": false + }, + "public.workload_abuse_detections": { + "name": "workload_abuse_detections", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "wallet_id": { + "name": "wallet_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "dseq": { + "name": "dseq", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "verdict": { + "name": "verdict", + "type": "workload_abuse_verdict", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "probe_status": { + "name": "probe_status", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "signals": { + "name": "signals", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "evidence_excerpt": { + "name": "evidence_excerpt", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "workload_abuse_action", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'detected'" + }, + "enforcement_error": { + "name": "enforcement_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workload_abuse_detections_user_id_idx": { + "name": "workload_abuse_detections_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workload_abuse_detections_dseq_idx": { + "name": "workload_abuse_detections_dseq_idx", + "columns": [ + { + "expression": "dseq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workload_abuse_detections_user_id_userSetting_id_fk": { + "name": "workload_abuse_detections_user_id_userSetting_id_fk", + "tableFrom": "workload_abuse_detections", + "tableTo": "userSetting", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workload_probe_evidence": { + "name": "workload_probe_evidence", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuid_generate_v4()" + }, + "wallet_id": { + "name": "wallet_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "dseq": { + "name": "dseq", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service": { + "name": "service", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "probe_status": { + "name": "probe_status", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "verdict": { + "name": "verdict", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true + }, + "detection_id": { + "name": "detection_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "accelerator": { + "name": "accelerator", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "process_origins": { + "name": "process_origins", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "net_shape": { + "name": "net_shape", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "behavioural_findings": { + "name": "behavioural_findings", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workload_probe_evidence_wallet_dseq_created_idx": { + "name": "workload_probe_evidence_wallet_dseq_created_idx", + "columns": [ + { + "expression": "wallet_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dseq", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workload_probe_evidence_created_idx": { + "name": "workload_probe_evidence_created_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.stripe_transaction_status": { + "name": "stripe_transaction_status", + "schema": "public", + "values": [ + "created", + "pending", + "requires_action", + "succeeded", + "failed", + "refunded", + "canceled" + ] + }, + "public.stripe_transaction_type": { + "name": "stripe_transaction_type", + "schema": "public", + "values": [ + "payment_intent", + "coupon_claim", + "manual_credit" + ] + }, + "public.auto_reload_mode": { + "name": "auto_reload_mode", + "schema": "public", + "values": [ + "prediction", + "threshold" + ] + }, + "public.blocked_email_domain_source": { + "name": "blocked_email_domain_source", + "schema": "public", + "values": [ + "auto", + "manual" + ] + }, + "public.blocked_email_domain_status": { + "name": "blocked_email_domain_status", + "schema": "public", + "values": [ + "blocked", + "allowed" + ] + }, + "public.workload_abuse_action": { + "name": "workload_abuse_action", + "schema": "public", + "values": [ + "detected", + "enforcing", + "enforced", + "enforcement_failed" + ] + }, + "public.workload_abuse_verdict": { + "name": "workload_abuse_verdict", + "schema": "public", + "values": [ + "hard", + "soft", + "proxy", + "behavioural" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/apps/api/drizzle/meta/_journal.json b/apps/api/drizzle/meta/_journal.json index 4eb242cd4a..67948e3926 100644 --- a/apps/api/drizzle/meta/_journal.json +++ b/apps/api/drizzle/meta/_journal.json @@ -414,6 +414,13 @@ "when": 1789839246549, "tag": "0058_workload_probe_evidence", "breakpoints": true + }, + { + "idx": 59, + "version": "7", + "when": 1789901600865, + "tag": "0059_workload_abuse_behavioural_verdict", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/api/src/workload-abuse/config/env.config.spec.ts b/apps/api/src/workload-abuse/config/env.config.spec.ts index c4858948a9..01738260cb 100644 --- a/apps/api/src/workload-abuse/config/env.config.spec.ts +++ b/apps/api/src/workload-abuse/config/env.config.spec.ts @@ -36,6 +36,14 @@ describe("workload abuse env config", () => { expect(config.WORKLOAD_ABUSE_SIGNAL_ARTIFACT_MIN_MB).toBe(256); }); + it("defaults behavioural enforcement to off behind a multi-probe agreement", () => { + const config = envSchema.parse({}); + + expect(config.WORKLOAD_ABUSE_BEHAVIOURAL_ENFORCEMENT_ENABLED).toBe(false); + expect(config.WORKLOAD_ABUSE_BEHAVIOURAL_AGREEMENT_PROBES).toBe(3); + expect(config.WORKLOAD_ABUSE_BEHAVIOURAL_MIN_WINDOW_MINUTES).toBe(120); + }); + it("parses the relay endpoint list", () => { const config = envSchema.parse({ WORKLOAD_ABUSE_SIGNAL_RELAY_ENDPOINTS: JSON.stringify(["10.0.0.7", "10.0.0.8:443"]) }); @@ -85,6 +93,30 @@ describe("workload abuse env config", () => { expect(config.WORKLOAD_ABUSE_SIGNAL_ARTIFACT_MIN_MB).toBe(256); }); + it("falls back to the enforcement defaults when those variables are set but blank", () => { + const config = envSchema.parse({ + WORKLOAD_ABUSE_BEHAVIOURAL_ENFORCEMENT_ENABLED: "", + WORKLOAD_ABUSE_BEHAVIOURAL_AGREEMENT_PROBES: "", + WORKLOAD_ABUSE_BEHAVIOURAL_MIN_WINDOW_MINUTES: " " + }); + + expect(config.WORKLOAD_ABUSE_BEHAVIOURAL_ENFORCEMENT_ENABLED).toBe(false); + expect(config.WORKLOAD_ABUSE_BEHAVIOURAL_AGREEMENT_PROBES).toBe(3); + expect(config.WORKLOAD_ABUSE_BEHAVIOURAL_MIN_WINDOW_MINUTES).toBe(120); + }); + + it("still reads the enforcement variables when they carry a value", () => { + const config = envSchema.parse({ + WORKLOAD_ABUSE_BEHAVIOURAL_ENFORCEMENT_ENABLED: "true", + WORKLOAD_ABUSE_BEHAVIOURAL_AGREEMENT_PROBES: "5", + WORKLOAD_ABUSE_BEHAVIOURAL_MIN_WINDOW_MINUTES: "240" + }); + + expect(config.WORKLOAD_ABUSE_BEHAVIOURAL_ENFORCEMENT_ENABLED).toBe(true); + expect(config.WORKLOAD_ABUSE_BEHAVIOURAL_AGREEMENT_PROBES).toBe(5); + expect(config.WORKLOAD_ABUSE_BEHAVIOURAL_MIN_WINDOW_MINUTES).toBe(240); + }); + it("still reads the behavioural signal variables when they carry a value", () => { const config = envSchema.parse({ WORKLOAD_ABUSE_BEHAVIOURAL_SIGNALS_ENABLED: "true", diff --git a/apps/api/src/workload-abuse/config/env.config.ts b/apps/api/src/workload-abuse/config/env.config.ts index 6d6fd14e7f..8fb9c1e480 100644 --- a/apps/api/src/workload-abuse/config/env.config.ts +++ b/apps/api/src/workload-abuse/config/env.config.ts @@ -121,7 +121,17 @@ export const envSchema = z.object({ WORKLOAD_ABUSE_SIGNAL_ACCEL_MIN_VRAM_MB: z.preprocess(blankToUndefined, z.number({ coerce: true }).int().positive().default(1_024)), WORKLOAD_ABUSE_SIGNAL_ARTIFACT_MIN_MB: z.preprocess(blankToUndefined, z.number({ coerce: true }).int().positive().default(256)), /** The list itself lives in Doppler: these are our own endpoints, so a reader learns how the exclusion works but not what it covers. */ - WORKLOAD_ABUSE_SIGNAL_RELAY_ENDPOINTS: z.preprocess(blankToUndefined, z.string().default("[]").transform(parseRelayEndpoints)) + WORKLOAD_ABUSE_SIGNAL_RELAY_ENDPOINTS: z.preprocess(blankToUndefined, z.string().default("[]").transform(parseRelayEndpoints)), + WORKLOAD_ABUSE_BEHAVIOURAL_ENFORCEMENT_ENABLED: z.preprocess( + blankToUndefined, + z + .enum(["true", "false"]) + .default("false") + .transform(value => value === "true") + ), + WORKLOAD_ABUSE_BEHAVIOURAL_AGREEMENT_PROBES: z.preprocess(blankToUndefined, z.number({ coerce: true }).int().positive().default(3)), + /** Back-to-back probes can land minutes apart, so a streak also has to span real time before it counts. */ + WORKLOAD_ABUSE_BEHAVIOURAL_MIN_WINDOW_MINUTES: z.preprocess(blankToUndefined, z.number({ coerce: true }).int().positive().default(120)) }); export type WorkloadAbuseConfig = z.infer; diff --git a/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.spec.ts b/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.spec.ts new file mode 100644 index 0000000000..72fd1b1a51 --- /dev/null +++ b/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.spec.ts @@ -0,0 +1,125 @@ +import { describe, expect, it } from "vitest"; + +import { type AgreementRow, findBehaviouralAgreement } from "./agreement"; +import type { BehaviouralFinding } from "./types"; + +const NOW = new Date("2026-09-20T12:00:00.000Z"); + +const BOTH_SIGNALS: BehaviouralFinding[] = [ + { signal: "accel_without_artifacts", detail: {} }, + { signal: "network_isolated", detail: {} } +]; + +describe("findBehaviouralAgreement", () => { + it("agrees once enough consecutive probes see the same shape over a long enough window", () => { + const { rows, params } = setup({ minutesAgo: [10, 130, 250] }); + + expect(findBehaviouralAgreement(rows, params)).toEqual({ agreed: true, service: "web", streak: 3, spanMinutes: 250 }); + }); + + it("holds off while the streak is one probe short", () => { + const { rows, params } = setup({ minutesAgo: [10, 130] }); + + expect(findBehaviouralAgreement(rows, params)).toMatchObject({ agreed: false, streak: 2 }); + }); + + it("holds off while the streak is younger than the minimum window", () => { + const { rows, params } = setup({ minutesAgo: [1, 3, 5] }); + + expect(findBehaviouralAgreement(rows, params)).toMatchObject({ agreed: false, streak: 3, spanMinutes: 5 }); + }); + + it("breaks the streak on the first probe that saw a different shape", () => { + const { rows, params } = setup({ minutesAgo: [10, 130, 250], findingsByIndex: { 1: [{ signal: "network_isolated", detail: {} }] } }); + + expect(findBehaviouralAgreement(rows, params)).toMatchObject({ agreed: false, streak: 1 }); + }); + + it("steps over a probe that collected nothing without breaking the streak", () => { + const { rows, params } = setup({ minutesAgo: [10, 130, 250, 320], statusByIndex: { 1: "shell_unavailable" } }); + + expect(findBehaviouralAgreement(rows, params)).toMatchObject({ agreed: true, streak: 3, spanMinutes: 320 }); + }); + + it("agrees on the service that holds the streak rather than on the deployment as a whole", () => { + const { rows, params } = setup({ minutesAgo: [10, 130, 250] }); + const sidecar = rows.map(row => ({ ...row, service: "sidecar", behaviouralFindings: null })); + + expect(findBehaviouralAgreement([...rows, ...sidecar], params)).toMatchObject({ agreed: true, service: "web", streak: 3 }); + }); + + it("orders a history that arrives out of order before counting the streak", () => { + const { rows, params } = setup({ minutesAgo: [250, 10, 130], findingsByIndex: { 0: [{ signal: "network_isolated", detail: {} }] } }); + + expect(findBehaviouralAgreement(rows, params)).toMatchObject({ agreed: false, streak: 2, spanMinutes: 130 }); + }); + + it("breaks the streak on a probe that recorded no findings at all", () => { + const { rows, params } = setup({ minutesAgo: [10, 130, 250], nullFindingsAt: 1 }); + + expect(findBehaviouralAgreement(rows, params)).toMatchObject({ agreed: false, streak: 1 }); + }); + + it("agrees once the streak is exactly as old as the minimum window", () => { + const { rows, params } = setup({ minutesAgo: [10, 60, 120], minWindowMinutes: 120 }); + + expect(findBehaviouralAgreement(rows, params)).toMatchObject({ agreed: true, streak: 3, spanMinutes: 120 }); + }); + + it("prefers the service that agreed over one carrying a longer streak that has not", () => { + const { rows, params } = setup({ minutesAgo: [10, 130, 250] }); + const sidecar = setup({ minutesAgo: [1, 2, 3, 4] }).rows.map(row => ({ ...row, service: "sidecar" })); + + expect(findBehaviouralAgreement([...rows, ...sidecar], params)).toMatchObject({ agreed: true, service: "web", streak: 3 }); + }); + + it("reports the longest streak while no service has agreed", () => { + const { rows, params } = setup({ minutesAgo: [10, 130] }); + const sidecar = setup({ minutesAgo: [1, 2, 3, 4] }).rows.map(row => ({ ...row, service: "sidecar" })); + + expect(findBehaviouralAgreement([...rows, ...sidecar], params)).toMatchObject({ agreed: false, service: "sidecar", streak: 4 }); + }); + + it("keeps the first service when two are equally short of agreement", () => { + const { rows, params } = setup({ minutesAgo: [10, 130] }); + const sidecar = rows.map(row => ({ ...row, service: "sidecar" })); + + expect(findBehaviouralAgreement([...rows, ...sidecar], params)).toMatchObject({ agreed: false, service: "web", streak: 2 }); + }); + + it("counts a service seen on two providers at once as two series rather than one longer streak", () => { + const { rows, params } = setup({ minutesAgo: [10, 130] }); + const reclaiming = setup({ minutesAgo: [11, 131], provider: "akash1reclaiming" }).rows; + + expect(findBehaviouralAgreement([...rows, ...reclaiming], params)).toMatchObject({ agreed: false, streak: 2 }); + }); + + it("finds no agreement in an empty history", () => { + const { params } = setup({ minutesAgo: [] }); + + expect(findBehaviouralAgreement([], params)).toEqual({ agreed: false, service: null, streak: 0, spanMinutes: 0 }); + }); + + function setup(input: { + minutesAgo: number[]; + agreementProbes?: number; + minWindowMinutes?: number; + findingsByIndex?: Record; + statusByIndex?: Record; + nullFindingsAt?: number; + provider?: string; + }) { + const rows: AgreementRow[] = input.minutesAgo.map((minutes, index) => ({ + service: "web", + provider: input.provider ?? "akash1provider", + createdAt: new Date(NOW.getTime() - minutes * 60_000), + shellStatus: input.statusByIndex?.[index] ?? "completed", + behaviouralFindings: index === input.nullFindingsAt ? null : input.findingsByIndex?.[index] ?? BOTH_SIGNALS + })); + + return { + rows, + params: { agreementProbes: input.agreementProbes ?? 3, minWindowMinutes: input.minWindowMinutes ?? 120, now: NOW } + }; + } +}); diff --git a/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.ts b/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.ts new file mode 100644 index 0000000000..774eb0cbfe --- /dev/null +++ b/apps/api/src/workload-abuse/lib/behavioural-signals/agreement.ts @@ -0,0 +1,74 @@ +import { isBehaviouralCandidate } from "./evaluate-behavioural-signals"; +import { type BehaviouralFinding, COMPLETE_SHELL_STATUS } from "./types"; + +const MS_PER_MINUTE = 60_000; + +export type AgreementRow = { + service: string; + provider: string; + createdAt: Date; + shellStatus: string; + behaviouralFindings: BehaviouralFinding[] | null; +}; + +export type BehaviouralAgreement = { + agreed: boolean; + service: string | null; + streak: number; + spanMinutes: number; +}; + +export type AgreementParams = { + agreementProbes: number; + minWindowMinutes: number; + now?: Date; +}; + +/** A probe that could not collect anything neither advances nor breaks the streak, because a flaking provider would otherwise put agreement out of reach. */ +export function findBehaviouralAgreement(rows: AgreementRow[], params: AgreementParams): BehaviouralAgreement { + const now = params.now ?? new Date(); + const perService = [...groupByService(rows).values()].map(serviceRows => measureService(serviceRows, params, now)); + + return perService.reduce(pickStrongest, { agreed: false, service: null, streak: 0, spanMinutes: 0 }); +} + +/** A lease under reclamation overlaps the one replacing it, so a single run can write two rows for one service and only a per provider series counts distinct probes. */ +function groupByService(rows: AgreementRow[]): Map { + const grouped = new Map(); + + for (const row of rows) { + const key = `${row.service}/${row.provider}`; + const serviceRows = grouped.get(key) ?? []; + serviceRows.push(row); + grouped.set(key, serviceRows); + } + + return grouped; +} + +function measureService(rows: AgreementRow[], params: AgreementParams, now: Date): BehaviouralAgreement { + const newestFirst = [...rows].sort((left, right) => right.createdAt.getTime() - left.createdAt.getTime()); + const streak: AgreementRow[] = []; + + for (const row of newestFirst) { + if (row.shellStatus !== COMPLETE_SHELL_STATUS) continue; + if (!row.behaviouralFindings || !isBehaviouralCandidate(row.behaviouralFindings)) break; + streak.push(row); + } + + const oldest = streak.at(-1); + const spanMinutes = oldest ? Math.floor((now.getTime() - oldest.createdAt.getTime()) / MS_PER_MINUTE) : 0; + + return { + agreed: streak.length >= params.agreementProbes && spanMinutes >= params.minWindowMinutes, + service: newestFirst[0].service, + streak: streak.length, + spanMinutes + }; +} + +function pickStrongest(strongest: BehaviouralAgreement, candidate: BehaviouralAgreement): BehaviouralAgreement { + if (candidate.agreed !== strongest.agreed) return candidate.agreed ? candidate : strongest; + + return candidate.streak > strongest.streak ? candidate : strongest; +} diff --git a/apps/api/src/workload-abuse/model-schemas/workload-abuse-detection/workload-abuse-detection.schema.ts b/apps/api/src/workload-abuse/model-schemas/workload-abuse-detection/workload-abuse-detection.schema.ts index 208008a367..bb34a9ad08 100644 --- a/apps/api/src/workload-abuse/model-schemas/workload-abuse-detection/workload-abuse-detection.schema.ts +++ b/apps/api/src/workload-abuse/model-schemas/workload-abuse-detection/workload-abuse-detection.schema.ts @@ -4,7 +4,7 @@ import { index, integer, jsonb, pgEnum, pgTable, text, timestamp, uuid, varchar import { Users } from "@src/user/model-schemas"; import type { DetectionSignal } from "@src/workload-abuse/lib/evidence-scanner/evidence-scanner"; -export const workloadAbuseVerdictEnum = pgEnum("workload_abuse_verdict", ["hard", "soft", "proxy"]); +export const workloadAbuseVerdictEnum = pgEnum("workload_abuse_verdict", ["hard", "soft", "proxy", "behavioural"]); export const workloadAbuseActionEnum = pgEnum("workload_abuse_action", ["detected", "enforcing", "enforced", "enforcement_failed"]); diff --git a/apps/api/src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository.integration.ts b/apps/api/src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository.integration.ts index 42931d9348..307fee154e 100644 --- a/apps/api/src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository.integration.ts +++ b/apps/api/src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository.integration.ts @@ -7,7 +7,7 @@ import { UserRepository } from "@src/user/repositories"; import { WorkloadAbuseDetectionRepository } from "./workload-abuse-detection.repository"; describe(WorkloadAbuseDetectionRepository.name, () => { - describe("findRecentHardTargets", () => { + describe("findRecentDetectedTargets", () => { it("lists each deployment with a confirmed detection inside the window once", async () => { const { repository, walletId, createDetection } = await setup(); await createDetection({ dseq: "1", verdict: "hard" }); @@ -15,7 +15,16 @@ describe(WorkloadAbuseDetectionRepository.name, () => { await createDetection({ dseq: "2", verdict: "soft" }); await createDetection({ dseq: "3", verdict: "hard", createdAt: subHours(new Date(), 30) }); - const targets = await repository.findRecentHardTargets({ since: subHours(new Date(), 26) }); + const targets = await repository.findRecentDetectedTargets({ since: subHours(new Date(), 26) }); + + expect(targets.filter(target => target.walletId === walletId)).toEqual([{ walletId, dseq: "1" }]); + }); + + it("lists a deployment confirmed by repeated behaviour alongside the rest", async () => { + const { repository, walletId, createDetection } = await setup(); + await createDetection({ dseq: "1", verdict: "behavioural" }); + + const targets = await repository.findRecentDetectedTargets({ since: subHours(new Date(), 26) }); expect(targets.filter(target => target.walletId === walletId)).toEqual([{ walletId, dseq: "1" }]); }); @@ -46,7 +55,7 @@ describe(WorkloadAbuseDetectionRepository.name, () => { async function createDetection(input: { dseq: string; - verdict: "hard" | "soft" | "proxy"; + verdict: "hard" | "soft" | "proxy" | "behavioural"; action?: "detected" | "enforcing" | "enforced" | "enforcement_failed"; createdAt?: Date; }) { diff --git a/apps/api/src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository.ts b/apps/api/src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository.ts index 180f201201..0a5a8180e3 100644 --- a/apps/api/src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository.ts +++ b/apps/api/src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository.ts @@ -24,11 +24,11 @@ export class WorkloadAbuseDetectionRepository extends BaseRepository> { + async findRecentDetectedTargets({ since }: { since: Date }): Promise> { return await this.cursor .selectDistinct({ walletId: this.table.walletId, dseq: this.table.dseq }) .from(this.table) - .where(and(eq(this.table.verdict, "hard"), gt(this.table.createdAt, since))); + .where(and(inArray(this.table.verdict, ["hard", "behavioural"]), gt(this.table.createdAt, since))); } /** One detection per wallet, since the wipe covers the whole wallet whichever of its detections re-queues it. */ diff --git a/apps/api/src/workload-abuse/services/behavioural-signal-replay/behavioural-signal-replay.service.spec.ts b/apps/api/src/workload-abuse/services/behavioural-signal-replay/behavioural-signal-replay.service.spec.ts index d00d726c31..6e67575bba 100644 --- a/apps/api/src/workload-abuse/services/behavioural-signal-replay/behavioural-signal-replay.service.spec.ts +++ b/apps/api/src/workload-abuse/services/behavioural-signal-replay/behavioural-signal-replay.service.spec.ts @@ -97,8 +97,8 @@ describe(BehaviouralSignalReplayService.name, () => { until: new Date("2026-09-01T00:00:00.000Z") }); expect(summary.deployments).toEqual([ - { source: "database", label: "42/1000001/web", probes: 2, accelFires: 2, networkFires: 2, candidateProbes: 2, longestAgreement: 2 }, - { source: "database", label: "42/1000001/sidecar", probes: 1, accelFires: 0, networkFires: 1, candidateProbes: 0, longestAgreement: 0 } + { source: "database", label: "42/1000001/web/akash1provider", probes: 2, accelFires: 2, networkFires: 2, candidateProbes: 2, longestAgreement: 2 }, + { source: "database", label: "42/1000001/sidecar/akash1provider", probes: 1, accelFires: 0, networkFires: 1, candidateProbes: 0, longestAgreement: 0 } ]); }); @@ -110,7 +110,7 @@ describe(BehaviouralSignalReplayService.name, () => { const summary = await service.replay({ since: new Date("2026-08-01T00:00:00.000Z") }); expect(summary.deployments).toEqual([ - { source: "database", label: "42/1000001/web", probes: 1, accelFires: 1, networkFires: 1, candidateProbes: 1, longestAgreement: 1 } + { source: "database", label: "42/1000001/web/akash1provider", probes: 1, accelFires: 1, networkFires: 1, candidateProbes: 1, longestAgreement: 1 } ]); }); @@ -185,6 +185,7 @@ describe(BehaviouralSignalReplayService.name, () => { return mock({ walletId: 42, dseq: "1000001", + provider: "akash1provider", shellStatus: "completed", accelerator: [{ name: "accelerator-0", utilPct: 99, memUsedMb: 20_480, memTotalMb: 24_576, processes: [{ pid: 1234, name: "worker", vramMb: 18_000 }] }], artifacts: [{ path: "/opt/worker", sizeBytes: 4_194_304 }], diff --git a/apps/api/src/workload-abuse/services/behavioural-signal-replay/behavioural-signal-replay.service.ts b/apps/api/src/workload-abuse/services/behavioural-signal-replay/behavioural-signal-replay.service.ts index a61c19a088..4ae9fc5829 100644 --- a/apps/api/src/workload-abuse/services/behavioural-signal-replay/behavioural-signal-replay.service.ts +++ b/apps/api/src/workload-abuse/services/behavioural-signal-replay/behavioural-signal-replay.service.ts @@ -124,7 +124,7 @@ export class BehaviouralSignalReplayService { for (const row of rows) { if (row.shellStatus !== COMPLETE_SHELL_STATUS) continue; - const label = `${row.walletId}/${row.dseq}/${row.service}`; + const label = `${row.walletId}/${row.dseq}/${row.service}/${row.provider}`; const series = byDeployment.get(label) ?? { source: "database" as const, label, snapshots: [] }; series.snapshots.push(row); byDeployment.set(label, series); diff --git a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.integration.ts b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.integration.ts index dd46ba05e8..735b9b1e0d 100644 --- a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.integration.ts +++ b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.integration.ts @@ -142,6 +142,36 @@ describe(ProbeTrialDeploymentHandler.name, () => { expect(evidence.behaviouralFindings).toBeNull(); }); + it("confirms the deployment and stops probing once the recorded shape agrees across probes", async () => { + const { probeDeployment, findDetections, findNextProbe, seedShapeHistory } = await setup({ + verdict: "clean", + behaviouralSignalsEnabled: true, + behaviouralEnforcementEnabled: true, + shellEvidence: [{ service: "ssh", provider: "akash1provider", status: "completed", evidence: ISOLATED_ACCELERATED_EVIDENCE }] + }); + await seedShapeHistory([130, 250]); + + await probeDeployment(); + + expect(await findDetections()).toMatchObject([{ verdict: "behavioural", probeStatus: "probed" }]); + expect(await findNextProbe()).toBeUndefined(); + }); + + it("keeps probing while the recorded shape is one probe short of agreement", async () => { + const { probeDeployment, findDetections, findNextProbe, seedShapeHistory } = await setup({ + verdict: "clean", + behaviouralSignalsEnabled: true, + behaviouralEnforcementEnabled: true, + shellEvidence: [{ service: "ssh", provider: "akash1provider", status: "completed", evidence: ISOLATED_ACCELERATED_EVIDENCE }] + }); + await seedShapeHistory([130]); + + await probeDeployment(); + + expect(await findDetections()).toHaveLength(0); + expect((await findNextProbe())?.data).toMatchObject({ attempt: 2 }); + }); + it("records nothing more for a deployment already judged abusive", async () => { const { probeDeployment, findDetections, probe, seedExistingDetection } = await setup({ verdict: "hard" }); await seedExistingDetection(); @@ -159,6 +189,7 @@ describe(ProbeTrialDeploymentHandler.name, () => { probeStatus?: ProbeReport["probeStatus"]; shellEvidence?: ProbeReport["shellEvidence"]; behaviouralSignalsEnabled?: boolean; + behaviouralEnforcementEnabled?: boolean; }) { const { enqueue, startWorkers } = await jobWorkers(); const detectionRepository = container.resolve(WorkloadAbuseDetectionRepository); @@ -170,7 +201,8 @@ describe(ProbeTrialDeploymentHandler.name, () => { const readConfig = config.get.bind(config); const overrides: Record = { WORKLOAD_ABUSE_PROBE_ENABLED: true, - WORKLOAD_ABUSE_BEHAVIOURAL_SIGNALS_ENABLED: input.behaviouralSignalsEnabled ?? false + WORKLOAD_ABUSE_BEHAVIOURAL_SIGNALS_ENABLED: input.behaviouralSignalsEnabled ?? false, + WORKLOAD_ABUSE_BEHAVIOURAL_ENFORCEMENT_ENABLED: input.behaviouralEnforcementEnabled ?? false }; vi.spyOn(config, "get").mockImplementation((key => (key in overrides ? overrides[key] : readConfig(key))) as typeof config.get); @@ -190,6 +222,22 @@ describe(ProbeTrialDeploymentHandler.name, () => { probe, findDetections: () => detectionRepository.find({ walletId: wallet.id, dseq }), findEvidence: () => evidenceRepository.find({ walletId: wallet.id, dseq }), + seedShapeHistory: (minutesAgo: number[]) => + evidenceRepository.insertMany( + minutesAgo.map(minutes => ({ + walletId: wallet.id, + dseq, + provider: "akash1provider", + service: "ssh", + shellStatus: "completed", + verdict: "clean", + createdAt: new Date(Date.now() - minutes * 60_000), + behaviouralFindings: [ + { signal: "accel_without_artifacts", detail: {} }, + { signal: "network_isolated", detail: {} } + ] + })) + ), seedExistingDetection: () => detectionRepository.create({ userId: user.id, diff --git a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts index e8278f01b1..fd13db5da0 100644 --- a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts +++ b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.spec.ts @@ -7,7 +7,10 @@ import type { WorkloadAbuseDetectionOutput, WorkloadAbuseDetectionRepository } from "@src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository"; -import type { WorkloadProbeEvidenceOutput } from "@src/workload-abuse/repositories/workload-probe-evidence/workload-probe-evidence.repository"; +import type { + WorkloadProbeEvidenceOutput, + WorkloadProbeEvidenceRepository +} from "@src/workload-abuse/repositories/workload-probe-evidence/workload-probe-evidence.repository"; import { EnforceTrialAbuse } from "@src/workload-abuse/services/enforce-trial-abuse/enforce-trial-abuse.handler"; import type { ProbeEvidenceService, RecordedBehaviouralFindings } from "@src/workload-abuse/services/probe-evidence/probe-evidence.service"; import type { ProbeReport, TrialWorkloadProbeService } from "@src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service"; @@ -228,6 +231,160 @@ describe(ProbeTrialDeploymentHandler.name, () => { }); }); + it("records a detection and queues the wipe once the same shape holds across probes", async () => { + const { handler, wallet, detectionRepository, jobQueueService, instrumentation } = setup({ + report: createReport({ verdict: "clean" }), + behaviouralEnforcement: true, + enforcementMode: "enforce", + deploymentEvidence: createShapeHistory([10, 130, 250]) + }); + + await handler.handle(PAYLOAD); + + expect(detectionRepository.create).toHaveBeenCalledWith( + expect.objectContaining({ + dseq: PAYLOAD.dseq, + verdict: "behavioural", + evidenceExcerpt: expect.stringContaining("service:web probes:3 spanMinutes:") + }) + ); + expect(instrumentation.recordDetection).toHaveBeenCalledWith("behavioural"); + expect(jobQueueService.enqueue).toHaveBeenCalledWith(new EnforceTrialAbuse({ walletId: wallet.id, detectionId: "detection-1" }), { + singletonKey: `enforceTrialAbuse.${wallet.id}` + }); + }); + + it("reads back at least as far as the window the shape has to span, however the probe cadence is set", async () => { + const { handler, detectionRepository } = setup({ + report: createReport({ verdict: "clean" }), + behaviouralEnforcement: true, + probeIntervalMinutes: 10, + maxAttempts: 5, + minWindowMinutes: 120, + deploymentEvidence: createShapeHistory([10, 130, 250]) + }); + + await handler.handle(PAYLOAD); + + expect(detectionRepository.create).toHaveBeenCalledWith(expect.objectContaining({ verdict: "behavioural" })); + }); + + it("reads evidence back across the whole probe schedule when it outlasts the minimum window", async () => { + const { handler, wallet, evidenceRepository } = setup({ + report: createReport({ verdict: "clean" }), + behaviouralEnforcement: true, + probeIntervalMinutes: 60, + maxAttempts: 5, + minWindowMinutes: 120, + now: new Date("2026-09-20T12:00:00.000Z") + }); + + await handler.handle(PAYLOAD); + + expect(evidenceRepository.findRecentForDeployment).toHaveBeenCalledWith({ + walletId: wallet.id, + dseq: PAYLOAD.dseq, + since: new Date("2026-09-20T07:00:00.000Z") + }); + }); + + it("queues the wipe against the detection a confirmed shape already has instead of recording a second one", async () => { + const { handler, wallet, detectionRepository, jobQueueService, probeJobService } = setup({ + report: createReport({ verdict: "clean" }), + behaviouralEnforcement: true, + enforcementMode: "enforce", + existingBehaviouralDetection: true, + deploymentEvidence: createShapeHistory([10, 130, 250]) + }); + + await handler.handle(PAYLOAD); + + expect(detectionRepository.findOneBy).toHaveBeenCalledWith({ walletId: wallet.id, dseq: PAYLOAD.dseq, verdict: "behavioural" }); + expect(detectionRepository.create).not.toHaveBeenCalled(); + expect(probeJobService.scheduleNext).not.toHaveBeenCalled(); + expect(jobQueueService.enqueue).toHaveBeenCalledWith(new EnforceTrialAbuse({ walletId: wallet.id, detectionId: "detection-2" }), { + singletonKey: `enforceTrialAbuse.${wallet.id}` + }); + }); + + it("names every provider the shape was seen on and stops probing once it is recorded", async () => { + const { handler, detectionRepository, probeJobService } = setup({ + report: createReport({ + verdict: "clean", + leases: [ + { + provider: "akash1first", + hostUri: "https://first.example", + gseq: 1, + oseq: 1, + services: ["web"], + shellStatuses: ["completed"], + logStatus: "completed" + }, + { + provider: "akash1second", + hostUri: "https://second.example", + gseq: 2, + oseq: 1, + services: ["web"], + shellStatuses: ["completed"], + logStatus: "completed" + } + ] + }), + behaviouralEnforcement: true, + deploymentEvidence: createShapeHistory([10, 130, 250]) + }); + + await handler.handle(PAYLOAD); + + expect(detectionRepository.create).toHaveBeenCalledWith( + expect.objectContaining({ + provider: "akash1first,akash1second", + evidenceExcerpt: expect.stringMatching(/^\[behavioural\] service:web probes:3 spanMinutes:\d+\n\[soft\/pool-port\] shell:ssh: pool:3333$/) + }) + ); + expect(probeJobService.scheduleNext).not.toHaveBeenCalled(); + }); + + it("keeps probing while the shape has held for fewer probes than required", async () => { + const { handler, detectionRepository, probeJobService } = setup({ + report: createReport({ verdict: "clean" }), + behaviouralEnforcement: true, + deploymentEvidence: createShapeHistory([10, 130]) + }); + + await handler.handle(PAYLOAD); + + expect(detectionRepository.create).not.toHaveBeenCalled(); + expect(probeJobService.scheduleNext).toHaveBeenCalledWith(PAYLOAD); + }); + + it("keeps probing while the shape is younger than the minimum window", async () => { + const { handler, detectionRepository, probeJobService } = setup({ + report: createReport({ verdict: "clean" }), + behaviouralEnforcement: true, + deploymentEvidence: createShapeHistory([1, 3, 5]) + }); + + await handler.handle(PAYLOAD); + + expect(detectionRepository.create).not.toHaveBeenCalled(); + expect(probeJobService.scheduleNext).toHaveBeenCalledWith(PAYLOAD); + }); + + it("records nothing from a repeated shape while behavioural enforcement is off", async () => { + const { handler, detectionRepository, evidenceRepository } = setup({ + report: createReport({ verdict: "clean" }), + deploymentEvidence: createShapeHistory([10, 130, 250]) + }); + + await handler.handle(PAYLOAD); + + expect(evidenceRepository.findRecentForDeployment).not.toHaveBeenCalled(); + expect(detectionRepository.create).not.toHaveBeenCalled(); + }); + it("stops after the last allowed attempt", async () => { const { handler, probeJobService } = setup({ report: createReport({ verdict: "clean" }), maxAttempts: 2 }); @@ -287,16 +444,41 @@ describe(ProbeTrialDeploymentHandler.name, () => { }); }); + function createShapeHistory(minutesAgo: number[]) { + return minutesAgo.map(minutes => + mock({ + service: "web", + provider: "akash1provider", + shellStatus: "completed", + createdAt: new Date(Date.now() - minutes * 60_000), + behaviouralFindings: [ + { signal: "accel_without_artifacts", detail: {} }, + { signal: "network_isolated", detail: {} } + ] + }) + ); + } + function setup(input: { enabled?: boolean; wallet?: ReturnType | null; report?: ProbeReport; maxAttempts?: number; existingDetection?: boolean; + existingBehaviouralDetection?: boolean; + now?: Date; enforcementMode?: "detect" | "enforce"; evidenceRows?: WorkloadProbeEvidenceOutput[]; behaviouralFindings?: RecordedBehaviouralFindings[]; + deploymentEvidence?: WorkloadProbeEvidenceOutput[]; + behaviouralEnforcement?: boolean; + agreementProbes?: number; + minWindowMinutes?: number; + probeIntervalMinutes?: number; }) { + if (input.now) vi.useFakeTimers({ now: input.now, toFake: ["Date"] }); + else vi.useRealTimers(); + const wallet = input.wallet === undefined ? createUserWallet({ isTrialing: true }) : input.wallet; const userWalletRepository = mock(); userWalletRepository.findById.mockResolvedValue(wallet ?? undefined); @@ -305,7 +487,15 @@ describe(ProbeTrialDeploymentHandler.name, () => { const probeJobService = mock(); const detectionRepository = mock(); detectionRepository.create.mockResolvedValue(mock({ id: "detection-1" })); - detectionRepository.findOneBy.mockResolvedValue(input.existingDetection ? mock({ id: "detection-0" }) : undefined); + const hardDetection = mock({ id: "detection-0" }); + const behaviouralDetection = mock({ id: "detection-2" }); + detectionRepository.findOneBy.mockImplementation(async query => { + if (query?.verdict === "hard") return input.existingDetection ? hardDetection : undefined; + if (query?.verdict === "behavioural") return input.existingBehaviouralDetection ? behaviouralDetection : undefined; + return undefined; + }); + const evidenceRepository = mock(); + evidenceRepository.findRecentForDeployment.mockResolvedValue(input.deploymentEvidence ?? []); const probeEvidenceService = mock(); probeEvidenceService.recordEvidence.mockImplementation( async ({ shellEvidence }) => input.evidenceRows ?? shellEvidence.map(entry => mock({ service: entry.service })) @@ -315,7 +505,11 @@ describe(ProbeTrialDeploymentHandler.name, () => { const config = mockConfigService({ WORKLOAD_ABUSE_PROBE_ENABLED: input.enabled ?? true, WORKLOAD_ABUSE_PROBE_MAX_PER_DEPLOYMENT: input.maxAttempts ?? 30, - WORKLOAD_ABUSE_ENFORCEMENT_MODE: input.enforcementMode ?? "detect" + WORKLOAD_ABUSE_ENFORCEMENT_MODE: input.enforcementMode ?? "detect", + WORKLOAD_ABUSE_BEHAVIOURAL_ENFORCEMENT_ENABLED: input.behaviouralEnforcement ?? false, + WORKLOAD_ABUSE_BEHAVIOURAL_AGREEMENT_PROBES: input.agreementProbes ?? 3, + WORKLOAD_ABUSE_BEHAVIOURAL_MIN_WINDOW_MINUTES: input.minWindowMinutes ?? 120, + WORKLOAD_ABUSE_PROBE_INTERVAL_MIN: input.probeIntervalMinutes ?? 60 }); const jobQueueService = mock(); const logger = mock>(); @@ -326,6 +520,7 @@ describe(ProbeTrialDeploymentHandler.name, () => { probeService, probeJobService, detectionRepository, + evidenceRepository, probeEvidenceService, instrumentation, config, @@ -340,6 +535,7 @@ describe(ProbeTrialDeploymentHandler.name, () => { probeService, probeJobService, detectionRepository, + evidenceRepository, probeEvidenceService, instrumentation, jobQueueService, diff --git a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts index 51cea2ad86..e68fbf4064 100644 --- a/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts +++ b/apps/api/src/workload-abuse/services/probe-trial-deployment/probe-trial-deployment.handler.ts @@ -2,9 +2,11 @@ import { inject, singleton } from "tsyringe"; import { isWalletInitialized, UserWalletRepository } from "@src/billing/repositories"; import { type CreateLogger, JOB_NAME, type JobHandler, type JobPayload, type JobPermissions, JobQueueService, LOGGER_FACTORY } from "@src/core"; +import { type BehaviouralAgreement, findBehaviouralAgreement } from "@src/workload-abuse/lib/behavioural-signals/agreement"; import { withoutFileContents } from "@src/workload-abuse/lib/evidence-scanner/evidence-scanner"; import { truncateToUtf8Bytes } from "@src/workload-abuse/lib/utf8-text/utf8-text"; import { WorkloadAbuseDetectionRepository } from "@src/workload-abuse/repositories/workload-abuse-detection/workload-abuse-detection.repository"; +import { WorkloadProbeEvidenceRepository } from "@src/workload-abuse/repositories/workload-probe-evidence/workload-probe-evidence.repository"; import { EnforceTrialAbuse, enforceTrialAbuseKeyFor } from "@src/workload-abuse/services/enforce-trial-abuse/enforce-trial-abuse.handler"; import { ProbeEvidenceService } from "@src/workload-abuse/services/probe-evidence/probe-evidence.service"; import { type ProbeReport, TrialWorkloadProbeService } from "@src/workload-abuse/services/trial-workload-probe/trial-workload-probe.service"; @@ -15,6 +17,11 @@ import { WorkloadAbuseInstrumentationService } from "@src/workload-abuse/service /** Loki splits a line past 16 KiB into unparseable partials, and a clean verdict has nowhere else to keep what the shell saw. */ const MAX_LOGGED_EXCERPT_BYTES = 4_096; +/** The probe that confirms a shape can be a clean one, so the row has to carry the streak that confirmed it rather than what that last probe saw. */ +function toShapeExcerpt(agreement: BehaviouralAgreement, excerpt: string): string { + return [`[behavioural] service:${agreement.service} probes:${agreement.streak} spanMinutes:${agreement.spanMinutes}`, excerpt].join("\n"); +} + /** Re-reads the wallet and the chain on every run, so a probe that waited an hour decides on what is true when it runs, not when it was queued. */ @singleton() export class ProbeTrialDeploymentHandler implements JobHandler { @@ -32,6 +39,7 @@ export class ProbeTrialDeploymentHandler implements JobHandler= this.config.get("WORKLOAD_ABUSE_PROBE_MAX_PER_DEPLOYMENT")) { this.logger.info({ event: "TRIAL_WORKLOAD_PROBE_FINISHED", reason: "MAX_ATTEMPTS", ...context, userId: wallet.userId }); return; @@ -179,6 +189,65 @@ export class ProbeTrialDeploymentHandler implements JobHandler + ): Promise { + if (!this.config.get("WORKLOAD_ABUSE_BEHAVIOURAL_ENFORCEMENT_ENABLED")) return false; + + const rows = await this.evidenceRepository.findRecentForDeployment({ walletId: wallet.id, dseq, since: this.#evidenceLookbackStart() }); + const agreement = findBehaviouralAgreement(rows, { + agreementProbes: this.config.get("WORKLOAD_ABUSE_BEHAVIOURAL_AGREEMENT_PROBES"), + minWindowMinutes: this.config.get("WORKLOAD_ABUSE_BEHAVIOURAL_MIN_WINDOW_MINUTES") + }); + + if (!agreement.agreed) return false; + + const existing = await this.detectionRepository.findOneBy({ walletId: wallet.id, dseq, verdict: "behavioural" }); + + if (existing) { + await this.#enforce(wallet, existing.id, context); + return true; + } + + const detection = await this.detectionRepository.create({ + userId: wallet.userId, + walletId: wallet.id, + dseq, + provider: report.leases.map(lease => lease.provider).join(","), + verdict: "behavioural", + probeStatus: report.probeStatus, + signals: report.signals, + evidenceExcerpt: withoutFileContents(toShapeExcerpt(agreement, report.excerpt)) + }); + this.instrumentation.recordDetection("behavioural"); + + this.logger.warn({ + event: "TRIAL_WORKLOAD_SHAPE_CONFIRMED", + ...context, + userId: wallet.userId, + detectionId: detection.id, + service: agreement.service, + probes: agreement.streak, + spanMinutes: agreement.spanMinutes + }); + + await this.#enforce(wallet, detection.id, context); + + return true; + } + + /** Reading back less far than the window agreement has to span would put agreement out of reach whatever the workload does. */ + #evidenceLookbackStart(): Date { + const probeSpanMinutes = this.config.get("WORKLOAD_ABUSE_PROBE_INTERVAL_MIN") * this.config.get("WORKLOAD_ABUSE_PROBE_MAX_PER_DEPLOYMENT"); + const lookbackMinutes = Math.max(probeSpanMinutes, this.config.get("WORKLOAD_ABUSE_BEHAVIOURAL_MIN_WINDOW_MINUTES")); + + return new Date(Date.now() - lookbackMinutes * 60_000); + } + /** Detect mode records the verdict and stops there, so a rollout can be compared against manual review before anything is wiped. */ async #enforce(wallet: { id: number; userId: string }, detectionId: string, context: Record): Promise { if (this.config.get("WORKLOAD_ABUSE_ENFORCEMENT_MODE") !== "enforce") { diff --git a/apps/api/src/workload-abuse/services/trial-workload-probe-job/trial-workload-probe-job.service.spec.ts b/apps/api/src/workload-abuse/services/trial-workload-probe-job/trial-workload-probe-job.service.spec.ts index a159b0fb2e..e9439f7fee 100644 --- a/apps/api/src/workload-abuse/services/trial-workload-probe-job/trial-workload-probe-job.service.spec.ts +++ b/apps/api/src/workload-abuse/services/trial-workload-probe-job/trial-workload-probe-job.service.spec.ts @@ -194,7 +194,7 @@ describe(TrialWorkloadProbeJobService.name, () => { await service.reconcile({ dryRun: false }); - expect(detectionRepository.findRecentHardTargets).toHaveBeenCalledWith({ since: subHours(LEASE_CREATED_AT, 26) }); + expect(detectionRepository.findRecentDetectedTargets).toHaveBeenCalledWith({ since: subHours(LEASE_CREATED_AT, 26) }); expect(jobQueueService.enqueue).toHaveBeenCalledTimes(1); expect(jobQueueService.enqueue).toHaveBeenCalledWith( expect.objectContaining({ data: expect.objectContaining({ walletId: 2, dseq: "2" }) }), @@ -232,7 +232,7 @@ describe(TrialWorkloadProbeJobService.name, () => { const deploymentSettingRepository = mock(); deploymentSettingRepository.findLiveTrialDeployments.mockResolvedValue(input.live ?? []); const detectionRepository = mock(); - detectionRepository.findRecentHardTargets.mockResolvedValue(input.detected ?? []); + detectionRepository.findRecentDetectedTargets.mockResolvedValue(input.detected ?? []); const config = mockConfigService({ WORKLOAD_ABUSE_PROBE_ENABLED: input.enabled ?? true, WORKLOAD_ABUSE_PROBE_INITIAL_DELAYS_MIN: [5, 20, 60], diff --git a/apps/api/src/workload-abuse/services/trial-workload-probe-job/trial-workload-probe-job.service.ts b/apps/api/src/workload-abuse/services/trial-workload-probe-job/trial-workload-probe-job.service.ts index afdce86528..32ed5fe7a3 100644 --- a/apps/api/src/workload-abuse/services/trial-workload-probe-job/trial-workload-probe-job.service.ts +++ b/apps/api/src/workload-abuse/services/trial-workload-probe-job/trial-workload-probe-job.service.ts @@ -107,7 +107,7 @@ export class TrialWorkloadProbeJobService { if (dryRun) return; const pendingKeys = await this.jobQueueService.findPendingSingletonKeys(ProbeTrialDeployment[JOB_NAME]); - const detectedTargets = await this.detectionRepository.findRecentHardTargets({ since: subHours(new Date(), maxAgeHours) }); + const detectedTargets = await this.detectionRepository.findRecentDetectedTargets({ since: subHours(new Date(), maxAgeHours) }); const detectedKeys = new Set(detectedTargets.map(probeTrialDeploymentKeyFor)); let scheduled = 0; let alreadyScheduled = 0; diff --git a/apps/api/src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service.ts b/apps/api/src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service.ts index 7729d65b25..f43f148720 100644 --- a/apps/api/src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service.ts +++ b/apps/api/src/workload-abuse/services/workload-abuse-instrumentation/workload-abuse-instrumentation.service.ts @@ -8,6 +8,8 @@ export type DomainBlockResult = "blocked" | "raced" | "skipped" | "dry_run" | "f export type EvidenceWriteOperation = "insert" | "findings" | "purge"; +export type DetectionVerdict = WorkloadVerdict | "behavioural"; + @singleton() export class WorkloadAbuseInstrumentationService { private readonly meter: Meter; @@ -48,7 +50,7 @@ export class WorkloadAbuseInstrumentationService { this.probes.add(1, { verdict: input.verdict, probe_status: input.probeStatus }); } - recordDetection(verdict: WorkloadVerdict): void { + recordDetection(verdict: DetectionVerdict): void { this.detections.add(1, { verdict }); }