diff --git a/backend/.env.example b/backend/.env.example index d1af1d81..a6d7b635 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -94,3 +94,24 @@ PAYMENT_WEBHOOK_SECRET=change-me-in-every-environment # a timeout falls back to the webhook/real-time channel, it never marks a # payment CONFIRMED on its own. PAYMENT_VERIFY_TIMEOUT_MS=3000 + +# Payments reconciliation engine (issue #1572) +# A payment isn't eligible for its first reconciliation pass until it's +# been AWAITING_CONFIRMATION for at least this long — gives the webhook a +# fair chance first. +PAYMENT_RECONCILE_DUE_AFTER_MINUTES=5 +# Exponential backoff (in minutes) between reconciliation polls for a +# still-pending payment: base * 2^attempts, capped at the max below — a +# bank-side hold that stays "still processing" gets polled less and less +# often instead of every cron tick. +PAYMENT_RECONCILE_BACKOFF_BASE_MINUTES=5 +PAYMENT_RECONCILE_BACKOFF_MAX_MINUTES=60 +# Upper bound on how many AWAITING_CONFIRMATION payments one reconciliation +# pass will consider. +PAYMENT_RECONCILE_MAX_BATCH=500 +# A payment unresolved this long escalates to MANUAL_REVIEW — but only on a +# pass where the provider was actually reachable; a provider outage never +# by itself causes escalation (see ReconciliationService). +PAYMENT_MANUAL_REVIEW_AFTER_HOURS=24 +# Logs a WARN-level alert when the manual-review queue depth exceeds this. +PAYMENT_MANUAL_REVIEW_ALERT_THRESHOLD=20 diff --git a/backend/src/app.module.ts b/backend/src/app.module.ts index 93c65283..8480b6be 100644 --- a/backend/src/app.module.ts +++ b/backend/src/app.module.ts @@ -1,6 +1,7 @@ import { Module } from '@nestjs/common'; import { ConfigModule, ConfigService } from '@nestjs/config'; import { TypeOrmModule } from '@nestjs/typeorm'; +import { ScheduleModule } from '@nestjs/schedule'; import { AppController } from './app.controller'; import { AppService } from './app.service'; import { AuthModule } from './auth/auth.module'; @@ -9,6 +10,8 @@ import { PaymentsModule } from './payments/payments.module'; @Module({ imports: [ ConfigModule.forRoot({ isGlobal: true }), + // Powers ReconciliationService's @Cron job (issue #1572). + ScheduleModule.forRoot(), TypeOrmModule.forRootAsync({ inject: [ConfigService], useFactory: (config: ConfigService) => ({ diff --git a/backend/src/database/migrations/1755870474000-AddPaymentReconciliationFields.ts b/backend/src/database/migrations/1755870474000-AddPaymentReconciliationFields.ts new file mode 100644 index 00000000..dd558a39 --- /dev/null +++ b/backend/src/database/migrations/1755870474000-AddPaymentReconciliationFields.ts @@ -0,0 +1,70 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddPaymentReconciliationFields1755870474000 implements MigrationInterface { + name = 'AddPaymentReconciliationFields1755870474000'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + CREATE TYPE "payments_failure_reason_enum" AS ENUM ( + 'DECLINED', 'EXPIRED', 'PROVIDER_ERROR', 'ABANDONED' + ) + `); + + // New failure-taxonomy/escalation statuses (issue #1572) on top of + // #1570's original enum. + await queryRunner.query(` + ALTER TYPE "payments_status_enum" ADD VALUE IF NOT EXISTS 'MANUAL_REVIEW' + `); + await queryRunner.query(` + ALTER TYPE "payments_status_enum" ADD VALUE IF NOT EXISTS 'DISPUTED' + `); + await queryRunner.query(` + ALTER TYPE "payments_status_enum" ADD VALUE IF NOT EXISTS 'VOIDED' + `); + + await queryRunner.query(` + ALTER TABLE "payments" + ADD COLUMN "failure_reason" "payments_failure_reason_enum", + ADD COLUMN "reconciliation_attempts" integer NOT NULL DEFAULT 0, + ADD COLUMN "provider_error_streak" integer NOT NULL DEFAULT 0, + ADD COLUMN "last_reconciled_at" timestamptz, + ADD COLUMN "manual_review_reason" text + `); + + await queryRunner.query(` + CREATE TABLE "payment_refunds" ( + "id" uuid NOT NULL DEFAULT gen_random_uuid(), + "payment_id" uuid NOT NULL, + "amount" bigint NOT NULL, + "reason" text NOT NULL, + "actor_id" uuid, + "created_at" TIMESTAMP NOT NULL DEFAULT now(), + CONSTRAINT "pk_payment_refunds" PRIMARY KEY ("id"), + CONSTRAINT "fk_payment_refunds_payment" FOREIGN KEY ("payment_id") + REFERENCES "payments"("id") ON DELETE NO ACTION ON UPDATE NO ACTION + ) + `); + + await queryRunner.query(` + CREATE INDEX "idx_payment_refunds_payment_id" ON "payment_refunds" ("payment_id") + `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX "idx_payment_refunds_payment_id"`); + await queryRunner.query(`DROP TABLE "payment_refunds"`); + await queryRunner.query(` + ALTER TABLE "payments" + DROP COLUMN "manual_review_reason", + DROP COLUMN "last_reconciled_at", + DROP COLUMN "provider_error_streak", + DROP COLUMN "reconciliation_attempts", + DROP COLUMN "failure_reason" + `); + // Postgres has no DROP VALUE for enums — reverting MANUAL_REVIEW / + // DISPUTED / VOIDED would require recreating payments_status_enum, + // which isn't safe to do automatically without knowing whether any row + // already uses them. + await queryRunner.query(`DROP TYPE "payments_failure_reason_enum"`); + } +} diff --git a/backend/src/payments/README.md b/backend/src/payments/README.md new file mode 100644 index 00000000..33f457f9 --- /dev/null +++ b/backend/src/payments/README.md @@ -0,0 +1,100 @@ +# Payments module + +Payment domain model, initiation, confirmation, reconciliation, and refunds +for the booking platform. This module is built incrementally across a +payment track of issues (see `payment-state-machine.ts` for the canonical +list of legal state transitions — that file, not this README, is the source +of truth if the two ever disagree). + +## Status lifecycle + +`INITIATED → AWAITING_CONFIRMATION → CONFIRMED → (PARTIALLY_REFUNDED →) REFUNDED` + +Off that happy path: + +- `INITIATED → FAILED | EXPIRED` +- `AWAITING_CONFIRMATION → FAILED | EXPIRED | MANUAL_REVIEW` +- `CONFIRMED → DISPUTED` +- `DISPUTED → REFUNDED` +- `MANUAL_REVIEW → CONFIRMED | FAILED | VOIDED` + +## Failure taxonomy (issue #1572) + +`Payment#failureReason` records **why** a payment ended up `FAILED` or +`EXPIRED` — first-class enum data, not free text (`payment-failure-reason.enum.ts`): + +| Reason | Meaning | +| ---------------- | ------------------------------------------------------------------------ | +| `DECLINED` | The provider explicitly rejected the charge. | +| `EXPIRED` | No confirmation (webhook or reconciliation) arrived within the TTL. | +| `PROVIDER_ERROR` | Talking to the provider itself failed (5xx/timeout) — not a verdict. | +| `ABANDONED` | The payment never progressed past `INITIATED` before expiring. | + +## Reconciliation engine + +`ReconciliationService` runs on a schedule (`@Cron`, every 5 minutes by +default) and does two things: + +1. **Expiry sweep** — `INITIATED`/`AWAITING_CONFIRMATION` payments past + `Payment#expiresAt` become `EXPIRED` (`ABANDONED` vs `EXPIRED` reason + depending on which state they expired from). +2. **Reconciliation** — every `AWAITING_CONFIRMATION` payment old enough + (`PAYMENT_RECONCILE_DUE_AFTER_MINUTES`) and due for its next poll (an + exponential backoff schedule per payment — see + `PAYMENT_RECONCILE_BACKOFF_*`) is re-verified directly against the + provider, reusing the same `verifyByReference` call and the same + idempotent `PaymentConfirmationService.apply` path the webhook and + verify-on-return flows use. A payment resolved this way can never be + double-applied, and re-running the job is always safe. + +A payment still unresolved after `PAYMENT_MANUAL_REVIEW_AFTER_HOURS` +escalates to `MANUAL_REVIEW` — **but only on a reconciliation pass where the +provider was actually reachable**. A provider-side outage (every verify +call in a batch throwing/timing out) never by itself escalates anything — +`Payment#providerErrorStreak` tracks consecutive unreachable attempts +separately from the age-based threshold, specifically so one bad run can't +mass-flag every in-flight payment. + +### Admin recovery actions (`PaymentsAdminController`, `ADMIN` role only) + +- `GET /payments/admin/manual-review` — the review queue, oldest first. +- `GET /payments/admin/metrics` — manual-review queue depth + alert status. +- `POST /payments/admin/:id/force-reconcile` — reconcile one payment now, + bypassing the backoff schedule. +- `POST /payments/admin/:id/resolve-manually` — `{ resolution, reason }`, + reason required and audited via `Payment#manualReviewReason`. +- `POST /payments/admin/:id/void` — `{ reason }`, closes a payment out + without resolving it `CONFIRMED`/`FAILED`. +- `POST /payments/admin/:id/refunds` — `{ amount, reason }`, see below. + +## Refunds (partial-outcome support) + +A `Payment`'s refunded amount is always `SUM(payment_refunds.amount)` for +that payment — never a single boolean — so multiple partial refunds can +accumulate against one payment (`RefundsService`, `refund.entity.ts`). + +`RefundsService.requestRefund` locks the payment row for the duration of +the check-and-insert (`pessimistic_write`), so two refund requests that +would together exceed the captured amount can never both succeed: the +loser gets a `409 Conflict`, not a corrupted ledger. The payment's status +moves to `PARTIALLY_REFUNDED` or `REFUNDED` depending on whether the +refunded total has reached the captured amount. Provider-side execution of +the refund happens *after* the ledger commits (best-effort, retried via +`utils/retry-with-backoff.ts`) — the ledger, not the provider call, is the +source of truth for "was this refund accepted." + +## Retry/backoff utility + +`utils/retry-with-backoff.ts` is a small, generic, independently-tested +exponential-backoff-with-jitter helper for our own outbound provider calls +— capped attempts, and an optional `isRetryable` predicate so a terminal +error (4xx) fails fast instead of burning the full attempt budget. Used by +`RefundsService`'s post-commit provider refund call. + +## Metrics & alerting + +`ReconciliationService.getMetrics()` reports the manual-review queue depth +and whether it exceeds `PAYMENT_MANUAL_REVIEW_ALERT_THRESHOLD`; the +scheduled job logs a `WARN`-level alert when it does. This is a +log-based signal by design — this module doesn't assume any particular +metrics/observability backend is wired up yet. diff --git a/backend/src/payments/adapters/sandbox-rail.adapter.ts b/backend/src/payments/adapters/sandbox-rail.adapter.ts index 2e3f1038..d0b58e12 100644 --- a/backend/src/payments/adapters/sandbox-rail.adapter.ts +++ b/backend/src/payments/adapters/sandbox-rail.adapter.ts @@ -88,4 +88,10 @@ export class SandboxRailAdapter implements PaymentRailAdapter { ): value is PaymentVerificationOutcome { return value === 'confirmed' || value === 'failed' || value === 'pending'; } + + // No real provider wired up yet — a refund here always "succeeds" + // (matches initiate()/verifyByReference()'s sandbox placeholder nature). + async refund(_providerReference: string, _amount: number): Promise { + return; + } } diff --git a/backend/src/payments/dto/create-refund.dto.ts b/backend/src/payments/dto/create-refund.dto.ts new file mode 100644 index 00000000..c1e95d9c --- /dev/null +++ b/backend/src/payments/dto/create-refund.dto.ts @@ -0,0 +1,16 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsInt, IsNotEmpty, IsPositive, IsString } from 'class-validator'; + +export class CreateRefundDto { + @ApiProperty({ + description: 'Minor units (e.g. cents) — same convention as Payment#amount', + }) + @IsInt() + @IsPositive() + amount: number; + + @ApiProperty() + @IsString() + @IsNotEmpty() + reason: string; +} diff --git a/backend/src/payments/dto/payment-response.dto.ts b/backend/src/payments/dto/payment-response.dto.ts index 9920bb94..2e33aa5a 100644 --- a/backend/src/payments/dto/payment-response.dto.ts +++ b/backend/src/payments/dto/payment-response.dto.ts @@ -1,6 +1,7 @@ import { ApiProperty } from '@nestjs/swagger'; import { PaymentRail } from '../enums/payment-rail.enum'; import { PaymentStatus } from '../enums/payment-status.enum'; +import { PaymentFailureReason } from '../enums/payment-failure-reason.enum'; import { Payment } from '../entities/payment.entity'; export class PaymentResponseDto { @@ -18,6 +19,10 @@ export class PaymentResponseDto { @ApiProperty({ nullable: true }) provider: string | null; @ApiProperty({ nullable: true }) providerReference: string | null; @ApiProperty({ nullable: true }) expiresAt: Date | null; + @ApiProperty({ enum: PaymentFailureReason, nullable: true }) + failureReason: PaymentFailureReason | null; + @ApiProperty() reconciliationAttempts: number; + @ApiProperty({ nullable: true }) manualReviewReason: string | null; @ApiProperty() createdAt: Date; @ApiProperty() updatedAt: Date; @@ -33,6 +38,9 @@ export class PaymentResponseDto { dto.provider = payment.provider; dto.providerReference = payment.providerReference; dto.expiresAt = payment.expiresAt; + dto.failureReason = payment.failureReason; + dto.reconciliationAttempts = payment.reconciliationAttempts; + dto.manualReviewReason = payment.manualReviewReason; dto.createdAt = payment.createdAt; dto.updatedAt = payment.updatedAt; return dto; diff --git a/backend/src/payments/dto/refund-response.dto.ts b/backend/src/payments/dto/refund-response.dto.ts new file mode 100644 index 00000000..3428047b --- /dev/null +++ b/backend/src/payments/dto/refund-response.dto.ts @@ -0,0 +1,19 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { PaymentResponseDto } from './payment-response.dto'; + +export class RefundResponseDto { + @ApiProperty() id: string; + @ApiProperty() paymentId: string; + @ApiProperty() amount: number; + @ApiProperty() reason: string; + @ApiProperty({ nullable: true }) actorId: string | null; + @ApiProperty() createdAt: Date; +} + +export class RequestRefundResponseDto { + @ApiProperty({ type: PaymentResponseDto }) + payment: PaymentResponseDto; + + @ApiProperty({ type: RefundResponseDto }) + refund: RefundResponseDto; +} diff --git a/backend/src/payments/dto/resolve-payment-manually.dto.ts b/backend/src/payments/dto/resolve-payment-manually.dto.ts new file mode 100644 index 00000000..924805f8 --- /dev/null +++ b/backend/src/payments/dto/resolve-payment-manually.dto.ts @@ -0,0 +1,14 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsIn, IsNotEmpty, IsString } from 'class-validator'; +import { PaymentStatus } from '../enums/payment-status.enum'; + +export class ResolvePaymentManuallyDto { + @ApiProperty({ enum: [PaymentStatus.CONFIRMED, PaymentStatus.FAILED] }) + @IsIn([PaymentStatus.CONFIRMED, PaymentStatus.FAILED]) + resolution: PaymentStatus.CONFIRMED | PaymentStatus.FAILED; + + @ApiProperty({ description: 'Required — audited alongside the resolution' }) + @IsString() + @IsNotEmpty() + reason: string; +} diff --git a/backend/src/payments/dto/void-payment.dto.ts b/backend/src/payments/dto/void-payment.dto.ts new file mode 100644 index 00000000..ffe2b6be --- /dev/null +++ b/backend/src/payments/dto/void-payment.dto.ts @@ -0,0 +1,9 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsNotEmpty, IsString } from 'class-validator'; + +export class VoidPaymentDto { + @ApiProperty({ description: 'Required — audited alongside the void action' }) + @IsString() + @IsNotEmpty() + reason: string; +} diff --git a/backend/src/payments/entities/payment.entity.ts b/backend/src/payments/entities/payment.entity.ts index 3fd7d355..051c1cdd 100644 --- a/backend/src/payments/entities/payment.entity.ts +++ b/backend/src/payments/entities/payment.entity.ts @@ -8,6 +8,7 @@ import { } from 'typeorm'; import { PaymentRail } from '../enums/payment-rail.enum'; import { PaymentStatus } from '../enums/payment-status.enum'; +import { PaymentFailureReason } from '../enums/payment-failure-reason.enum'; @Entity('payments') @Index(['userId', 'idempotencyKey'], { unique: true }) @@ -63,6 +64,38 @@ export class Payment { @Column({ type: 'timestamptz', name: 'expires_at', nullable: true }) expiresAt: Date | null; + // ── Reconciliation (issue #1572) ──────────────────────────────────────── + + /** Set alongside a FAILED or EXPIRED status — WHY, not just WHAT. */ + @Column({ + type: 'enum', + enum: PaymentFailureReason, + name: 'failure_reason', + nullable: true, + }) + failureReason: PaymentFailureReason | null; + + /** Every reconciliation pass that touched this payment, resolved or not — drives backoff scheduling. */ + @Column({ type: 'int', name: 'reconciliation_attempts', default: 0 }) + reconciliationAttempts: number; + + /** + * Consecutive reconciliation passes where the provider itself was + * unreachable (not "provider said pending"). Reset to 0 the moment a + * verify call successfully reaches the provider again. Used to withhold + * MANUAL_REVIEW escalation during a provider outage — see + * ReconciliationService. + */ + @Column({ type: 'int', name: 'provider_error_streak', default: 0 }) + providerErrorStreak: number; + + @Column({ type: 'timestamptz', name: 'last_reconciled_at', nullable: true }) + lastReconciledAt: Date | null; + + /** Reason text for a MANUAL_REVIEW escalation or an admin resolve/void action — audited. */ + @Column({ type: 'text', name: 'manual_review_reason', nullable: true }) + manualReviewReason: string | null; + @CreateDateColumn({ name: 'created_at' }) createdAt: Date; diff --git a/backend/src/payments/entities/refund.entity.ts b/backend/src/payments/entities/refund.entity.ts new file mode 100644 index 00000000..f3a3719b --- /dev/null +++ b/backend/src/payments/entities/refund.entity.ts @@ -0,0 +1,43 @@ +import { + Column, + CreateDateColumn, + Entity, + Index, + PrimaryGeneratedColumn, +} from 'typeorm'; + +/** + * Append-only refund ledger (issue #1572) — a Payment can have many Refund + * rows (partial refunds), so "amount refunded" is always + * SUM(refunds.amount) for a payment, never a single boolean on Payment + * itself. Presence of a row here means the refund was accepted into the + * ledger atomically (see RefundsService); there is no separate pending + * state because provider-side execution is best-effort/logged after the + * ledger commit, not modeled as its own lifecycle in this MVP. + */ +@Entity('payment_refunds') +export class Refund { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Index() + @Column({ type: 'uuid', name: 'payment_id' }) + paymentId: string; + + /** Minor units — same convention as Payment#amount. */ + @Column({ + type: 'bigint', + transformer: { to: (v: number) => v, from: (v: string) => parseInt(v, 10) }, + }) + amount: number; + + @Column({ type: 'text' }) + reason: string; + + /** Null for a system/automated refund; the admin's user id otherwise. */ + @Column({ type: 'uuid', name: 'actor_id', nullable: true }) + actorId: string | null; + + @CreateDateColumn({ name: 'created_at' }) + createdAt: Date; +} diff --git a/backend/src/payments/enums/confirmation-source.enum.ts b/backend/src/payments/enums/confirmation-source.enum.ts index d78c4cb4..406e3f35 100644 --- a/backend/src/payments/enums/confirmation-source.enum.ts +++ b/backend/src/payments/enums/confirmation-source.enum.ts @@ -1,4 +1,7 @@ export enum ConfirmationSource { WEBHOOK = 'WEBHOOK', VERIFY_RETURN = 'VERIFY_RETURN', + // Scheduled reconciliation resolved this independent of any webhook + // (issue #1572) — see ReconciliationService. + RECONCILIATION = 'RECONCILIATION', } diff --git a/backend/src/payments/enums/payment-failure-reason.enum.ts b/backend/src/payments/enums/payment-failure-reason.enum.ts new file mode 100644 index 00000000..bf0c4670 --- /dev/null +++ b/backend/src/payments/enums/payment-failure-reason.enum.ts @@ -0,0 +1,16 @@ +/** + * Failure taxonomy (issue #1572) as first-class data rather than free text + * — stored on Payment#failureReason alongside a terminal status (FAILED or + * EXPIRED). Distinct from PaymentStatus: the status is WHAT happened to the + * payment lifecycle, this is WHY. + */ +export enum PaymentFailureReason { + /** Provider explicitly rejected the charge (card declined, insufficient funds, etc). */ + DECLINED = 'DECLINED', + /** No confirmation arrived (webhook or reconciliation) within the payment's TTL. */ + EXPIRED = 'EXPIRED', + /** Talking to the provider itself failed (5xx / timeout) — not a provider verdict. */ + PROVIDER_ERROR = 'PROVIDER_ERROR', + /** The payment never progressed past INITIATED before expiring — the user never returned. */ + ABANDONED = 'ABANDONED', +} diff --git a/backend/src/payments/enums/payment-status.enum.ts b/backend/src/payments/enums/payment-status.enum.ts index a4294425..00b5c517 100644 --- a/backend/src/payments/enums/payment-status.enum.ts +++ b/backend/src/payments/enums/payment-status.enum.ts @@ -6,6 +6,15 @@ export enum PaymentStatus { EXPIRED = 'EXPIRED', REFUNDED = 'REFUNDED', PARTIALLY_REFUNDED = 'PARTIALLY_REFUNDED', + // Escalation tier (issue #1572): a payment reconciliation could not + // resolve automatically after the long threshold — never silently + // retried forever, surfaced to admins with a reason instead. + MANUAL_REVIEW = 'MANUAL_REVIEW', + // Chargeback / dispute flagged against an already-CONFIRMED payment. + DISPUTED = 'DISPUTED', + // Admin recovery action: a MANUAL_REVIEW payment deliberately closed out + // without resolving to CONFIRMED or FAILED (e.g. abandoned booking). + VOIDED = 'VOIDED', } /** diff --git a/backend/src/payments/interfaces/payment-rail-adapter.interface.ts b/backend/src/payments/interfaces/payment-rail-adapter.interface.ts index 2ec1e8fd..c910ea63 100644 --- a/backend/src/payments/interfaces/payment-rail-adapter.interface.ts +++ b/backend/src/payments/interfaces/payment-rail-adapter.interface.ts @@ -47,4 +47,12 @@ export interface PaymentRailAdapter { verifyByReference( providerReference: string, ): Promise; + + /** + * Executes a (partial) refund against the provider for an already-CONFIRMED + * payment (issue #1572). Called only AFTER the refund ledger entry is + * already committed — this is the best-effort provider-side execution, + * not the source of truth for "was this refund accepted" (the ledger is). + */ + refund(providerReference: string, amount: number): Promise; } diff --git a/backend/src/payments/payment-state-machine.spec.ts b/backend/src/payments/payment-state-machine.spec.ts index 9393fac6..f605c838 100644 --- a/backend/src/payments/payment-state-machine.spec.ts +++ b/backend/src/payments/payment-state-machine.spec.ts @@ -12,8 +12,15 @@ describe('payment state machine', () => { [PaymentStatus.AWAITING_CONFIRMATION, PaymentStatus.CONFIRMED], [PaymentStatus.AWAITING_CONFIRMATION, PaymentStatus.FAILED], [PaymentStatus.AWAITING_CONFIRMATION, PaymentStatus.EXPIRED], + [PaymentStatus.AWAITING_CONFIRMATION, PaymentStatus.MANUAL_REVIEW], [PaymentStatus.CONFIRMED, PaymentStatus.REFUNDED], [PaymentStatus.CONFIRMED, PaymentStatus.PARTIALLY_REFUNDED], + [PaymentStatus.CONFIRMED, PaymentStatus.DISPUTED], + [PaymentStatus.PARTIALLY_REFUNDED, PaymentStatus.REFUNDED], + [PaymentStatus.MANUAL_REVIEW, PaymentStatus.CONFIRMED], + [PaymentStatus.MANUAL_REVIEW, PaymentStatus.FAILED], + [PaymentStatus.MANUAL_REVIEW, PaymentStatus.VOIDED], + [PaymentStatus.DISPUTED, PaymentStatus.REFUNDED], ]; it.each(VALID_TRANSITIONS)('allows %s -> %s', (from, to) => { @@ -43,7 +50,7 @@ describe('payment state machine', () => { PaymentStatus.FAILED, PaymentStatus.EXPIRED, PaymentStatus.REFUNDED, - PaymentStatus.PARTIALLY_REFUNDED, + PaymentStatus.VOIDED, ]; for (const status of terminal) { for (const to of ALL_STATUSES) { diff --git a/backend/src/payments/payment-state-machine.ts b/backend/src/payments/payment-state-machine.ts index 1351be30..433a8e4a 100644 --- a/backend/src/payments/payment-state-machine.ts +++ b/backend/src/payments/payment-state-machine.ts @@ -16,15 +16,30 @@ const ALLOWED_TRANSITIONS: Record = { PaymentStatus.CONFIRMED, PaymentStatus.FAILED, PaymentStatus.EXPIRED, + // Escalation tier (issue #1572) — reconciliation couldn't resolve this + // automatically after the long threshold. + PaymentStatus.MANUAL_REVIEW, ], [PaymentStatus.CONFIRMED]: [ PaymentStatus.REFUNDED, PaymentStatus.PARTIALLY_REFUNDED, + PaymentStatus.DISPUTED, ], [PaymentStatus.FAILED]: [], [PaymentStatus.EXPIRED]: [], [PaymentStatus.REFUNDED]: [], - [PaymentStatus.PARTIALLY_REFUNDED]: [], + // A later partial refund can complete into a full refund; the status + // itself doesn't change on every additional partial (the refund ledger + // tracks that), only when the refunded total reaches the captured amount. + [PaymentStatus.PARTIALLY_REFUNDED]: [PaymentStatus.REFUNDED], + // Admin recovery actions (issue #1572) — mark-resolved-manually or void. + [PaymentStatus.MANUAL_REVIEW]: [ + PaymentStatus.CONFIRMED, + PaymentStatus.FAILED, + PaymentStatus.VOIDED, + ], + [PaymentStatus.DISPUTED]: [PaymentStatus.REFUNDED], + [PaymentStatus.VOIDED]: [], }; export function canTransition(from: PaymentStatus, to: PaymentStatus): boolean { diff --git a/backend/src/payments/payments-admin.controller.ts b/backend/src/payments/payments-admin.controller.ts new file mode 100644 index 00000000..7263cce8 --- /dev/null +++ b/backend/src/payments/payments-admin.controller.ts @@ -0,0 +1,136 @@ +import { + Body, + Controller, + Get, + Param, + ParseUUIDPipe, + Post, + UseGuards, +} from '@nestjs/common'; +import { + ApiBearerAuth, + ApiOperation, + ApiResponse, + ApiTags, +} from '@nestjs/swagger'; +import { JwtAuthGuard } from '../auth/guards/jwt-auth.guard'; +import { RolesGuard } from '../auth/guards/roles.guard'; +import { Roles } from '../auth/decorators/roles.decorator'; +import { CurrentUser } from '../auth/decorators/current-user.decorator'; +import { RequestUser } from '../auth/interfaces/authenticated-request.interface'; +import { UserRole } from '../auth/enums/user-role.enum'; +import { ReconciliationService } from './reconciliation.service'; +import { RefundsService } from './refunds.service'; +import { PaymentResponseDto } from './dto/payment-response.dto'; +import { ResolvePaymentManuallyDto } from './dto/resolve-payment-manually.dto'; +import { VoidPaymentDto } from './dto/void-payment.dto'; +import { CreateRefundDto } from './dto/create-refund.dto'; +import { + RefundResponseDto, + RequestRefundResponseDto, +} from './dto/refund-response.dto'; + +/** + * Admin-only payment recovery/refund actions (issue #1572). Every mutating + * action here requires a reason, which is logged (Payment#manualReviewReason + * for the recovery actions) — bounded manual intervention, never a silent + * bypass of the state machine. + */ +@ApiTags('payments-admin') +@ApiBearerAuth() +@UseGuards(JwtAuthGuard, RolesGuard) +@Roles(UserRole.ADMIN) +@Controller('payments/admin') +export class PaymentsAdminController { + constructor( + private readonly reconciliationService: ReconciliationService, + private readonly refundsService: RefundsService, + ) {} + + @Get('manual-review') + @ApiOperation({ + summary: 'List payments awaiting manual review, oldest first', + }) + @ApiResponse({ status: 200, type: [PaymentResponseDto] }) + async listManualReview(): Promise { + const payments = await this.reconciliationService.listManualReview(); + return payments.map((payment) => PaymentResponseDto.fromEntity(payment)); + } + + @Get('metrics') + @ApiOperation({ + summary: + 'Reconciliation metrics — manual-review queue depth and alert status', + }) + getMetrics() { + return this.reconciliationService.getMetrics(); + } + + @Post(':id/force-reconcile') + @ApiOperation({ + summary: + 'Immediately re-verify one payment against the provider, bypassing the due-schedule', + }) + @ApiResponse({ status: 200, type: PaymentResponseDto }) + async forceReconcile( + @Param('id', ParseUUIDPipe) id: string, + ): Promise { + const payment = await this.reconciliationService.forceReconcileNow(id); + return PaymentResponseDto.fromEntity(payment); + } + + @Post(':id/resolve-manually') + @ApiOperation({ + summary: + 'Resolve a MANUAL_REVIEW payment by hand (reason required, audited)', + }) + @ApiResponse({ status: 200, type: PaymentResponseDto }) + async resolveManually( + @Param('id', ParseUUIDPipe) id: string, + @Body() dto: ResolvePaymentManuallyDto, + ): Promise { + const payment = await this.reconciliationService.resolveManually( + id, + dto.resolution, + dto.reason, + ); + return PaymentResponseDto.fromEntity(payment); + } + + @Post(':id/void') + @ApiOperation({ + summary: + 'Void a MANUAL_REVIEW payment without resolving it (reason required, audited)', + }) + @ApiResponse({ status: 200, type: PaymentResponseDto }) + async void( + @Param('id', ParseUUIDPipe) id: string, + @Body() dto: VoidPaymentDto, + ): Promise { + const payment = await this.reconciliationService.void(id, dto.reason); + return PaymentResponseDto.fromEntity(payment); + } + + @Post(':id/refunds') + @ApiOperation({ + summary: + 'Issue a (partial) refund against a CONFIRMED/PARTIALLY_REFUNDED payment', + }) + @ApiResponse({ status: 201, type: RequestRefundResponseDto }) + async requestRefund( + @Param('id', ParseUUIDPipe) id: string, + @Body() dto: CreateRefundDto, + @CurrentUser() currentUser: RequestUser, + ): Promise { + const { payment, refund } = await this.refundsService.requestRefund( + id, + dto.amount, + dto.reason, + currentUser.id, + ); + return { + payment: PaymentResponseDto.fromEntity(payment), + refund: refund as unknown as RefundResponseDto, + }; + } +} diff --git a/backend/src/payments/payments.module.ts b/backend/src/payments/payments.module.ts index d94eb979..e3dd1784 100644 --- a/backend/src/payments/payments.module.ts +++ b/backend/src/payments/payments.module.ts @@ -2,22 +2,32 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; import { Payment } from './entities/payment.entity'; import { ConfirmationEvent } from './entities/confirmation-event.entity'; +import { Refund } from './entities/refund.entity'; import { PaymentsService } from './payments.service'; import { PaymentConfirmationService } from './payment-confirmation.service'; +import { ReconciliationService } from './reconciliation.service'; +import { RefundsService } from './refunds.service'; import { PaymentsGateway } from './payments.gateway'; import { PaymentsController } from './payments.controller'; import { PaymentWebhookController } from './payment-webhook.controller'; +import { PaymentsAdminController } from './payments-admin.controller'; import { SandboxRailAdapter } from './adapters/sandbox-rail.adapter'; @Module({ - imports: [TypeOrmModule.forFeature([Payment, ConfirmationEvent])], - controllers: [PaymentsController, PaymentWebhookController], + imports: [TypeOrmModule.forFeature([Payment, ConfirmationEvent, Refund])], + controllers: [ + PaymentsController, + PaymentWebhookController, + PaymentsAdminController, + ], providers: [ PaymentsService, PaymentConfirmationService, + ReconciliationService, + RefundsService, PaymentsGateway, SandboxRailAdapter, ], - exports: [PaymentsService, PaymentConfirmationService], + exports: [PaymentsService, PaymentConfirmationService, ReconciliationService], }) export class PaymentsModule {} diff --git a/backend/src/payments/reconciliation.service.spec.ts b/backend/src/payments/reconciliation.service.spec.ts new file mode 100644 index 00000000..a378d06b --- /dev/null +++ b/backend/src/payments/reconciliation.service.spec.ts @@ -0,0 +1,484 @@ +import { + NotFoundException, + UnprocessableEntityException, +} from '@nestjs/common'; +import { FindOperator } from 'typeorm'; +import { ReconciliationService } from './reconciliation.service'; +import { Payment } from './entities/payment.entity'; +import { PaymentRail } from './enums/payment-rail.enum'; +import { PaymentStatus } from './enums/payment-status.enum'; +import { PaymentFailureReason } from './enums/payment-failure-reason.enum'; +import { ConfirmationSource } from './enums/confirmation-source.enum'; + +function makePayment(overrides: Partial = {}): Payment { + const now = new Date(); + return { + id: 'payment-1', + bookingId: 'booking-1', + userId: 'user-1', + amount: 1000, + currency: 'USD', + rail: PaymentRail.FIAT, + provider: null, + providerReference: 'sandbox_ref_1', + status: PaymentStatus.AWAITING_CONFIRMATION, + idempotencyKey: 'key-1', + metadata: null, + expiresAt: null, + failureReason: null, + reconciliationAttempts: 0, + providerErrorStreak: 0, + lastReconciledAt: null, + manualReviewReason: null, + createdAt: now, + updatedAt: now, + ...overrides, + } as Payment; +} + +function matchesCondition( + payment: Payment, + key: string, + condition: unknown, +): boolean { + const value = (payment as unknown as Record)[key]; + if (condition instanceof FindOperator) { + if (condition.type === 'lessThan') { + const target = condition.value as Date; + return value instanceof Date && value.getTime() < target.getTime(); + } + throw new Error(`Unsupported FindOperator in test fake: ${condition.type}`); + } + return value === condition; +} + +function matchesWhere( + payment: Payment, + where: Record, +): boolean { + return Object.entries(where).every(([key, condition]) => + matchesCondition(payment, key, condition), + ); +} + +/** Minimal in-memory Repository fake — enough to exercise real TypeORM `where` semantics. */ +function makePaymentRepository(seed: Payment[]) { + const rows = [...seed]; + + return { + find: jest.fn( + async (options?: { + where?: Record | Record[]; + take?: number; + }) => { + let result = rows; + if (options?.where) { + const clauses = Array.isArray(options.where) + ? options.where + : [options.where]; + result = rows.filter((p) => + clauses.some((clause) => matchesWhere(p, clause)), + ); + } + if (options?.take) { + result = result.slice(0, options.take); + } + return result.map((p) => ({ ...p })); + }, + ), + findOne: jest.fn(async (options: { where: Record }) => { + const found = rows.find((p) => matchesWhere(p, options.where)); + return found ? { ...found } : null; + }), + save: jest.fn(async (entity: Payment) => { + const index = rows.findIndex((p) => p.id === entity.id); + if (index >= 0) { + rows[index] = { ...entity }; + } + return { ...entity }; + }), + update: jest.fn(async (id: string, partial: Partial) => { + const index = rows.findIndex((p) => p.id === id); + if (index >= 0) { + rows[index] = { ...rows[index], ...partial }; + } + return { affected: index >= 0 ? 1 : 0 }; + }), + count: jest.fn(async (options?: { where?: Record }) => { + if (!options?.where) return rows.length; + return rows.filter((p) => matchesWhere(p, options.where!)).length; + }), + _rows: rows, + }; +} + +function makeConfigService(overrides: Record = {}) { + const values: Record = { + PAYMENT_RECONCILE_DUE_AFTER_MINUTES: 5, + PAYMENT_RECONCILE_BACKOFF_BASE_MINUTES: 5, + PAYMENT_RECONCILE_BACKOFF_MAX_MINUTES: 60, + PAYMENT_MANUAL_REVIEW_AFTER_HOURS: 24, + PAYMENT_VERIFY_TIMEOUT_MS: 3000, + PAYMENT_RECONCILE_MAX_BATCH: 500, + PAYMENT_MANUAL_REVIEW_ALERT_THRESHOLD: 20, + ...overrides, + }; + return { + get: jest.fn((key: string, fallback?: number) => values[key] ?? fallback), + }; +} + +describe('ReconciliationService', () => { + let railAdapter: { verifyByReference: jest.Mock }; + let confirmationService: { apply: jest.Mock }; + let gateway: { emitPaymentUpdate: jest.Mock }; + + beforeEach(() => { + railAdapter = { verifyByReference: jest.fn() }; + confirmationService = { apply: jest.fn() }; + gateway = { emitPaymentUpdate: jest.fn() }; + }); + + function build( + seed: Payment[], + configOverrides: Record = {}, + ) { + const paymentRepository = makePaymentRepository(seed); + const config = makeConfigService(configOverrides); + const service = new ReconciliationService( + paymentRepository as any, + confirmationService as any, + railAdapter as any, + gateway as any, + config as any, + ); + return { service, paymentRepository }; + } + + const minutesAgo = (n: number, from: Date = new Date()) => + new Date(from.getTime() - n * 60_000); + const hoursAgo = (n: number, from: Date = new Date()) => + new Date(from.getTime() - n * 3_600_000); + + describe('reconcileDueBatch — resolves a webhook-never-arrives scenario', () => { + it('resolves a due, old-enough payment via the provider verify call', async () => { + const now = new Date(); + const payment = makePayment({ createdAt: minutesAgo(10, now) }); + confirmationService.apply.mockResolvedValue({ + ...payment, + status: PaymentStatus.CONFIRMED, + }); + railAdapter.verifyByReference.mockResolvedValue({ outcome: 'confirmed' }); + + const { service } = build([payment]); + const summary = await service.reconcileDueBatch(now); + + expect(summary.candidates).toBe(1); + expect(summary.resolved).toBe(1); + expect(confirmationService.apply).toHaveBeenCalledWith( + payment.providerReference, + 'confirmed', + ConfirmationSource.RECONCILIATION, + expect.any(String), + ); + }); + + it('excludes payments still within the initial grace window', async () => { + const now = new Date(); + const payment = makePayment({ createdAt: minutesAgo(1, now) }); + + const { service } = build([payment]); + const summary = await service.reconcileDueBatch(now); + + expect(summary.candidates).toBe(0); + expect(railAdapter.verifyByReference).not.toHaveBeenCalled(); + }); + + it('excludes a payment still within its backoff window from a prior attempt', async () => { + const now = new Date(); + const payment = makePayment({ + createdAt: minutesAgo(30, now), + reconciliationAttempts: 1, + // backoff for 1 prior attempt = 5 * 2^1 = 10 minutes + lastReconciledAt: minutesAgo(3, now), + }); + + const { service } = build([payment]); + const summary = await service.reconcileDueBatch(now); + + expect(summary.candidates).toBe(0); + expect(railAdapter.verifyByReference).not.toHaveBeenCalled(); + }); + + it('includes a payment once its backoff window has elapsed', async () => { + const now = new Date(); + const payment = makePayment({ + createdAt: minutesAgo(30, now), + reconciliationAttempts: 1, + lastReconciledAt: minutesAgo(11, now), // backoff was 10 minutes + }); + railAdapter.verifyByReference.mockResolvedValue({ outcome: 'pending' }); + + const { service } = build([payment]); + const summary = await service.reconcileDueBatch(now); + + expect(summary.candidates).toBe(1); + expect(summary.pending).toBe(1); + }); + }); + + describe('reconcileDueBatch — provider outage does not cause false-positive escalation', () => { + it('does not escalate an old payment to MANUAL_REVIEW when this attempt is itself a provider outage', async () => { + const now = new Date(); + const payment = makePayment({ createdAt: hoursAgo(48, now) }); // well past the 24h threshold + railAdapter.verifyByReference.mockRejectedValue( + new Error('provider unreachable'), + ); + + const { service, paymentRepository } = build([payment]); + const summary = await service.reconcileDueBatch(now); + + expect(summary.providerErrors).toBe(1); + expect(summary.escalatedToManualReview).toBe(0); + const updated = await paymentRepository.findOne({ + where: { id: payment.id }, + }); + expect(updated!.status).toBe(PaymentStatus.AWAITING_CONFIRMATION); + expect(updated!.providerErrorStreak).toBe(1); + }); + + it('escalates an old payment to MANUAL_REVIEW once the provider is reachable but still pending', async () => { + const now = new Date(); + const payment = makePayment({ createdAt: hoursAgo(48, now) }); + railAdapter.verifyByReference.mockResolvedValue({ outcome: 'pending' }); + + const { service, paymentRepository } = build([payment]); + const summary = await service.reconcileDueBatch(now); + + expect(summary.escalatedToManualReview).toBe(1); + const updated = await paymentRepository.findOne({ + where: { id: payment.id }, + }); + expect(updated!.status).toBe(PaymentStatus.MANUAL_REVIEW); + expect(updated!.manualReviewReason).toMatch(/Unresolved after/); + expect(gateway.emitPaymentUpdate).toHaveBeenCalledWith( + payment.id, + PaymentStatus.MANUAL_REVIEW, + ); + }); + + it('resets the provider error streak after a successful contact', async () => { + const now = new Date(); + const payment = makePayment({ + createdAt: minutesAgo(30, now), + providerErrorStreak: 4, + }); + railAdapter.verifyByReference.mockResolvedValue({ outcome: 'pending' }); + + const { service, paymentRepository } = build([payment]); + await service.reconcileDueBatch(now); + + const updated = await paymentRepository.findOne({ + where: { id: payment.id }, + }); + expect(updated!.providerErrorStreak).toBe(0); + }); + }); + + describe('reconcileDueBatch — idempotency', () => { + it('is safe to run twice: a resolved payment is not reprocessed on the second pass', async () => { + const now = new Date(); + const payment = makePayment({ createdAt: minutesAgo(10, now) }); + confirmationService.apply.mockResolvedValue({ + ...payment, + status: PaymentStatus.CONFIRMED, + }); + railAdapter.verifyByReference.mockResolvedValue({ outcome: 'confirmed' }); + + const { service, paymentRepository } = build([payment]); + await service.reconcileDueBatch(now); + // The confirmationService.apply mock doesn't actually flip the fake + // repository's row to CONFIRMED (it's mocked, not the real + // idempotent implementation) — simulate what apply() would really do + // so the second pass's status=AWAITING_CONFIRMATION query excludes it. + await paymentRepository.update(payment.id, { + status: PaymentStatus.CONFIRMED, + }); + + await service.reconcileDueBatch(new Date(now.getTime() + 60_000)); + + expect(confirmationService.apply).toHaveBeenCalledTimes(1); + }); + }); + + describe('sweepExpired (via reconcileDueBatch)', () => { + it('expires an INITIATED payment past its TTL with reason ABANDONED', async () => { + const now = new Date(); + const payment = makePayment({ + status: PaymentStatus.INITIATED, + expiresAt: minutesAgo(1, now), + }); + + const { service, paymentRepository } = build([payment]); + const summary = await service.reconcileDueBatch(now); + + expect(summary.expiredSwept).toBe(1); + const updated = await paymentRepository.findOne({ + where: { id: payment.id }, + }); + expect(updated!.status).toBe(PaymentStatus.EXPIRED); + expect(updated!.failureReason).toBe(PaymentFailureReason.ABANDONED); + }); + + it('expires an AWAITING_CONFIRMATION payment past its TTL with reason EXPIRED', async () => { + const now = new Date(); + const payment = makePayment({ + status: PaymentStatus.AWAITING_CONFIRMATION, + expiresAt: minutesAgo(1, now), + createdAt: minutesAgo(30, now), + }); + + const { service, paymentRepository } = build([payment]); + const summary = await service.reconcileDueBatch(now); + + expect(summary.expiredSwept).toBe(1); + const updated = await paymentRepository.findOne({ + where: { id: payment.id }, + }); + expect(updated!.status).toBe(PaymentStatus.EXPIRED); + expect(updated!.failureReason).toBe(PaymentFailureReason.EXPIRED); + // Already swept to EXPIRED — must not also be treated as an + // AWAITING_CONFIRMATION reconciliation candidate in the same pass. + expect(railAdapter.verifyByReference).not.toHaveBeenCalled(); + }); + + it('does not expire a payment with no expiresAt set', async () => { + const now = new Date(); + const payment = makePayment({ + status: PaymentStatus.INITIATED, + expiresAt: null, + }); + + const { service } = build([payment]); + const summary = await service.reconcileDueBatch(now); + + expect(summary.expiredSwept).toBe(0); + }); + }); + + describe('admin recovery actions', () => { + it('forceReconcileNow reconciles a payment immediately, bypassing the due schedule', async () => { + const now = new Date(); + const payment = makePayment({ createdAt: minutesAgo(1, now) }); // would NOT be due yet + confirmationService.apply.mockResolvedValue({ + ...payment, + status: PaymentStatus.CONFIRMED, + }); + railAdapter.verifyByReference.mockResolvedValue({ outcome: 'confirmed' }); + + const { service } = build([payment]); + await service.forceReconcileNow(payment.id); + + expect(railAdapter.verifyByReference).toHaveBeenCalledWith( + payment.providerReference, + ); + }); + + it('forceReconcileNow rejects a payment that is not AWAITING_CONFIRMATION', async () => { + const payment = makePayment({ status: PaymentStatus.CONFIRMED }); + const { service } = build([payment]); + + await expect(service.forceReconcileNow(payment.id)).rejects.toThrow( + UnprocessableEntityException, + ); + }); + + it('resolveManually requires a reason', async () => { + const payment = makePayment({ status: PaymentStatus.MANUAL_REVIEW }); + const { service } = build([payment]); + + await expect( + service.resolveManually(payment.id, PaymentStatus.CONFIRMED, ''), + ).rejects.toThrow(/reason is required/); + }); + + it('resolveManually transitions MANUAL_REVIEW to CONFIRMED with the given reason', async () => { + const payment = makePayment({ status: PaymentStatus.MANUAL_REVIEW }); + const { service } = build([payment]); + + const result = await service.resolveManually( + payment.id, + PaymentStatus.CONFIRMED, + 'Verified manually with the provider dashboard', + ); + + expect(result.status).toBe(PaymentStatus.CONFIRMED); + expect(result.manualReviewReason).toBe( + 'Verified manually with the provider dashboard', + ); + }); + + it('resolveManually sets a DECLINED failureReason when resolving to FAILED', async () => { + const payment = makePayment({ status: PaymentStatus.MANUAL_REVIEW }); + const { service } = build([payment]); + + const result = await service.resolveManually( + payment.id, + PaymentStatus.FAILED, + 'Confirmed declined', + ); + + expect(result.status).toBe(PaymentStatus.FAILED); + expect(result.failureReason).toBe(PaymentFailureReason.DECLINED); + }); + + it('void requires a reason and transitions MANUAL_REVIEW to VOIDED', async () => { + const payment = makePayment({ status: PaymentStatus.MANUAL_REVIEW }); + const { service } = build([payment]); + + await expect(service.void(payment.id, '')).rejects.toThrow( + /reason is required/, + ); + + const result = await service.void( + payment.id, + 'Booking cancelled by shipper', + ); + expect(result.status).toBe(PaymentStatus.VOIDED); + }); + + it('throws NotFoundException for an unknown payment id', async () => { + const { service } = build([]); + await expect(service.forceReconcileNow('missing')).rejects.toThrow( + NotFoundException, + ); + }); + + it('listManualReview returns only MANUAL_REVIEW payments', async () => { + const inReview = makePayment({ + id: 'p1', + status: PaymentStatus.MANUAL_REVIEW, + }); + const confirmed = makePayment({ + id: 'p2', + status: PaymentStatus.CONFIRMED, + }); + const { service } = build([inReview, confirmed]); + + const result = await service.listManualReview(); + expect(result.map((p) => p.id)).toEqual(['p1']); + }); + + it('getMetrics reports the manual-review queue depth and alerting flag', async () => { + const payments = Array.from({ length: 25 }, (_, i) => + makePayment({ id: `p${i}`, status: PaymentStatus.MANUAL_REVIEW }), + ); + const { service } = build(payments); + + const metrics = await service.getMetrics(); + expect(metrics.manualReviewQueueDepth).toBe(25); + expect(metrics.alertThreshold).toBe(20); + expect(metrics.alerting).toBe(true); + }); + }); +}); diff --git a/backend/src/payments/reconciliation.service.ts b/backend/src/payments/reconciliation.service.ts new file mode 100644 index 00000000..563de6d8 --- /dev/null +++ b/backend/src/payments/reconciliation.service.ts @@ -0,0 +1,389 @@ +import { + BadRequestException, + Injectable, + Logger, + NotFoundException, + UnprocessableEntityException, +} from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Cron, CronExpression } from '@nestjs/schedule'; +import { LessThan, Repository } from 'typeorm'; +import { Payment } from './entities/payment.entity'; +import { PaymentStatus } from './enums/payment-status.enum'; +import { PaymentFailureReason } from './enums/payment-failure-reason.enum'; +import { ConfirmationSource } from './enums/confirmation-source.enum'; +import { assertValidTransition } from './payment-state-machine'; +import { PaymentConfirmationService } from './payment-confirmation.service'; +import { PaymentsGateway } from './payments.gateway'; +import { SandboxRailAdapter } from './adapters/sandbox-rail.adapter'; +import { withTimeout } from './utils/with-timeout'; + +export interface ReconciliationSummary { + candidates: number; + resolved: number; + pending: number; + providerErrors: number; + escalatedToManualReview: number; + expiredSwept: number; +} + +export interface ReconciliationMetrics { + manualReviewQueueDepth: number; + alertThreshold: number; + alerting: boolean; +} + +type ReconcileOutcome = 'resolved' | 'pending' | 'provider_error' | 'escalated'; + +/** + * Scheduled reconciliation (issue #1572): treats provider truth as + * authoritative and self-heals drift for payments a webhook never + * confirmed. Reuses PaymentConfirmationService.apply — the same idempotent + * path the webhook and verify-on-return paths use — so a payment resolved + * here can never be double-applied, and re-running this job is always safe. + */ +@Injectable() +export class ReconciliationService { + private readonly logger = new Logger(ReconciliationService.name); + + constructor( + @InjectRepository(Payment) + private readonly paymentRepository: Repository, + private readonly confirmationService: PaymentConfirmationService, + private readonly railAdapter: SandboxRailAdapter, + private readonly gateway: PaymentsGateway, + private readonly config: ConfigService, + ) {} + + @Cron(CronExpression.EVERY_5_MINUTES) + async handleCron(): Promise { + const summary = await this.reconcileDueBatch(); + this.logger.log(`Reconciliation pass: ${JSON.stringify(summary)}`); + const metrics = await this.getMetrics(); + if (metrics.alerting) { + this.logger.warn( + `ALERT: manual-review queue depth ${metrics.manualReviewQueueDepth} ` + + `exceeds threshold ${metrics.alertThreshold}`, + ); + } + } + + /** + * The directly-testable core: sweeps expired payments, then reconciles + * every AWAITING_CONFIRMATION payment that's due for a poll (past the + * short threshold, respecting each payment's own backoff schedule). + */ + async reconcileDueBatch( + now: Date = new Date(), + ): Promise { + const expiredSwept = await this.sweepExpired(now); + + const maxBatch = this.config.get( + 'PAYMENT_RECONCILE_MAX_BATCH', + 500, + ); + const awaiting = await this.paymentRepository.find({ + where: { status: PaymentStatus.AWAITING_CONFIRMATION }, + take: maxBatch, + }); + + const summary: ReconciliationSummary = { + candidates: 0, + resolved: 0, + pending: 0, + providerErrors: 0, + escalatedToManualReview: 0, + expiredSwept, + }; + + for (const payment of awaiting) { + if (!this.isDueForPoll(payment, now)) { + continue; + } + summary.candidates++; + const outcome = await this.reconcileOne(payment, now); + switch (outcome) { + case 'resolved': + summary.resolved++; + break; + case 'pending': + summary.pending++; + break; + case 'provider_error': + summary.providerErrors++; + break; + case 'escalated': + summary.escalatedToManualReview++; + break; + } + } + + return summary; + } + + /** Admin recovery action: reconcile one payment immediately, bypassing the due-schedule. */ + async forceReconcileNow(paymentId: string): Promise { + const payment = await this.getPaymentOrThrow(paymentId); + if (payment.status !== PaymentStatus.AWAITING_CONFIRMATION) { + throw new UnprocessableEntityException( + `Payment in status ${payment.status} is not awaiting confirmation`, + ); + } + await this.reconcileOne(payment, new Date()); + return this.getPaymentOrThrow(paymentId); + } + + /** Admin recovery action: resolve a MANUAL_REVIEW payment by hand — reason required, audited. */ + async resolveManually( + paymentId: string, + resolution: PaymentStatus.CONFIRMED | PaymentStatus.FAILED, + reason: string, + ): Promise { + this.assertReason(reason, 'resolve a payment manually'); + const payment = await this.getPaymentOrThrow(paymentId); + assertValidTransition(payment.status, resolution); + + payment.status = resolution; + payment.manualReviewReason = reason; + if (resolution === PaymentStatus.FAILED) { + payment.failureReason = PaymentFailureReason.DECLINED; + } + const saved = await this.paymentRepository.save(payment); + this.gateway.emitPaymentUpdate(saved.id, saved.status); + this.logger.log( + `Payment ${paymentId} manually resolved to ${resolution}: ${reason}`, + ); + return saved; + } + + /** Admin recovery action: closes a payment out without resolving it CONFIRMED/FAILED. */ + async void(paymentId: string, reason: string): Promise { + this.assertReason(reason, 'void a payment'); + const payment = await this.getPaymentOrThrow(paymentId); + assertValidTransition(payment.status, PaymentStatus.VOIDED); + + payment.status = PaymentStatus.VOIDED; + payment.manualReviewReason = reason; + const saved = await this.paymentRepository.save(payment); + this.gateway.emitPaymentUpdate(saved.id, saved.status); + this.logger.log(`Payment ${paymentId} voided: ${reason}`); + return saved; + } + + async listManualReview(): Promise { + return this.paymentRepository.find({ + where: { status: PaymentStatus.MANUAL_REVIEW }, + order: { updatedAt: 'ASC' }, + }); + } + + async getMetrics(): Promise { + const manualReviewQueueDepth = await this.paymentRepository.count({ + where: { status: PaymentStatus.MANUAL_REVIEW }, + }); + const alertThreshold = this.config.get( + 'PAYMENT_MANUAL_REVIEW_ALERT_THRESHOLD', + 20, + ); + return { + manualReviewQueueDepth, + alertThreshold, + alerting: manualReviewQueueDepth > alertThreshold, + }; + } + + /** + * INITIATED past its TTL never even reached the provider (the user never + * returned to complete checkout) — ABANDONED. AWAITING_CONFIRMATION past + * its TTL did start, but nothing ever confirmed it in time — EXPIRED. + */ + private async sweepExpired(now: Date): Promise { + const expired = await this.paymentRepository.find({ + where: [ + { status: PaymentStatus.INITIATED, expiresAt: LessThan(now) }, + { + status: PaymentStatus.AWAITING_CONFIRMATION, + expiresAt: LessThan(now), + }, + ], + }); + + for (const payment of expired) { + const reason = + payment.status === PaymentStatus.INITIATED + ? PaymentFailureReason.ABANDONED + : PaymentFailureReason.EXPIRED; + assertValidTransition(payment.status, PaymentStatus.EXPIRED); + payment.status = PaymentStatus.EXPIRED; + payment.failureReason = reason; + await this.paymentRepository.save(payment); + this.gateway.emitPaymentUpdate(payment.id, payment.status); + } + + return expired.length; + } + + private async reconcileOne( + payment: Payment, + now: Date, + ): Promise { + if (!payment.providerReference) { + this.logger.warn( + `Payment ${payment.id} is AWAITING_CONFIRMATION with no providerReference`, + ); + return this.recordAttempt(payment, now, { providerError: true }); + } + + const timeoutMs = this.config.get( + 'PAYMENT_VERIFY_TIMEOUT_MS', + 3000, + ); + let outcome: 'confirmed' | 'failed' | 'pending'; + try { + const result = await withTimeout( + this.railAdapter.verifyByReference(payment.providerReference), + timeoutMs, + ); + outcome = result.outcome; + } catch (error) { + this.logger.warn( + `Reconciliation verify failed for payment ${payment.id}: ` + + (error instanceof Error ? error.message : String(error)), + ); + return this.recordAttempt(payment, now, { providerError: true }); + } + + if (outcome === 'pending') { + return this.recordAttempt(payment, now, { providerError: false }); + } + + const rawPayloadHash = PaymentConfirmationService.hashPayload( + Buffer.from( + JSON.stringify({ + providerReference: payment.providerReference, + outcome, + }), + ), + ); + const applied = await this.confirmationService.apply( + payment.providerReference, + outcome, + ConfirmationSource.RECONCILIATION, + rawPayloadHash, + ); + + await this.paymentRepository.update(payment.id, { + lastReconciledAt: now, + providerErrorStreak: 0, + reconciliationAttempts: payment.reconciliationAttempts + 1, + // Only tag a failure reason if this call is what actually resolved it + // to FAILED — a race where the webhook already settled it first + // (to CONFIRMED, say) must not be overwritten. + ...(applied?.status === PaymentStatus.FAILED + ? { failureReason: PaymentFailureReason.DECLINED } + : {}), + }); + + return 'resolved'; + } + + /** + * Persists the attempt (always — this is what makes a re-run idempotent + * and what drives backoff scheduling), then decides whether this + * payment has earned MANUAL_REVIEW escalation. + * + * Escalation is deliberately withheld whenever THIS attempt itself was a + * provider error — a provider-side outage must never mass-flag every + * in-flight payment after a single bad run. Only an attempt that + * successfully reached the provider (and got "still pending") can push + * an old-enough payment into MANUAL_REVIEW. + */ + private async recordAttempt( + payment: Payment, + now: Date, + opts: { providerError: boolean }, + ): Promise { + const attempts = payment.reconciliationAttempts + 1; + const providerErrorStreak = opts.providerError + ? payment.providerErrorStreak + 1 + : 0; + + await this.paymentRepository.update(payment.id, { + reconciliationAttempts: attempts, + providerErrorStreak, + lastReconciledAt: now, + }); + + const manualReviewAfterHours = this.config.get( + 'PAYMENT_MANUAL_REVIEW_AFTER_HOURS', + 24, + ); + const ageMs = now.getTime() - payment.createdAt.getTime(); + const shouldEscalate = + !opts.providerError && ageMs >= manualReviewAfterHours * 3_600_000; + + if (!shouldEscalate) { + return opts.providerError ? 'provider_error' : 'pending'; + } + + const fresh = await this.getPaymentOrThrow(payment.id); + assertValidTransition(fresh.status, PaymentStatus.MANUAL_REVIEW); + fresh.status = PaymentStatus.MANUAL_REVIEW; + fresh.manualReviewReason = `Unresolved after ${attempts} reconciliation attempts (${(ageMs / 3_600_000).toFixed(1)}h old)`; + await this.paymentRepository.save(fresh); + this.gateway.emitPaymentUpdate(fresh.id, fresh.status); + return 'escalated'; + } + + /** + * Base backoff doubles per attempt (capped) — a bank-side hold that + * stays "still processing" for a long time gets polled less and less + * often instead of every single cron tick. + */ + private isDueForPoll(payment: Payment, now: Date): boolean { + const dueAfterMinutes = this.config.get( + 'PAYMENT_RECONCILE_DUE_AFTER_MINUTES', + 5, + ); + const ageMs = now.getTime() - payment.createdAt.getTime(); + if (ageMs < dueAfterMinutes * 60_000) { + return false; + } + if (!payment.lastReconciledAt) { + return true; + } + + const backoffMinutes = this.computeBackoffMinutes( + payment.reconciliationAttempts, + ); + const dueAt = payment.lastReconciledAt.getTime() + backoffMinutes * 60_000; + return now.getTime() >= dueAt; + } + + private computeBackoffMinutes(attempts: number): number { + const baseMinutes = this.config.get( + 'PAYMENT_RECONCILE_BACKOFF_BASE_MINUTES', + 5, + ); + const maxMinutes = this.config.get( + 'PAYMENT_RECONCILE_BACKOFF_MAX_MINUTES', + 60, + ); + return Math.min(baseMinutes * 2 ** attempts, maxMinutes); + } + + private assertReason(reason: string, action: string): void { + if (!reason?.trim()) { + throw new BadRequestException(`A reason is required to ${action}`); + } + } + + private async getPaymentOrThrow(id: string): Promise { + const payment = await this.paymentRepository.findOne({ where: { id } }); + if (!payment) { + throw new NotFoundException('Payment not found'); + } + return payment; + } +} diff --git a/backend/src/payments/refunds.service.spec.ts b/backend/src/payments/refunds.service.spec.ts new file mode 100644 index 00000000..c05497be --- /dev/null +++ b/backend/src/payments/refunds.service.spec.ts @@ -0,0 +1,248 @@ +import { + BadRequestException, + ConflictException, + NotFoundException, + UnprocessableEntityException, +} from '@nestjs/common'; +import { RefundsService } from './refunds.service'; +import { Payment } from './entities/payment.entity'; +import { Refund } from './entities/refund.entity'; +import { PaymentRail } from './enums/payment-rail.enum'; +import { PaymentStatus } from './enums/payment-status.enum'; + +function makePayment(overrides: Partial = {}): Payment { + return { + id: 'payment-1', + bookingId: 'booking-1', + userId: 'user-1', + amount: 1000, + currency: 'USD', + rail: PaymentRail.FIAT, + provider: null, + providerReference: 'sandbox_ref_1', + status: PaymentStatus.CONFIRMED, + idempotencyKey: 'key-1', + metadata: null, + expiresAt: null, + failureReason: null, + reconciliationAttempts: 0, + providerErrorStreak: 0, + lastReconciledAt: null, + manualReviewReason: null, + createdAt: new Date(), + updatedAt: new Date(), + ...overrides, + } as Payment; +} + +/** + * A small in-memory ledger simulation shared across sequential calls in a + * test — this is how a single-threaded jest run demonstrates the atomicity + * guarantee: bookRefund() always re-reads "refunded so far" fresh inside + * its transaction, so a second call sees the first's already-booked refund, + * exactly as a real row lock would force in a genuinely concurrent DB. + */ +function makeHarness(initialPayment: Payment) { + let payment = { ...initialPayment }; + const refunds: Refund[] = []; + let nextRefundId = 1; + + const paymentRepoQueryBuilder = { + setLock: jest.fn().mockReturnThis(), + where: jest.fn().mockReturnThis(), + getOne: jest.fn(async () => ({ ...payment })), + }; + + const refundRepoQueryBuilder = { + select: jest.fn().mockReturnThis(), + where: jest.fn().mockReturnThis(), + getRawOne: jest.fn(async () => ({ + total: String(refunds.reduce((sum, r) => sum + r.amount, 0)), + })), + }; + + const manager = { + getRepository: jest.fn((entity: unknown) => { + if (entity === Payment) { + return { + createQueryBuilder: jest.fn(() => paymentRepoQueryBuilder), + save: jest.fn(async (p: Payment) => { + payment = { ...p }; + return payment; + }), + }; + } + if (entity === Refund) { + return { + createQueryBuilder: jest.fn(() => refundRepoQueryBuilder), + create: jest.fn((data: Partial) => ({ ...data }) as Refund), + save: jest.fn(async (r: Refund) => { + const saved = { + ...r, + id: `refund-${nextRefundId++}`, + createdAt: new Date(), + } as Refund; + refunds.push(saved); + return saved; + }), + }; + } + throw new Error('Unexpected entity in mock manager.getRepository'); + }), + }; + + const paymentRepository = { + manager: { + transaction: jest.fn(async (cb: (m: typeof manager) => unknown) => + cb(manager), + ), + }, + }; + + return { + paymentRepository, + refundRepository: {}, + refunds, + getPayment: () => payment, + }; +} + +describe('RefundsService', () => { + let railAdapter: { refund: jest.Mock }; + let gateway: { emitPaymentUpdate: jest.Mock }; + + beforeEach(() => { + railAdapter = { refund: jest.fn().mockResolvedValue(undefined) }; + gateway = { emitPaymentUpdate: jest.fn() }; + }); + + function build(payment: Payment) { + const harness = makeHarness(payment); + const service = new RefundsService( + harness.paymentRepository as any, + harness.refundRepository as any, + railAdapter as any, + gateway as any, + ); + return { service, harness }; + } + + it('rejects a non-positive amount', async () => { + const { service } = build(makePayment()); + await expect( + service.requestRefund('payment-1', 0, 'reason', null), + ).rejects.toThrow(BadRequestException); + }); + + it('rejects a missing reason', async () => { + const { service } = build(makePayment()); + await expect( + service.requestRefund('payment-1', 100, ' ', null), + ).rejects.toThrow(BadRequestException); + }); + + it('rejects refunding a payment that is not CONFIRMED or PARTIALLY_REFUNDED', async () => { + const { service } = build(makePayment({ status: PaymentStatus.INITIATED })); + await expect( + service.requestRefund('payment-1', 100, 'not eligible', null), + ).rejects.toThrow(UnprocessableEntityException); + }); + + it('books a partial refund and moves the payment to PARTIALLY_REFUNDED', async () => { + const { service, harness } = build(makePayment({ amount: 1000 })); + + const result = await service.requestRefund( + 'payment-1', + 400, + 'partial', + 'admin-1', + ); + + expect(result.payment.status).toBe(PaymentStatus.PARTIALLY_REFUNDED); + expect(result.refund.amount).toBe(400); + expect(harness.refunds).toHaveLength(1); + expect(gateway.emitPaymentUpdate).toHaveBeenCalledWith( + 'payment-1', + PaymentStatus.PARTIALLY_REFUNDED, + ); + }); + + it('books a full refund and moves the payment to REFUNDED', async () => { + const { service } = build(makePayment({ amount: 1000 })); + + const result = await service.requestRefund( + 'payment-1', + 1000, + 'full refund', + null, + ); + + expect(result.payment.status).toBe(PaymentStatus.REFUNDED); + }); + + it('accumulates multiple partial refunds and completes into REFUNDED once the total matches', async () => { + const { service, harness } = build(makePayment({ amount: 1000 })); + + const first = await service.requestRefund('payment-1', 400, 'first', null); + expect(first.payment.status).toBe(PaymentStatus.PARTIALLY_REFUNDED); + + const second = await service.requestRefund( + 'payment-1', + 600, + 'second', + null, + ); + expect(second.payment.status).toBe(PaymentStatus.REFUNDED); + expect(harness.refunds).toHaveLength(2); + }); + + it('atomically rejects a second concurrent refund that would exceed the captured amount', async () => { + const { service } = build(makePayment({ amount: 1000 })); + + const first = await service.requestRefund('payment-1', 700, 'first', null); + expect(first.payment.status).toBe(PaymentStatus.PARTIALLY_REFUNDED); + + // Second refund request for 700 more would total 1400 > 1000 captured — + // rebooking must see the first refund's already-committed state (the + // "atomic" guarantee under test) and reject rather than double-refund. + await expect( + service.requestRefund('payment-1', 700, 'second (racing)', null), + ).rejects.toThrow(ConflictException); + }); + + it('throws NotFoundException when the payment does not exist', async () => { + const harness = makeHarness(makePayment()); + harness.paymentRepository.manager.transaction = jest.fn(async (cb: any) => + cb({ + getRepository: jest.fn(() => ({ + createQueryBuilder: jest.fn(() => ({ + setLock: jest.fn().mockReturnThis(), + where: jest.fn().mockReturnThis(), + getOne: jest.fn(async () => null), + })), + })), + }), + ); + const service = new RefundsService( + harness.paymentRepository as any, + {} as any, + railAdapter as any, + gateway as any, + ); + + await expect( + service.requestRefund('missing-payment', 100, 'reason', null), + ).rejects.toThrow(NotFoundException); + }); + + it('logs but does not throw when the provider-side refund call fails after retries', async () => { + railAdapter.refund.mockRejectedValue(new Error('provider down')); + const { service } = build(makePayment({ amount: 1000 })); + + await expect( + service.requestRefund('payment-1', 400, 'partial', null), + ).resolves.toMatchObject({ + payment: { status: PaymentStatus.PARTIALLY_REFUNDED }, + }); + }); +}); diff --git a/backend/src/payments/refunds.service.ts b/backend/src/payments/refunds.service.ts new file mode 100644 index 00000000..29ba885b --- /dev/null +++ b/backend/src/payments/refunds.service.ts @@ -0,0 +1,156 @@ +import { + BadRequestException, + ConflictException, + Injectable, + Logger, + NotFoundException, + UnprocessableEntityException, +} from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { EntityManager, Repository } from 'typeorm'; +import { Payment } from './entities/payment.entity'; +import { Refund } from './entities/refund.entity'; +import { PaymentStatus } from './enums/payment-status.enum'; +import { assertValidTransition } from './payment-state-machine'; +import { SandboxRailAdapter } from './adapters/sandbox-rail.adapter'; +import { PaymentsGateway } from './payments.gateway'; +import { retryWithBackoff } from './utils/retry-with-backoff'; + +const REFUNDABLE_STATUSES: ReadonlySet = new Set([ + PaymentStatus.CONFIRMED, + PaymentStatus.PARTIALLY_REFUNDED, +]); + +export interface RefundResult { + payment: Payment; + refund: Refund; +} + +/** + * Refund/partial-refund sub-ledger (issue #1572), layered on #1570's + * transition guard. A Payment's "amount refunded" is always + * SUM(refunds.amount), never a single boolean — this is what lets multiple + * partial refunds accumulate against one payment. + */ +@Injectable() +export class RefundsService { + private readonly logger = new Logger(RefundsService.name); + + constructor( + @InjectRepository(Payment) + private readonly paymentRepository: Repository, + @InjectRepository(Refund) + private readonly refundRepository: Repository, + private readonly railAdapter: SandboxRailAdapter, + private readonly gateway: PaymentsGateway, + ) {} + + /** + * Atomically validates and books a refund against `paymentId`: locks the + * payment row for the duration of the check-and-insert so two concurrent + * refund requests that would together exceed the captured amount can + * never both succeed — the loser sees a 409, not a corrupted ledger + * (issue #1572's tested double-refund race). + */ + async requestRefund( + paymentId: string, + amount: number, + reason: string, + actorId: string | null, + ): Promise { + if (!Number.isInteger(amount) || amount <= 0) { + throw new BadRequestException( + 'Refund amount must be a positive integer (minor units)', + ); + } + if (!reason?.trim()) { + throw new BadRequestException('Refund reason is required'); + } + + const { payment, refund } = + await this.paymentRepository.manager.transaction(async (manager) => + this.bookRefund(manager, paymentId, amount, reason, actorId), + ); + + this.gateway.emitPaymentUpdate(payment.id, payment.status); + + // Best-effort provider-side execution — the ledger above is already the + // source of truth for "was this refund accepted." A failure here is + // logged, not rolled back: reconciling a booked-but-not-yet-executed + // refund against the provider is out of scope for this issue (would + // need a saga/outbox, tracked separately). + try { + await retryWithBackoff( + () => this.railAdapter.refund(payment.providerReference ?? '', amount), + { maxAttempts: 3, baseDelayMs: 50, maxDelayMs: 500 }, + ); + } catch (error) { + this.logger.error( + `Refund ${refund.id} was booked but the provider call failed after retries: ` + + (error instanceof Error ? error.message : String(error)), + ); + } + + return { payment, refund }; + } + + private async bookRefund( + manager: EntityManager, + paymentId: string, + amount: number, + reason: string, + actorId: string | null, + ): Promise { + const payment = await manager + .getRepository(Payment) + .createQueryBuilder('payment') + .setLock('pessimistic_write') + .where('payment.id = :paymentId', { paymentId }) + .getOne(); + + if (!payment) { + throw new NotFoundException('Payment not found'); + } + if (!REFUNDABLE_STATUSES.has(payment.status)) { + throw new UnprocessableEntityException( + `Payment in status ${payment.status} cannot be refunded`, + ); + } + + const alreadyRefunded = await manager + .getRepository(Refund) + .createQueryBuilder('refund') + .select('COALESCE(SUM(refund.amount), 0)', 'total') + .where('refund.payment_id = :paymentId', { paymentId }) + .getRawOne<{ total: string }>(); + const refundedSoFar = Number(alreadyRefunded?.total ?? 0); + const remaining = payment.amount - refundedSoFar; + + if (amount > remaining) { + throw new ConflictException( + `Refund of ${amount} exceeds the remaining refundable amount (${remaining})`, + ); + } + + const refund = manager.getRepository(Refund).create({ + paymentId, + amount, + reason, + actorId, + }); + const savedRefund = await manager.getRepository(Refund).save(refund); + + const nextStatus = + refundedSoFar + amount >= payment.amount + ? PaymentStatus.REFUNDED + : PaymentStatus.PARTIALLY_REFUNDED; + + if (nextStatus !== payment.status) { + assertValidTransition(payment.status, nextStatus); + payment.status = nextStatus; + } + const savedPayment = await manager.getRepository(Payment).save(payment); + + return { payment: savedPayment, refund: savedRefund }; + } +} diff --git a/backend/src/payments/utils/retry-with-backoff.spec.ts b/backend/src/payments/utils/retry-with-backoff.spec.ts new file mode 100644 index 00000000..4ee84d39 --- /dev/null +++ b/backend/src/payments/utils/retry-with-backoff.spec.ts @@ -0,0 +1,121 @@ +import { retryWithBackoff } from './retry-with-backoff'; + +function fakeSleep(recorded: number[]) { + return async (ms: number): Promise => { + recorded.push(ms); + }; +} + +describe('retryWithBackoff', () => { + it('returns the result on the first successful attempt without sleeping', async () => { + const fn = jest.fn().mockResolvedValue('ok'); + const sleep = jest.fn(); + + const result = await retryWithBackoff(fn, { + maxAttempts: 3, + baseDelayMs: 100, + sleep, + }); + + expect(result).toBe('ok'); + expect(fn).toHaveBeenCalledTimes(1); + expect(sleep).not.toHaveBeenCalled(); + }); + + it('retries after a failure and eventually succeeds', async () => { + const fn = jest + .fn() + .mockRejectedValueOnce(new Error('boom')) + .mockRejectedValueOnce(new Error('boom again')) + .mockResolvedValueOnce('ok'); + const delays: number[] = []; + + const result = await retryWithBackoff(fn, { + maxAttempts: 3, + baseDelayMs: 100, + sleep: fakeSleep(delays), + }); + + expect(result).toBe('ok'); + expect(fn).toHaveBeenCalledTimes(3); + expect(delays).toHaveLength(2); + }); + + it('throws the last error once maxAttempts is exhausted', async () => { + const error = new Error('always fails'); + const fn = jest.fn().mockRejectedValue(error); + + await expect( + retryWithBackoff(fn, { + maxAttempts: 3, + baseDelayMs: 10, + sleep: jest.fn(), + }), + ).rejects.toThrow('always fails'); + expect(fn).toHaveBeenCalledTimes(3); + }); + + it('fails fast on a terminal (non-retryable) error without exhausting attempts', async () => { + class TerminalError extends Error {} + const fn = jest.fn().mockRejectedValue(new TerminalError('bad request')); + + await expect( + retryWithBackoff(fn, { + maxAttempts: 5, + baseDelayMs: 10, + sleep: jest.fn(), + isRetryable: (error) => !(error instanceof TerminalError), + }), + ).rejects.toThrow(TerminalError); + expect(fn).toHaveBeenCalledTimes(1); + }); + + it('grows the delay exponentially and applies jitter within [delay, 1.1*delay]', async () => { + const fn = jest + .fn() + .mockRejectedValueOnce(new Error('1')) + .mockRejectedValueOnce(new Error('2')) + .mockRejectedValueOnce(new Error('3')) + .mockResolvedValueOnce('ok'); + const delays: number[] = []; + + await retryWithBackoff(fn, { + maxAttempts: 4, + baseDelayMs: 100, + sleep: fakeSleep(delays), + }); + + expect(delays).toHaveLength(3); + expect(delays[0]).toBeGreaterThanOrEqual(100); + expect(delays[0]).toBeLessThanOrEqual(110); + expect(delays[1]).toBeGreaterThanOrEqual(200); + expect(delays[1]).toBeLessThanOrEqual(220); + expect(delays[2]).toBeGreaterThanOrEqual(400); + expect(delays[2]).toBeLessThanOrEqual(440); + }); + + it('caps the delay at maxDelayMs', async () => { + const fn = jest + .fn() + .mockRejectedValueOnce(new Error('1')) + .mockRejectedValueOnce(new Error('2')) + .mockResolvedValueOnce('ok'); + const delays: number[] = []; + + await retryWithBackoff(fn, { + maxAttempts: 3, + baseDelayMs: 100, + maxDelayMs: 150, + sleep: fakeSleep(delays), + }); + + // Second delay would be 200 uncapped; capped at 150 (+ up to 10% jitter). + expect(delays[1]).toBeLessThanOrEqual(165); + }); + + it('rejects a maxAttempts less than 1', async () => { + await expect( + retryWithBackoff(jest.fn(), { maxAttempts: 0, baseDelayMs: 10 }), + ).rejects.toThrow(/maxAttempts must be >= 1/); + }); +}); diff --git a/backend/src/payments/utils/retry-with-backoff.ts b/backend/src/payments/utils/retry-with-backoff.ts new file mode 100644 index 00000000..545b9eee --- /dev/null +++ b/backend/src/payments/utils/retry-with-backoff.ts @@ -0,0 +1,60 @@ +/** + * Shared retry/backoff utility (issue #1572) for our own outbound provider + * calls (refund, verify) — exponential backoff with jitter, a hard cap on + * attempts, and a caller-supplied distinction between retryable errors + * (timeout/5xx) and terminal ones (4xx), so a terminal error fails fast + * instead of burning through the full attempt budget. + */ +export interface RetryOptions { + /** Total attempts, including the first — must be >= 1. */ + maxAttempts: number; + /** Delay before the 2nd attempt; doubles each attempt after that. */ + baseDelayMs: number; + /** Upper bound on the (pre-jitter) delay, however many attempts have passed. */ + maxDelayMs?: number; + /** Defaults to "retry everything" — pass this to stop early on a terminal error. */ + isRetryable?: (error: unknown) => boolean; + /** Injectable for tests — defaults to a real setTimeout-based sleep. */ + sleep?: (ms: number) => Promise; +} + +function defaultSleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export async function retryWithBackoff( + fn: (attempt: number) => Promise, + options: RetryOptions, +): Promise { + const { + maxAttempts, + baseDelayMs, + maxDelayMs = Number.POSITIVE_INFINITY, + isRetryable = () => true, + sleep = defaultSleep, + } = options; + + if (maxAttempts < 1) { + throw new Error('retryWithBackoff: maxAttempts must be >= 1'); + } + + let lastError: unknown; + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + return await fn(attempt); + } catch (error) { + lastError = error; + const isLastAttempt = attempt === maxAttempts; + if (isLastAttempt || !isRetryable(error)) { + throw error; + } + const delay = Math.min(baseDelayMs * 2 ** (attempt - 1), maxDelayMs); + const jitter = delay * 0.1 * Math.random(); + await sleep(delay + jitter); + } + } + + // Unreachable — the loop above always either returns or throws — but + // keeps the compiler happy about a guaranteed return type. + throw lastError; +}