Payments 3/7: Failure Taxonomy, Retries, Partial Outcomes & Payment Reconciliation Engine

backend/.env.example

@@ -94,3 +94,24 @@ PAYMENT_WEBHOOK_SECRET=change-me-in-every-environment
 # a timeout falls back to the webhook/real-time channel, it never marks a
 # payment CONFIRMED on its own.
 PAYMENT_VERIFY_TIMEOUT_MS=3000
+
+# Payments reconciliation engine (issue #1572)
+# A payment isn't eligible for its first reconciliation pass until it's
+# been AWAITING_CONFIRMATION for at least this long — gives the webhook a
+# fair chance first.
+PAYMENT_RECONCILE_DUE_AFTER_MINUTES=5
+# Exponential backoff (in minutes) between reconciliation polls for a
+# still-pending payment: base * 2^attempts, capped at the max below — a
+# bank-side hold that stays "still processing" gets polled less and less
+# often instead of every cron tick.
+PAYMENT_RECONCILE_BACKOFF_BASE_MINUTES=5
+PAYMENT_RECONCILE_BACKOFF_MAX_MINUTES=60
+# Upper bound on how many AWAITING_CONFIRMATION payments one reconciliation
+# pass will consider.
+PAYMENT_RECONCILE_MAX_BATCH=500
+# A payment unresolved this long escalates to MANUAL_REVIEW — but only on a
+# pass where the provider was actually reachable; a provider outage never
+# by itself causes escalation (see ReconciliationService).
+PAYMENT_MANUAL_REVIEW_AFTER_HOURS=24
+# Logs a WARN-level alert when the manual-review queue depth exceeds this.
+PAYMENT_MANUAL_REVIEW_ALERT_THRESHOLD=20

backend/src/app.module.ts

@@ -1,6 +1,7 @@
 import { Module } from '@nestjs/common';
 import { ConfigModule, ConfigService } from '@nestjs/config';
 import { TypeOrmModule } from '@nestjs/typeorm';
+import { ScheduleModule } from '@nestjs/schedule';
 import { AppController } from './app.controller';
 import { AppService } from './app.service';
 import { AuthModule } from './auth/auth.module';
@@ -9,6 +10,8 @@ import { PaymentsModule } from './payments/payments.module';
 @Module({
   imports: [
     ConfigModule.forRoot({ isGlobal: true }),
+    // Powers ReconciliationService's @Cron job (issue #1572).
+    ScheduleModule.forRoot(),
     TypeOrmModule.forRootAsync({
       inject: [ConfigService],
       useFactory: (config: ConfigService) => ({

backend/src/database/migrations/1755870474000-AddPaymentReconciliationFields.ts

@@ -0,0 +1,70 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class AddPaymentReconciliationFields1755870474000 implements MigrationInterface {
+  name = 'AddPaymentReconciliationFields1755870474000';
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+      CREATE TYPE "payments_failure_reason_enum" AS ENUM (
+        'DECLINED', 'EXPIRED', 'PROVIDER_ERROR', 'ABANDONED'
+      )
+    `);
+
+    // New failure-taxonomy/escalation statuses (issue #1572) on top of
+    // #1570's original enum.
+    await queryRunner.query(`
+      ALTER TYPE "payments_status_enum" ADD VALUE IF NOT EXISTS 'MANUAL_REVIEW'
+    `);
+    await queryRunner.query(`
+      ALTER TYPE "payments_status_enum" ADD VALUE IF NOT EXISTS 'DISPUTED'
+    `);
+    await queryRunner.query(`
+      ALTER TYPE "payments_status_enum" ADD VALUE IF NOT EXISTS 'VOIDED'
+    `);
+
+    await queryRunner.query(`
+      ALTER TABLE "payments"
+        ADD COLUMN "failure_reason" "payments_failure_reason_enum",
+        ADD COLUMN "reconciliation_attempts" integer NOT NULL DEFAULT 0,
+        ADD COLUMN "provider_error_streak" integer NOT NULL DEFAULT 0,
+        ADD COLUMN "last_reconciled_at" timestamptz,
+        ADD COLUMN "manual_review_reason" text
+    `);
+
+    await queryRunner.query(`
+      CREATE TABLE "payment_refunds" (
+        "id" uuid NOT NULL DEFAULT gen_random_uuid(),
+        "payment_id" uuid NOT NULL,
+        "amount" bigint NOT NULL,
+        "reason" text NOT NULL,
+        "actor_id" uuid,
+        "created_at" TIMESTAMP NOT NULL DEFAULT now(),
+        CONSTRAINT "pk_payment_refunds" PRIMARY KEY ("id"),
+        CONSTRAINT "fk_payment_refunds_payment" FOREIGN KEY ("payment_id")
+          REFERENCES "payments"("id") ON DELETE NO ACTION ON UPDATE NO ACTION
+      )
+    `);
+
+    await queryRunner.query(`
+      CREATE INDEX "idx_payment_refunds_payment_id" ON "payment_refunds" ("payment_id")
+    `);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`DROP INDEX "idx_payment_refunds_payment_id"`);
+    await queryRunner.query(`DROP TABLE "payment_refunds"`);
+    await queryRunner.query(`
+      ALTER TABLE "payments"
+        DROP COLUMN "manual_review_reason",
+        DROP COLUMN "last_reconciled_at",
+        DROP COLUMN "provider_error_streak",
+        DROP COLUMN "reconciliation_attempts",
+        DROP COLUMN "failure_reason"
+    `);
+    // Postgres has no DROP VALUE for enums — reverting MANUAL_REVIEW /
+    // DISPUTED / VOIDED would require recreating payments_status_enum,
+    // which isn't safe to do automatically without knowing whether any row
+    // already uses them.
+    await queryRunner.query(`DROP TYPE "payments_failure_reason_enum"`);
+  }
+}

backend/src/payments/README.md

@@ -0,0 +1,100 @@
+# Payments module
+
+Payment domain model, initiation, confirmation, reconciliation, and refunds
+for the booking platform. This module is built incrementally across a
+payment track of issues (see `payment-state-machine.ts` for the canonical
+list of legal state transitions — that file, not this README, is the source
+of truth if the two ever disagree).
+
+## Status lifecycle
+
+`INITIATED → AWAITING_CONFIRMATION → CONFIRMED → (PARTIALLY_REFUNDED →) REFUNDED`
+
+Off that happy path:
+
+- `INITIATED → FAILED | EXPIRED`
+- `AWAITING_CONFIRMATION → FAILED | EXPIRED | MANUAL_REVIEW`
+- `CONFIRMED → DISPUTED`
+- `DISPUTED → REFUNDED`
+- `MANUAL_REVIEW → CONFIRMED | FAILED | VOIDED`
+
+## Failure taxonomy (issue #1572)
+
+`Payment#failureReason` records **why** a payment ended up `FAILED` or
+`EXPIRED` — first-class enum data, not free text (`payment-failure-reason.enum.ts`):
+
+| Reason           | Meaning                                                                 |
+| ---------------- | ------------------------------------------------------------------------ |
+| `DECLINED`        | The provider explicitly rejected the charge.                            |
+| `EXPIRED`         | No confirmation (webhook or reconciliation) arrived within the TTL.     |
+| `PROVIDER_ERROR`  | Talking to the provider itself failed (5xx/timeout) — not a verdict.    |
+| `ABANDONED`       | The payment never progressed past `INITIATED` before expiring.         |
+
+## Reconciliation engine
+
+`ReconciliationService` runs on a schedule (`@Cron`, every 5 minutes by
+default) and does two things:
+
+1. **Expiry sweep** — `INITIATED`/`AWAITING_CONFIRMATION` payments past
+   `Payment#expiresAt` become `EXPIRED` (`ABANDONED` vs `EXPIRED` reason
+   depending on which state they expired from).
+2. **Reconciliation** — every `AWAITING_CONFIRMATION` payment old enough
+   (`PAYMENT_RECONCILE_DUE_AFTER_MINUTES`) and due for its next poll (an
+   exponential backoff schedule per payment — see
+   `PAYMENT_RECONCILE_BACKOFF_*`) is re-verified directly against the
+   provider, reusing the same `verifyByReference` call and the same
+   idempotent `PaymentConfirmationService.apply` path the webhook and
+   verify-on-return flows use. A payment resolved this way can never be
+   double-applied, and re-running the job is always safe.
+
+A payment still unresolved after `PAYMENT_MANUAL_REVIEW_AFTER_HOURS`
+escalates to `MANUAL_REVIEW` — **but only on a reconciliation pass where the
+provider was actually reachable**. A provider-side outage (every verify
+call in a batch throwing/timing out) never by itself escalates anything —
+`Payment#providerErrorStreak` tracks consecutive unreachable attempts
+separately from the age-based threshold, specifically so one bad run can't
+mass-flag every in-flight payment.
+
+### Admin recovery actions (`PaymentsAdminController`, `ADMIN` role only)
+
+- `GET /payments/admin/manual-review` — the review queue, oldest first.
+- `GET /payments/admin/metrics` — manual-review queue depth + alert status.
+- `POST /payments/admin/:id/force-reconcile` — reconcile one payment now,
+  bypassing the backoff schedule.
+- `POST /payments/admin/:id/resolve-manually` — `{ resolution, reason }`,
+  reason required and audited via `Payment#manualReviewReason`.
+- `POST /payments/admin/:id/void` — `{ reason }`, closes a payment out
+  without resolving it `CONFIRMED`/`FAILED`.
+- `POST /payments/admin/:id/refunds` — `{ amount, reason }`, see below.
+
+## Refunds (partial-outcome support)
+
+A `Payment`'s refunded amount is always `SUM(payment_refunds.amount)` for
+that payment — never a single boolean — so multiple partial refunds can
+accumulate against one payment (`RefundsService`, `refund.entity.ts`).
+
+`RefundsService.requestRefund` locks the payment row for the duration of
+the check-and-insert (`pessimistic_write`), so two refund requests that
+would together exceed the captured amount can never both succeed: the
+loser gets a `409 Conflict`, not a corrupted ledger. The payment's status
+moves to `PARTIALLY_REFUNDED` or `REFUNDED` depending on whether the
+refunded total has reached the captured amount. Provider-side execution of
+the refund happens *after* the ledger commits (best-effort, retried via
+`utils/retry-with-backoff.ts`) — the ledger, not the provider call, is the
+source of truth for "was this refund accepted."
+
+## Retry/backoff utility
+
+`utils/retry-with-backoff.ts` is a small, generic, independently-tested
+exponential-backoff-with-jitter helper for our own outbound provider calls
+— capped attempts, and an optional `isRetryable` predicate so a terminal
+error (4xx) fails fast instead of burning the full attempt budget. Used by
+`RefundsService`'s post-commit provider refund call.
+
+## Metrics & alerting
+
+`ReconciliationService.getMetrics()` reports the manual-review queue depth
+and whether it exceeds `PAYMENT_MANUAL_REVIEW_ALERT_THRESHOLD`; the
+scheduled job logs a `WARN`-level alert when it does. This is a
+log-based signal by design — this module doesn't assume any particular
+metrics/observability backend is wired up yet.

backend/src/payments/adapters/sandbox-rail.adapter.ts

@@ -88,4 +88,10 @@ export class SandboxRailAdapter implements PaymentRailAdapter {
   ): value is PaymentVerificationOutcome {
     return value === 'confirmed' || value === 'failed' || value === 'pending';
   }
+
+  // No real provider wired up yet — a refund here always "succeeds"
+  // (matches initiate()/verifyByReference()'s sandbox placeholder nature).
+  async refund(_providerReference: string, _amount: number): Promise<void> {
+    return;
+  }
 }

backend/src/payments/dto/create-refund.dto.ts

@@ -0,0 +1,16 @@
+import { ApiProperty } from '@nestjs/swagger';
+import { IsInt, IsNotEmpty, IsPositive, IsString } from 'class-validator';
+
+export class CreateRefundDto {
+  @ApiProperty({
+    description: 'Minor units (e.g. cents) — same convention as Payment#amount',
+  })
+  @IsInt()
+  @IsPositive()
+  amount: number;
+
+  @ApiProperty()
+  @IsString()
+  @IsNotEmpty()
+  reason: string;
+}

backend/src/payments/dto/payment-response.dto.ts

@@ -1,6 +1,7 @@
 import { ApiProperty } from '@nestjs/swagger';
 import { PaymentRail } from '../enums/payment-rail.enum';
 import { PaymentStatus } from '../enums/payment-status.enum';
+import { PaymentFailureReason } from '../enums/payment-failure-reason.enum';
 import { Payment } from '../entities/payment.entity';
 
 export class PaymentResponseDto {
@@ -18,6 +19,10 @@ export class PaymentResponseDto {
   @ApiProperty({ nullable: true }) provider: string | null;
   @ApiProperty({ nullable: true }) providerReference: string | null;
   @ApiProperty({ nullable: true }) expiresAt: Date | null;
+  @ApiProperty({ enum: PaymentFailureReason, nullable: true })
+  failureReason: PaymentFailureReason | null;
+  @ApiProperty() reconciliationAttempts: number;
+  @ApiProperty({ nullable: true }) manualReviewReason: string | null;
   @ApiProperty() createdAt: Date;
   @ApiProperty() updatedAt: Date;
 
@@ -33,6 +38,9 @@ export class PaymentResponseDto {
     dto.provider = payment.provider;
     dto.providerReference = payment.providerReference;
     dto.expiresAt = payment.expiresAt;
+    dto.failureReason = payment.failureReason;
+    dto.reconciliationAttempts = payment.reconciliationAttempts;
+    dto.manualReviewReason = payment.manualReviewReason;
     dto.createdAt = payment.createdAt;
     dto.updatedAt = payment.updatedAt;
     return dto;

backend/src/payments/dto/refund-response.dto.ts

@@ -0,0 +1,19 @@
+import { ApiProperty } from '@nestjs/swagger';
+import { PaymentResponseDto } from './payment-response.dto';
+
+export class RefundResponseDto {
+  @ApiProperty() id: string;
+  @ApiProperty() paymentId: string;
+  @ApiProperty() amount: number;
+  @ApiProperty() reason: string;
+  @ApiProperty({ nullable: true }) actorId: string | null;
+  @ApiProperty() createdAt: Date;
+}
+
+export class RequestRefundResponseDto {
+  @ApiProperty({ type: PaymentResponseDto })
+  payment: PaymentResponseDto;
+
+  @ApiProperty({ type: RefundResponseDto })
+  refund: RefundResponseDto;
+}

backend/src/payments/dto/resolve-payment-manually.dto.ts

@@ -0,0 +1,14 @@
+import { ApiProperty } from '@nestjs/swagger';
+import { IsIn, IsNotEmpty, IsString } from 'class-validator';
+import { PaymentStatus } from '../enums/payment-status.enum';
+
+export class ResolvePaymentManuallyDto {
+  @ApiProperty({ enum: [PaymentStatus.CONFIRMED, PaymentStatus.FAILED] })
+  @IsIn([PaymentStatus.CONFIRMED, PaymentStatus.FAILED])
+  resolution: PaymentStatus.CONFIRMED | PaymentStatus.FAILED;
+
+  @ApiProperty({ description: 'Required — audited alongside the resolution' })
+  @IsString()
+  @IsNotEmpty()
+  reason: string;
+}

backend/src/payments/dto/void-payment.dto.ts

@@ -0,0 +1,9 @@
+import { ApiProperty } from '@nestjs/swagger';
+import { IsNotEmpty, IsString } from 'class-validator';
+
+export class VoidPaymentDto {
+  @ApiProperty({ description: 'Required — audited alongside the void action' })
+  @IsString()
+  @IsNotEmpty()
+  reason: string;
+}

backend/src/payments/entities/payment.entity.ts

@@ -8,6 +8,7 @@ import {
 } from 'typeorm';
 import { PaymentRail } from '../enums/payment-rail.enum';
 import { PaymentStatus } from '../enums/payment-status.enum';
+import { PaymentFailureReason } from '../enums/payment-failure-reason.enum';
 
 @Entity('payments')
 @Index(['userId', 'idempotencyKey'], { unique: true })
@@ -63,6 +64,38 @@ export class Payment {
   @Column({ type: 'timestamptz', name: 'expires_at', nullable: true })
   expiresAt: Date | null;
 
+  // ── Reconciliation (issue #1572) ────────────────────────────────────────
+
+  /** Set alongside a FAILED or EXPIRED status — WHY, not just WHAT. */
+  @Column({
+    type: 'enum',
+    enum: PaymentFailureReason,
+    name: 'failure_reason',
+    nullable: true,
+  })
+  failureReason: PaymentFailureReason | null;
+
+  /** Every reconciliation pass that touched this payment, resolved or not — drives backoff scheduling. */
+  @Column({ type: 'int', name: 'reconciliation_attempts', default: 0 })
+  reconciliationAttempts: number;
+
+  /**
+   * Consecutive reconciliation passes where the provider itself was
+   * unreachable (not "provider said pending"). Reset to 0 the moment a
+   * verify call successfully reaches the provider again. Used to withhold
+   * MANUAL_REVIEW escalation during a provider outage — see
+   * ReconciliationService.
+   */
+  @Column({ type: 'int', name: 'provider_error_streak', default: 0 })
+  providerErrorStreak: number;
+
+  @Column({ type: 'timestamptz', name: 'last_reconciled_at', nullable: true })
+  lastReconciledAt: Date | null;
+
+  /** Reason text for a MANUAL_REVIEW escalation or an admin resolve/void action — audited. */
+  @Column({ type: 'text', name: 'manual_review_reason', nullable: true })
+  manualReviewReason: string | null;
+
   @CreateDateColumn({ name: 'created_at' })
   createdAt: Date;