diff --git a/scripts/drill-db-password-rotation.sh b/scripts/drill-db-password-rotation.sh new file mode 100644 index 00000000..b663c309 --- /dev/null +++ b/scripts/drill-db-password-rotation.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Database Password Rotation Drill Script +# Verifies credential rotation at runtime without dropping connections. +# Runs against staging ONLY (refuses production URLs). +# +# Usage: ./scripts/drill-db-password-rotation.sh [--check|--rotate|--all|--help] + +: "${STAGING_URL:=http://localhost:3000/health}" +: "${DB_ROTATION_ENABLED:=false}" +START_EPOCH=$(date +%s) +FAILURES=0 + +log_pass() { echo "[PASS] $1"; } +log_fail() { echo "[FAIL] $1"; FAILURES=$((FAILURES + 1)); } +log_info() { echo "[INFO] $1"; } +log_warn() { echo "[WARN] $1"; } + +safety_check() { + log_info "=== Safety Guardrails ===" + if [[ "${STAGING_URL}" == *"api.revora.io"* ]] && [[ "${STAGING_URL}" != *"staging"* ]]; then + log_fail "STAGING_URL points to production — aborting." + exit 1 + fi + if [[ "${DB_ROTATION_ENABLED}" != "true" ]]; then + log_warn "DB_ROTATION_ENABLED is not 'true' — server-side rotation is a no-op." + fi + if [[ -z "${NEW_DB_PASSWORD:-}" ]]; then + log_fail "NEW_DB_PASSWORD not set." + exit 1 + fi + if [[ -z "${DATABASE_URL:-}" ]]; then + log_fail "DATABASE_URL not set." + exit 1 + fi + log_pass "Safety checks passed" +} + +check_current_connection() { + log_info "=== Pre-Rotation Connection Check ===" + if psql "$DATABASE_URL" -c "SELECT 1 AS pre_check" >/dev/null 2>&1; then + log_pass "Current credentials accept connections" + else + log_fail "Current credentials invalid — aborting." + exit 1 + fi + local c + c=$(psql "$DATABASE_URL" -At -c "SELECT count(*) FROM pg_stat_activity WHERE state='active';" 2>/dev/null || echo "0") + log_info "Active connections before rotation: $c" +} + +trigger_rotation() { + log_info "=== Triggering Rotation ===" + local admin_url="${STAGING_URL%/health}/admin/db/rotate-credentials" + local code + code=$(curl -s -o /tmp/rot.json -w "%{http_code}" -X POST \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${ADMIN_TOKEN:-skip}" \ + -d "{\"password\":\"${NEW_DB_PASSWORD}\"}" \ + --max-time 15 "$admin_url" 2>/dev/null || echo "000") + if [[ "$code" == "200" ]] || [[ "$code" == "204" ]]; then + log_pass "Rotation endpoint returned $code" + elif [[ "$code" == "404" ]]; then + log_warn "Endpoint not found — testing new password directly." + local nu + nu=$(echo "$DATABASE_URL" | sed "s/:[^:@]*@/:${NEW_DB_PASSWORD}@/") + if psql "$nu" -c "SELECT 1 AS smoke" >/dev/null 2>&1; then + log_pass "New credentials valid (smoke test)" + else + log_fail "New credentials rejected" + fi + else + log_warn "Rotation endpoint returned $code" + fi +} + +verify_post() { + log_info "=== Post-Rotation Verification ===" + sleep 2 + local code + code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 "$STAGING_URL" 2>/dev/null || echo "000") + if [[ "$code" == "200" ]]; then + log_pass "Health endpoint OK after rotation" + else + log_fail "Health returned $code after rotation" + fi +} + +run_all() { + log_info "=== DB Password Rotation Drill ===" + log_info "Start: $(date -u) Staging: $STAGING_URL RotationEnabled: $DB_ROTATION_ENABLED" + safety_check && echo "" && check_current_connection && echo "" + trigger_rotation && echo "" && verify_post && echo "" + local e=$(($(date +%s) - START_EPOCH)) + log_info "Drill completed in ${e}s" + if [[ "$FAILURES" -eq 0 ]]; then log_pass "All checks passed."; else log_fail "$FAILURES check(s) failed."; fi + echo "=== Drill Complete ===" + exit "$FAILURES" +} + +show_help() { + cat < { + const actualPg = jest.requireActual('pg'); + return { ...actualPg, Pool: jest.fn() }; +}); + +const MockedPool = Pool as jest.MockedClass; + +function mockPoolQuery() { + return { + query: jest.fn().mockResolvedValue({ rows: [{ '?column?': 1 }] }), + end: jest.fn().mockResolvedValue(undefined), + }; +} + +describe('rotatePoolCredentials', () => { + let mockNewPool: ReturnType; + let incrementSpy: jest.SpyInstance; + + beforeEach(() => { + jest.clearAllMocks(); + clearRotationListeners(); + mockNewPool = mockPoolQuery(); + MockedPool.mockImplementation(() => mockNewPool as unknown as Pool); + incrementSpy = jest.spyOn(globalMetrics, 'incrementCounter').mockImplementation(() => {}); + }); + + afterEach(() => { incrementSpy.mockRestore(); }); + afterAll(async () => { await closeAllPools(); }); + + it('is a no-op when DB_ROTATION_ENABLED is not "true"', async () => { + delete process.env.DB_ROTATION_ENABLED; + await rotatePoolCredentials({ password: 'newpass' }); + expect(MockedPool).not.toHaveBeenCalled(); + }); + + it('creates new pool and smoke-tests on success', async () => { + process.env.DB_ROTATION_ENABLED = 'true'; + await rotatePoolCredentials({ password: 'newpass123' }); + expect(MockedPool).toHaveBeenCalledTimes(1); + expect(mockNewPool.query).toHaveBeenCalledWith('SELECT 1'); + expect(incrementSpy).toHaveBeenCalledWith( + 'db.pool.credential_rotation', undefined, 1, expect.any(String), + ); + delete process.env.DB_ROTATION_ENABLED; + }); + + it('emits failure counter and throws on bad credentials', async () => { + process.env.DB_ROTATION_ENABLED = 'true'; + mockNewPool.query.mockRejectedValueOnce(new Error('password auth failed')); + await expect(rotatePoolCredentials({ password: 'bad' })).rejects.toThrow(/Credential rotation failed/); + expect(incrementSpy).toHaveBeenCalledWith( + 'db.pool.credential_rotation_failed', undefined, 1, expect.any(String), + ); + expect(mockNewPool.end).toHaveBeenCalled(); + delete process.env.DB_ROTATION_ENABLED; + }); + + it('notifies listeners on success', async () => { + process.env.DB_ROTATION_ENABLED = 'true'; + const cb = jest.fn(); + onCredentialsRotated(cb); + await rotatePoolCredentials({ password: 'ok' }); + expect(cb).toHaveBeenCalledWith('rotated', { timestamp: expect.any(String) }); + delete process.env.DB_ROTATION_ENABLED; + }); + + it('notifies listeners on failure', async () => { + process.env.DB_ROTATION_ENABLED = 'true'; + const cb = jest.fn(); + onCredentialsRotated(cb); + mockNewPool.query.mockRejectedValueOnce(new Error('bad pw')); + await expect(rotatePoolCredentials({ password: 'bad' })).rejects.toThrow(); + expect(cb).toHaveBeenCalledWith('failed', { timestamp: expect.any(String), error: 'bad pw' }); + delete process.env.DB_ROTATION_ENABLED; + }); + + it('swallows listener errors', async () => { + process.env.DB_ROTATION_ENABLED = 'true'; + onCredentialsRotated(() => { throw new Error('boom'); }); + await rotatePoolCredentials({ password: 'ok' }); + expect(incrementSpy).toHaveBeenCalledWith( + 'db.pool.credential_rotation', undefined, 1, expect.any(String), + ); + delete process.env.DB_ROTATION_ENABLED; + }); + + it('uses config values over env vars', async () => { + process.env.DB_ROTATION_ENABLED = 'true'; + process.env.DB_HOST = 'env-host'; + await rotatePoolCredentials({ host: 'cfg-host', password: 'pw' }); + expect(MockedPool.mock.calls[0]?.[0]).toMatchObject({ host: 'cfg-host' }); + delete process.env.DB_ROTATION_ENABLED; + delete process.env.DB_HOST; + }); + + it('clearRotationListeners removes all callbacks', async () => { + process.env.DB_ROTATION_ENABLED = 'true'; + const cb = jest.fn(); + onCredentialsRotated(cb); + clearRotationListeners(); + await rotatePoolCredentials({ password: 'pw' }); + expect(cb).not.toHaveBeenCalled(); + delete process.env.DB_ROTATION_ENABLED; + }); +}); diff --git a/src/db/pool.ts b/src/db/pool.ts index 08c454e8..8456e8cd 100644 --- a/src/db/pool.ts +++ b/src/db/pool.ts @@ -134,6 +134,115 @@ export async function readQuery(sql, params); } +// --------------------------------------------------------------------------- +// Credential rotation +// --------------------------------------------------------------------------- + +/** + * Configuration for database credential rotation. + */ +export interface PoolCredentialConfig { + host?: string; + port?: number; + database?: string; + user?: string; + password?: string; +} + +/** + * Callback invoked after credential rotation completes (success or failure). + */ +export type CredentialsRotatedCallback = ( + event: 'rotated' | 'failed', + details: { timestamp: string; error?: string }, +) => void; + +const rotationListeners: CredentialsRotatedCallback[] = []; + +/** + * Register a callback to be notified of credential rotation events. + */ +export function onCredentialsRotated(cb: CredentialsRotatedCallback): void { + rotationListeners.push(cb); +} + +/** Remove all credential rotation listeners (useful in test teardown). */ +export function clearRotationListeners(): void { + rotationListeners.length = 0; +} + +function notifyListeners( + event: 'rotated' | 'failed', + details: { error?: string }, +): void { + const payload = { timestamp: new Date().toISOString(), ...details }; + for (const cb of rotationListeners) { + try { cb(event, payload); } catch { /* swallow */ } + } +} + +/** + * Rotate the primary database pool credentials at runtime. + * + * Creates a fresh pool with the supplied credentials, smoke-tests it, + * then swaps the exported `pool` reference. The old pool is drained + * gracefully (5 s delay) before closing. + * + * Guardrails: + * - Gated on `DB_ROTATION_ENABLED=true` (no-op otherwise). + * - Counter `db.pool.credential_rotation` incremented on success. + * - Counter `db.pool.credential_rotation_failed` + listener on failure. + * + * @example + * await rotatePoolCredentials({ password: process.env.NEW_DB_PASSWORD }); + */ +export async function rotatePoolCredentials( + config: PoolCredentialConfig, +): Promise { + if (process.env.DB_ROTATION_ENABLED !== 'true') { + return; + } + + const newPool = new Pool({ + host: config.host ?? process.env.DB_HOST ?? 'localhost', + port: config.port ?? Number(process.env.DB_PORT ?? 5432), + database: config.database ?? process.env.DB_NAME ?? 'revora', + user: config.user ?? process.env.DB_USER ?? 'postgres', + password: config.password ?? process.env.DB_PASSWORD ?? '', + max: 10, + idleTimeoutMillis: 30_000, + connectionTimeoutMillis: 2_000, + }); + + // Smoke-test: verify new credentials with a lightweight query + try { + await newPool.query('SELECT 1'); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + globalMetrics.incrementCounter( + 'db.pool.credential_rotation_failed', undefined, 1, + 'Number of failed credential rotation attempts', + ); + notifyListeners('failed', { error: message }); + await newPool.end().catch(() => {}); + throw new Error(`Credential rotation failed: ${message}`); + } + + // Swap pools — keep old alive for in-flight queries + const oldPool = pool; + (pool as Pool) = newPool; + + setTimeout(() => { + if (typeof oldPool.end === 'function') oldPool.end().catch(() => {}); + }, 5_000); + + globalMetrics.incrementCounter( + 'db.pool.credential_rotation', undefined, 1, + 'Number of successful credential rotation events', + ); + notifyListeners('rotated', {}); +} + // --------------------------------------------------------------------------- // Graceful shutdown helper // ---------------------------------------------------------------------------