diff --git a/.github/workflows/recover-coolify-production-ref.yml b/.github/workflows/recover-coolify-production-ref.yml new file mode 100644 index 0000000..562241f --- /dev/null +++ b/.github/workflows/recover-coolify-production-ref.yml @@ -0,0 +1,71 @@ +name: Recover Coolify production ref + +on: + workflow_dispatch: + inputs: + expected_main_sha: + description: Exact current main SHA already proved by native deployment + required: true + type: string + +permissions: + contents: read + +concurrency: + group: deploy-coolify + cancel-in-progress: false + +jobs: + recover: + name: Recover protected production ref without deployment + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - name: Validate disabled recovery request + id: validation + env: + COOLIFY_CD_ENABLED: ${{ vars.COOLIFY_CD_ENABLED }} + EXPECTED_MAIN_SHA: ${{ inputs.expected_main_sha }} + GH_TOKEN: ${{ github.token }} + run: | + set -Eeuo pipefail + test "$GITHUB_REPOSITORY" = "Codevena/AgoraHub" + test "$GITHUB_REF" = "refs/heads/main" + test "$COOLIFY_CD_ENABLED" = "false" + [[ "$EXPECTED_MAIN_SHA" =~ ^[0-9a-f]{40}$ ]] + CURRENT_MAIN_SHA=$(gh api \ + "repos/$GITHUB_REPOSITORY/git/ref/heads/main" \ + --jq '.object.sha') + test "$CURRENT_MAIN_SHA" = "$EXPECTED_MAIN_SHA" + printf 'expected_main_sha=%s\n' "$EXPECTED_MAIN_SHA" >> "$GITHUB_OUTPUT" + + - name: Checkout verified main revision + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + ref: ${{ steps.validation.outputs.expected_main_sha }} + persist-credentials: false + + - name: Setup pinned Node + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: 22.19.0 + + - name: Mint dedicated production-ref token + id: deploy-ref-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.DEPLOY_REF_APP_ID }} + private-key: ${{ secrets.DEPLOY_REF_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: AgoraHub + permission-contents: write + + - name: Recover exact production ref without deployment + env: + COOLIFY_CD_ENABLED: ${{ vars.COOLIFY_CD_ENABLED }} + EXPECTED_MAIN_SHA: ${{ steps.validation.outputs.expected_main_sha }} + GITHUB_TOKEN: ${{ steps.deploy-ref-token.outputs.token }} + run: >- + node --no-warnings --experimental-strip-types + scripts/recover-coolify-production-ref.ts diff --git a/docs/superpowers/plans/2026-08-11-ci-gated-coolify-deploy.md b/docs/superpowers/plans/2026-08-11-ci-gated-coolify-deploy.md index 3c3f416..d9dda52 100644 --- a/docs/superpowers/plans/2026-08-11-ci-gated-coolify-deploy.md +++ b/docs/superpowers/plans/2026-08-11-ci-gated-coolify-deploy.md @@ -8,7 +8,7 @@ **Tech Stack:** GitHub Actions, TypeScript 5.9, Vitest 4, Node 22 native `fetch`, Coolify 4.0.0-beta.470 API, Cloudflare Access, `gh`, `actionlint`. -## Binding post-checkpoint correction (2026-08-11) +## As-built security correction and execution status (2026-08-11) Task 2's security checkpoint found that a final mutable-ref read did not bind Coolify's queued commit. Current Coolify source confirms `POST /api/v1/deploy` @@ -16,9 +16,11 @@ has no commit parameter, while the HMAC-verified manual GitHub webhook passes payload `after` as the explicit queue `commit`. The implementation therefore uses the signed manual webhook and a separate read-only API token. -This section supersedes conflicting names, request shapes, credential scopes, -Cloudflare paths, embedded code examples, and mutation counts later in this -plan: +Tasks 1–3 below are the historical TDD and integration record. Their embedded +pre-correction code snippets must not be copied or executed; the merged files on +`main` are the implementation authority. Task 3A is the post-review recovery +correction that must be merged before Task 4 begins. Tasks 4–6 are the active +merge/cutover runbook and incorporate the corrected contract directly: - fixed webhook: `https://panel.codevena.dev/source/github/events/manual`; - variables: add `COOLIFY_WEBHOOK_URL`; @@ -34,22 +36,30 @@ plan: step is guarded by exact `true`; - Task 3 runs the 88 one-at-a-time mutations enumerated in its corrected list and manifest below. +- Task 3A adds a manual, non-deploying protected-ref recovery workflow that is + usable only while CD is exactly disabled. ## Global Constraints - The frozen tag `v0.2.0-alpha.0` remains on `63c4585114a01211b1767daed2ba99a1e78bad40`. - No direct deploy `workflow_dispatch`; a manual redeploy starts `CI` on `main` and must pass first. +- The separate ref-recovery workflow is manual-only, shares the deploy + concurrency group, requires CD exactly disabled, and never calls Coolify. - Coolify application UUID is `z5eij4n8c4ubvxsmtpb507p2`; deploy branch is `coolify-production`. - Enabled mode fails closed unless repository, API base URL, application UUID, and deploy branch exactly match the approved AgoraHub constants. - A dedicated GitHub App is the sole ruleset bypass actor allowed to update `coolify-production`; the workflow's ordinary `GITHUB_TOKEN` remains read-only. - Permanent CD mode is `COOLIFY_CD_ENABLED=true`; only the controlled cutover uses explicit `false`. -- Coolify credentials carry exactly `read` and `deploy`; never `write` or `root`. -- Every Coolify request uses the bearer token plus both Cloudflare Access service-token headers. +- Coolify's API token carries exactly `read`; never `deploy`, `write`, or `root`. +- Application/deployment GETs use the read-only bearer plus both Cloudflare + Access headers. The webhook POST uses both Access headers plus the HMAC + signature and never receives a Coolify bearer. - Missing/invalid mode always fails. When enabled, every missing variable or secret fails. - Deployments are serialized with `cancel-in-progress: false` until Coolify and the production smoke finish. -- The deployment queue must contain exactly one AgoraHub deployment; terminal status must be `finished` and `commit` must equal the successful CI `head_sha`. +- The signed webhook acknowledgement must contain exactly one successful + AgoraHub deployment; terminal status must be `finished` and `commit` must + equal the successful CI `head_sha`. - One injected monotonic deadline bounds the whole enabled controller to 20 minutes; each request timeout and poll sleep is capped to the remaining time. - Never write token values to Git, Brain, terminal history, review artifacts, or logs. @@ -63,6 +73,13 @@ plan: - Create `scripts/deploy-coolify.test.ts`: behavioral tests using injected `fetch` and `sleep`; no live GitHub/Coolify calls. - Modify `scripts/ci-workflow.test.ts`: structural contracts for the small GitHub Actions orchestrator. - Modify `.github/workflows/deploy.yml`: trusted-event admission, pinned checkout, controller invocation, conditional production smoke. +- Create `.github/workflows/recover-coolify-production-ref.yml`: recovery-only + non-force create/fast-forward of the protected production ref with the + dedicated App while CD is disabled. +- Create `scripts/recover-coolify-production-ref.ts`: dependency-free, + directly executable GitHub ref state machine with injected `fetch`. +- Create `scripts/recover-coolify-production-ref.test.ts`: behavioral GitHub API + request/state tests for the recovery controller. - Update `docs/superpowers/specs/2026-08-11-ci-gated-coolify-deploy-design.md` before implementation with the Round-1 security corrections: dedicated ref-writer identity/ruleset, fixed-target validation, absolute deadline, and full rollback after the branch-authority switch. - Update Brain notes after the live cutover, outside the Git repository. @@ -1133,6 +1150,408 @@ Do not push. Report the commit SHA and stop at the explicit push gate in Task 4. --- +### Rejected Task 3A draft — source-only workflow contracts (do not execute) + +The steps in this subsection are retained only as review history. The execution +review correctly rejected them because grepping YAML does not prove recovery +behavior. Execute only the binding Task 3A correction below. + +**Files:** +- Create: `.github/workflows/recover-coolify-production-ref.yml` +- Modify: `scripts/ci-workflow.test.ts` +- Modify: `docs/superpowers/plans/2026-08-11-ci-gated-coolify-deploy.md` + +**Interfaces:** +- Consumes: `COOLIFY_CD_ENABLED`, `DEPLOY_REF_APP_ID`, + `DEPLOY_REF_APP_PRIVATE_KEY`, required dispatch input `expected_main_sha`, and + the existing `deploy-coolify` concurrency group. +- Produces: a manually dispatched, non-deploying App-authenticated create or + non-force update of `refs/heads/coolify-production` to the exact current + `main` SHA. + +- [ ] **Step 1: Add failing recovery-workflow contract tests** + +Add `existsSync` to the `node:fs` import and append this suite to +`scripts/ci-workflow.test.ts`. The empty-string fallback ensures the first RED +is an assertion about the missing workflow rather than a file-read exception: + +```ts +describe("Production-ref recovery workflow", () => { + const recoveryUrl = new URL( + "../.github/workflows/recover-coolify-production-ref.yml", + import.meta.url + ); + const workflow = existsSync(recoveryUrl) + ? readFileSync(recoveryUrl, "utf8") + : ""; + + it("is a manual-only ref operation serialized with deployments", () => { + expect(workflow).toContain("workflow_dispatch:"); + expect(workflow).toContain("expected_main_sha:"); + expect(workflow).not.toContain("workflow_run:"); + expect(workflow).toContain("group: deploy-coolify"); + expect(workflow).toContain("cancel-in-progress: false"); + expect(workflow).toContain("contents: read"); + }); + + it("fails closed on repository, dispatch ref, interlock, and current main before secrets", () => { + expect(workflow).toContain('test "$GITHUB_REPOSITORY" = "Codevena/AgoraHub"'); + expect(workflow).toContain('test "$GITHUB_REF" = "refs/heads/main"'); + expect(workflow).toContain('test "$COOLIFY_CD_ENABLED" = "false"'); + expect(workflow).toContain('[[ "$EXPECTED_MAIN_SHA" =~ ^[0-9a-f]{40}$ ]]'); + expect(workflow).toContain("git/ref/heads/main"); + expect(workflow).toContain('test "$CURRENT_MAIN_SHA" = "$EXPECTED_MAIN_SHA"'); + const validation = workflow.slice( + workflow.indexOf("- name: Validate disabled recovery request"), + workflow.indexOf("- name: Mint dedicated production-ref token") + ); + expect(validation).not.toContain("secrets."); + }); + + it("uses only the dedicated App for a verified non-force production-ref change", () => { + expect(workflow).toContain( + "actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1" + ); + expect(workflow).toContain("DEPLOY_REF_APP_ID"); + expect(workflow).toContain("DEPLOY_REF_APP_PRIVATE_KEY"); + expect(workflow).toContain("permission-contents: write"); + expect(workflow).toContain("git/refs/heads/coolify-production"); + expect(workflow).toContain("git/ref/heads/coolify-production"); + expect(workflow).toContain("force: false"); + expect(workflow).toContain('test "$FINAL_REF_SHA" = "$EXPECTED_MAIN_SHA"'); + expect(workflow).not.toContain("force: true"); + }); + + it("cannot call Coolify or read deployment credentials", () => { + for (const forbidden of [ + "deploy-coolify.ts", + "COOLIFY_API_BASE_URL", + "COOLIFY_WEBHOOK_URL", + "COOLIFY_READ_TOKEN", + "COOLIFY_WEBHOOK_SECRET", + "CF_ACCESS_CLIENT_ID", + "CF_ACCESS_CLIENT_SECRET", + "release:smoke", + ]) expect(workflow).not.toContain(forbidden); + }); +}); +``` + +- [ ] **Step 2: Run the focused test and verify RED** + +Run: + +```bash +pnpm exec vitest run scripts/ci-workflow.test.ts --exclude '.worktrees/**' +``` + +Expected: the four new tests fail on missing recovery workflow content; the +existing deploy-workflow tests remain green. + +- [ ] **Step 3: Add the minimal recovery-only workflow** + +Create `.github/workflows/recover-coolify-production-ref.yml` with exactly this +contract: + +```yaml +name: Recover Coolify production ref + +on: + workflow_dispatch: + inputs: + expected_main_sha: + description: Exact current main SHA already proved by native deployment + required: true + type: string + +permissions: + contents: read + +concurrency: + group: deploy-coolify + cancel-in-progress: false + +jobs: + recover: + name: Recover protected production ref without deployment + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - name: Validate disabled recovery request + id: validation + env: + COOLIFY_CD_ENABLED: ${{ vars.COOLIFY_CD_ENABLED }} + EXPECTED_MAIN_SHA: ${{ inputs.expected_main_sha }} + GH_TOKEN: ${{ github.token }} + run: | + set -Eeuo pipefail + test "$GITHUB_REPOSITORY" = "Codevena/AgoraHub" + test "$GITHUB_REF" = "refs/heads/main" + test "$COOLIFY_CD_ENABLED" = "false" + [[ "$EXPECTED_MAIN_SHA" =~ ^[0-9a-f]{40}$ ]] + CURRENT_MAIN_SHA=$(gh api "repos/$GITHUB_REPOSITORY/git/ref/heads/main" --jq '.object.sha') + test "$CURRENT_MAIN_SHA" = "$EXPECTED_MAIN_SHA" + printf 'expected_main_sha=%s\n' "$EXPECTED_MAIN_SHA" >> "$GITHUB_OUTPUT" + + - name: Mint dedicated production-ref token + id: deploy-ref-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.DEPLOY_REF_APP_ID }} + private-key: ${{ secrets.DEPLOY_REF_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: AgoraHub + permission-contents: write + + - name: Create or fast-forward protected production ref + env: + EXPECTED_MAIN_SHA: ${{ steps.validation.outputs.expected_main_sha }} + GH_TOKEN: ${{ steps.deploy-ref-token.outputs.token }} + run: | + set -Eeuo pipefail + if ! jq -nc --arg sha "$EXPECTED_MAIN_SHA" \ + '{sha: $sha, force: false}' | \ + gh api --method PATCH \ + "repos/$GITHUB_REPOSITORY/git/refs/heads/coolify-production" \ + --input - >/dev/null; then + jq -nc --arg sha "$EXPECTED_MAIN_SHA" \ + '{ref: "refs/heads/coolify-production", sha: $sha}' | \ + gh api --method POST "repos/$GITHUB_REPOSITORY/git/refs" \ + --input - >/dev/null + fi + FINAL_REF_SHA=$(gh api \ + "repos/$GITHUB_REPOSITORY/git/ref/heads/coolify-production" \ + --jq '.object.sha') + test "$FINAL_REF_SHA" = "$EXPECTED_MAIN_SHA" +``` + +The PATCH never forces. Its fallback POST succeeds only when the ref is absent; +if PATCH failed for non-fast-forward, ruleset, authentication, or transport +reasons while the ref still exists, POST also fails closed. + +- [ ] **Step 4: Run focused tests and workflow syntax checks GREEN** + +Run: + +```bash +pnpm exec vitest run scripts/ci-workflow.test.ts --exclude '.worktrees/**' +actionlint .github/workflows/recover-coolify-production-ref.yml .github/workflows/deploy.yml +``` + +Expected: all workflow tests pass and `actionlint` emits no findings. + +- [ ] **Step 5: Mutation-check the recovery guards in a disposable copy** + +In a `mktemp -d` copy, apply these one-at-a-time mutations and require the named +focused test to fail before restoring the copy: + +1. replace `workflow_dispatch` with `workflow_run`; +2. replace the disabled interlock check value `false` with `true`; +3. remove the current-main equality check; +4. replace `force: false` with `force: true`; +5. remove the pinned App-token action line; +6. add `COOLIFY_READ_TOKEN` to the workflow. + +Expected: every mutation makes at least one of the four new tests fail; the +original worktree remains unchanged. + +- [ ] **Step 6: Checkpoint-review and commit the recovery workflow** + +Run `git diff --check`, the fast inline-diff checkpoint reviewer, and fix every +CRITICAL/WARN. Then commit only the workflow, its contract tests, and this plan: + +```bash +git add .github/workflows/recover-coolify-production-ref.yml scripts/ci-workflow.test.ts docs/superpowers/plans/2026-08-11-ci-gated-coolify-deploy.md +git commit -m "ci: add protected ref recovery path" +``` + +--- + +### Task 3A: Implement executable protected-ref recovery + +**Files:** +- Create: `scripts/recover-coolify-production-ref.test.ts` +- Create: `scripts/recover-coolify-production-ref.ts` +- Create: `.github/workflows/recover-coolify-production-ref.yml` +- Modify: `scripts/ci-workflow.test.ts` + +**Interfaces:** +- `readRecoveryConfig(env?: Record): RecoveryConfig` +- `recoverProductionRef(config: RecoveryConfig, dependencies?: { fetchImpl?: + FetchLike }): Promise<{ status: "recovered"; previousSha: string | null }>` +- CLI environment: exact-false `COOLIFY_CD_ENABLED`, `EXPECTED_MAIN_SHA`, exact + `GITHUB_REPOSITORY=Codevena/AgoraHub`, and short-lived App `GITHUB_TOKEN`. + +- [x] **Step 1: Write behavioral tests before the controller exists** + +Create `scripts/recover-coolify-production-ref.test.ts`. Load the not-yet-created +module with dynamic `import()` so RED is caused by the missing recovery feature. +Use real `Response` objects and an injected fetch queue that records URL/method, +headers, and body. Use the literal SHA fixtures +`aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`, +`bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb`, and +`cccccccccccccccccccccccccccccccccccccccc`. + +The tests must assert these behaviors independently: + +1. config accepts only CD exactly `false`, exact repository, lowercase 40-hex + expected SHA, and a non-empty token; +2. an existing old production ref emits GET main → GET production → plural-ref + PATCH with body `{ sha: expected, force: false }` → final GET, and returns the + old SHA; +3. exact 404 on the production-ref GET emits POST `/git/refs` with body + `{ ref: "refs/heads/coolify-production", sha: expected }`, then final GET, + and returns `previousSha: null`; +4. an already-current production ref performs no mutation and still performs + the final GET; +5. a changed current-main response rejects before the production-ref read; +6. a 403 production-ref read rejects without attempting POST creation; +7. a successful JSON `null` production-ref read rejects rather than being + confused with the exact-404 sentinel; +8. a 422 PATCH rejects without attempting POST creation; +9. a mismatched final ref rejects after an otherwise successful PATCH. + +For the update test, the central assertions are: + +```ts +expect(calls.map(({ url }) => url)).toEqual([ + "https://api.github.com/repos/Codevena/AgoraHub/git/ref/heads/main", + "https://api.github.com/repos/Codevena/AgoraHub/git/ref/heads/coolify-production", + "https://api.github.com/repos/Codevena/AgoraHub/git/refs/heads/coolify-production", + "https://api.github.com/repos/Codevena/AgoraHub/git/ref/heads/coolify-production", +]); +expect(calls[2].init.method).toBe("PATCH"); +expect(JSON.parse(String(calls[2].init.body))).toEqual({ + sha: EXPECTED_SHA, + force: false, +}); +``` + +- [x] **Step 2: Run focused tests and verify RED** + +```bash +pnpm exec vitest run scripts/recover-coolify-production-ref.test.ts --exclude '.worktrees/**' +``` + +Expected: FAIL because `scripts/recover-coolify-production-ref.ts` does not yet +exist. This missing-module failure is the intended missing-feature RED, not a +fixture or assertion typo. + +- [x] **Step 3: Implement the minimal dependency-free controller** + +Create `scripts/recover-coolify-production-ref.ts` using Node's built-in +`fetch`, `Response`, and `pathToFileURL` only. Validate the four CLI inputs before +networking. Send GitHub's JSON accept header, bearer App token, API version +`2022-11-28`, manual redirect handling, and JSON content type. + +The state machine is exact: + +```ts +const main = await requestRef(mainReadUrl, "GitHub main ref"); +if (main.sha !== config.expectedMainSha) { + throw new Error("GitHub main ref changed during recovery"); +} + +const production = await requestOptionalProductionRef(); +if (production === NOT_FOUND) { + await requestJson(createUrl, { + method: "POST", + body: JSON.stringify({ + ref: "refs/heads/coolify-production", + sha: config.expectedMainSha, + }), + }, "GitHub production ref creation"); +} else if (production.sha !== config.expectedMainSha) { + await requestJson(updateUrl, { + method: "PATCH", + body: JSON.stringify({ sha: config.expectedMainSha, force: false }), + }, "GitHub production ref update"); +} + +const finalRef = await requestRef(productionReadUrl, "GitHub final production ref"); +if (finalRef.sha !== config.expectedMainSha) { + throw new Error("GitHub final production ref does not match expected main SHA"); +} +``` + +Only the production-ref GET may translate exact HTTP 404 to the unique +`NOT_FOUND` symbol; a successful JSON `null` response and every other +non-2xx, redirect, invalid JSON/ref shape, or fetch failure throws with its +operation label. The CLI calls the exported functions and logs no credential or +response headers. It runs under pinned Node 22 with +`node --no-warnings --experimental-strip-types` and therefore uses no installed +dependency or experimental module-format warning. + +- [x] **Step 4: Run behavioral tests GREEN** + +```bash +pnpm exec vitest run scripts/recover-coolify-production-ref.test.ts --exclude '.worktrees/**' +``` + +Expected: all nine recovery-controller tests pass. + +- [x] **Step 5: Add minimal workflow wiring and source-boundary tests** + +Create `.github/workflows/recover-coolify-production-ref.yml` with manual-only +`expected_main_sha`, `contents: read`, the shared `deploy-coolify` concurrency +group, and a five-minute timeout. Its steps are ordered: + +1. secretless validation of repository, dispatch ref, CD exact false, lowercase + SHA, and current GitHub main through the ordinary read token; +2. exact-SHA checkout with `persist-credentials: false`; +3. pinned Node `22.19.0` setup; +4. pinned `actions/create-github-app-token` mint with App secrets and contents + write; +5. run `node --no-warnings --experimental-strip-types + scripts/recover-coolify-production-ref.ts` with only expected SHA, repository, + CD mode, and the short-lived App token. + +Append narrow source-boundary assertions to `scripts/ci-workflow.test.ts` for +manual-only trigger, shared non-canceling concurrency, validation step before the +first `secrets.` reference, pinned checkout/Node/App actions, controller command, +and absence of all Coolify/Cloudflare secret names and `release:smoke`. Core ref +behavior remains covered only by the executable controller tests. + +- [x] **Step 6: Run workflow and focused integration checks GREEN** + +```bash +pnpm exec vitest run scripts/recover-coolify-production-ref.test.ts scripts/ci-workflow.test.ts --exclude '.worktrees/**' +actionlint .github/workflows/recover-coolify-production-ref.yml .github/workflows/deploy.yml +node --no-warnings --experimental-strip-types scripts/recover-coolify-production-ref.ts +``` + +Expected: tests/actionlint pass. The direct CLI invocation fails closed before +networking with `COOLIFY_CD_ENABLED must be exactly false`; this proves the +executable entry point is active without making an external request. + +- [x] **Step 7: Mutation-check every recovery guard in a disposable copy** + +Mutate one at a time and run only the named focused test: + +1. accept CD `true` → config test fails; +2. remove current-main equality → drift test fails; +3. change update endpoint from plural `refs` to singular `ref` → update test + fails; +4. change `force: false` to `force: true` → update test fails; +5. treat every failed production GET as missing → 403/transport regression test + fails; +6. reuse parsed JSON `null` as the 404 sentinel → successful-null test fails; +7. fall back to POST after PATCH 422 → patch-failure test fails; +8. remove final-ref equality → final-mismatch test fails; +9. add `COOLIFY_READ_TOKEN` to the workflow → source-boundary test fails. + +Record mutation/result pairs in `.review/recovery-mutations.md` and prove the +original worktree diff is unchanged. + +- [x] **Step 8: Run checkpoint review and commit locally** + +Run focused checks, `git diff --check`, and a fast inline-diff review. Fix every +CRITICAL/WARN, rerun affected mutations and checks, then commit only the +controller, tests, workflow, plan, and spec. Do not push. + +--- + ### Task 4: Prepare the disabled cutover and obtain push authorization **Files:** @@ -1185,25 +1604,105 @@ branch. **Files:** - No repository file changes. -- External state: GitHub branch/App/ruleset/variables/secrets, Cloudflare Access application/service token, Coolify API token/application branch. +- External state: GitHub branch/App/ruleset/variables/secrets, Cloudflare Access + applications/service token, Coolify webhook secret/read-only API token, and + application branch. **Interfaces:** - Consumes: the final native merge SHA. -- Produces: one successful API deployment of that same SHA while native auto-deploy remains available as rollback. +- Produces: one successful HMAC-signed webhook deployment of that same SHA while + native auto-deploy remains available as rollback. - [ ] **Step 1: Create the pinned production branch at the live SHA** -Resolve the exact remote `main` SHA and require it to equal the live Coolify -deployment. Then create `refs/heads/coolify-production` at that SHA. Abort if the -branch already exists at a different SHA; never force before identifying why. +Start an explicit AgoraHub `main` change freeze that remains active through Task +6: no push, merge, tag-moving action, Dependabot merge, or manual `CI` dispatch. +Require no queued or in-progress `CI` run for `main`. Resolve the exact remote +`main` SHA and require it to equal the live Coolify deployment, running image, +and last finished deployment. Record, without payload contents, the GitHub App +delivery ID and timestamp of that known-good native `main` push; it is the +deterministic rollback delivery. Snapshot the current AgoraHub deployment UUIDs +and timestamps from `application_deployment_queues`. + +Then create `refs/heads/coolify-production` at that SHA. Abort if the branch +already exists at a different SHA; never force before identifying why. From +this point through Task 6, re-read remote `main` before every authority mutation. +Any drift or new eligible CI completion aborts the cutover. If `main` changed, +run the **changed-main recovery** defined after Step 5; if `main` is unchanged +but another invariant failed, run the same-SHA rollback transaction. Restart +from this step only after recording a new green, deployed baseline. - [ ] **Step 2: Create the dedicated production-ref writer and ruleset** -Create a private GitHub App named `AgoraHub CD Ref Writer` with only repository -`Contents: Read and write` plus mandatory metadata read, install it only on -`Codevena/AgoraHub`, and generate one private key. Transfer the App ID and private -key directly into GitHub secrets `DEPLOY_REF_APP_ID` and -`DEPLOY_REF_APP_PRIVATE_KEY`; never print or save the private key elsewhere. +Under the **Codevena organization**, create a private GitHub App named +`AgoraHub CD Ref Writer` with only repository `Contents: Read and write` plus +mandatory metadata read, and install it only on `Codevena/AgoraHub`. Do not +create it under a personal account. + +GitHub necessarily downloads a generated private key as a PEM. Create a +permission-restricted temporary directory on the local FileVault volume, choose +that directory as the browser download target, require exactly one regular +non-symlink PEM with mode `0600`, and validate its PEM envelope without printing +its contents. Feed the file to +`gh secret set DEPLOY_REF_APP_PRIVATE_KEY --repo Codevena/AgoraHub` through +stdin; feed the numeric App ID through stdin to `DEPLOY_REF_APP_ID`. Verify only +the two secret names with `gh secret list`, then immediately remove the PEM with +macOS `rm -P` and remove the empty temporary directory. Never place the private +key in a command argument, clipboard, shell history, log, repository, or Brain. + +Use this local transfer sequence, substituting no secret into the command line: + +```bash +( + set -Eeuo pipefail + CUTOVER_KEY_DIR=$(mktemp -d "${TMPDIR%/}/agorahub-cd-key.XXXXXX") + CUTOVER_KEY_FILE= + cleanup_cutover_key() { + local cleanup_file cleanup_status=0 + if [[ -n "${CUTOVER_KEY_DIR:-}" && -d "$CUTOVER_KEY_DIR" && ! -L "$CUTOVER_KEY_DIR" ]]; then + while IFS= read -r -d '' cleanup_file; do + chmod 600 "$cleanup_file" || cleanup_status=1 + rm -P -- "$cleanup_file" || cleanup_status=1 + done < <(find "$CUTOVER_KEY_DIR" -maxdepth 1 -type f -print0) + while IFS= read -r -d '' cleanup_file; do + rm -f -- "$cleanup_file" || cleanup_status=1 + done < <(find "$CUTOVER_KEY_DIR" -maxdepth 1 -type l -print0) + if rmdir "$CUTOVER_KEY_DIR"; then + CUTOVER_KEY_DIR= + else + cleanup_status=1 + fi + fi + CUTOVER_KEY_FILE= + unset DEPLOY_REF_APP_ID_VALUE + return "$cleanup_status" + } + trap cleanup_cutover_key EXIT + trap 'exit 129' HUP + trap 'exit 130' INT + trap 'exit 143' TERM + chmod 700 "$CUTOVER_KEY_DIR" + printf 'Choose this browser download directory: %s\n' "$CUTOVER_KEY_DIR" + IFS= read -r -p 'Press Return after the one GitHub App PEM has downloaded there: ' _ + test "$(find "$CUTOVER_KEY_DIR" -mindepth 1 -maxdepth 1 | wc -l | tr -d ' ')" = 1 + test "$(find "$CUTOVER_KEY_DIR" -maxdepth 1 -type f -name '*.private-key.pem' | wc -l | tr -d ' ')" = 1 + CUTOVER_KEY_FILE=$(find "$CUTOVER_KEY_DIR" -maxdepth 1 -type f -name '*.private-key.pem' -print -quit) + test -f "$CUTOVER_KEY_FILE" + test ! -L "$CUTOVER_KEY_FILE" + chmod 600 "$CUTOVER_KEY_FILE" + test "$(stat -f '%Lp' "$CUTOVER_KEY_FILE")" = 600 + head -n 1 "$CUTOVER_KEY_FILE" | grep -Eq '^-----BEGIN (RSA )?PRIVATE KEY-----$' + tail -n 1 "$CUTOVER_KEY_FILE" | grep -Eq '^-----END (RSA )?PRIVATE KEY-----$' + gh secret set DEPLOY_REF_APP_PRIVATE_KEY --repo Codevena/AgoraHub < "$CUTOVER_KEY_FILE" + IFS= read -rs -p 'Enter the numeric GitHub App ID: ' DEPLOY_REF_APP_ID_VALUE + printf '\n' + [[ "$DEPLOY_REF_APP_ID_VALUE" =~ ^[0-9]+$ ]] + printf '%s' "$DEPLOY_REF_APP_ID_VALUE" | gh secret set DEPLOY_REF_APP_ID --repo Codevena/AgoraHub + unset DEPLOY_REF_APP_ID_VALUE + cleanup_cutover_key + trap - EXIT HUP INT TERM +) +``` Create an active repository ruleset targeting only `refs/heads/coolify-production` with creation, update, deletion, and force-push @@ -1211,8 +1710,11 @@ restrictions. The App installation is the sole `always` bypass actor; do not add repository roles, users, teams, or another integration. Verify the ruleset JSON read-only: exact include ref, enforcement `active`, the four restrictions, and exactly one bypass actor whose integration/App ID equals `DEPLOY_REF_APP_ID`. -Also mint a short-lived App token through the pinned action and prove it can read -the ref; do not move the ref during this setup proof. +Do not mint a token ad hoc during setup. The normal deploy path mints only while +CD is enabled; the separate reviewed recovery workflow mints only while CD is +disabled and only to repair the protected ref after a proved drift recovery. +The successful Step-7 deploy run and controller ref reads remain the normal-path +token-mint/read proof. - [ ] **Step 3: Create the Cloudflare path-scoped Service Auth** @@ -1220,34 +1722,50 @@ In Cloudflare Zero Trust: 1. create service token `GitHub Actions - AgoraHub Coolify deploy` expiring 2027-08-11; -2. create two more-specific self-hosted Access applications, one for - `panel.codevena.dev/api/v1/deploy*` (queue and deployment polling) and one - exact path for +2. create three more-specific self-hosted Access applications: the exact manual + webhook path `panel.codevena.dev/source/github/events/manual`, the exact path + for `panel.codevena.dev/api/v1/applications/z5eij4n8c4ubvxsmtpb507p2` - (pre-queue branch verification); do not expose any broader `/api/v1/*` path; -3. add a Service Auth policy to both applications whose include rule is only + (pre-queue branch verification), and + `panel.codevena.dev/api/v1/deployments/*` (polling); do not expose + `/api/v1/deploy` or any broader `/api/v1/*` path; +3. add a Service Auth policy to all three applications whose include rule is only that service token; -4. verify an anonymous request to each application is denied by Cloudflare Access with an expected - `3xx`, `401`, or `403`; retain enough response metadata to prove the denial - came from Access and that no Coolify JSON reached the caller; -5. separately verify on both route families that the new service-token headers - reach Coolify and get Coolify's expected bearer-layer `401` without an - Authorization header. +4. verify an anonymous request to each application is denied by Cloudflare + Access with an expected `3xx`, `401`, or `403`; retain enough response + metadata to prove the denial came from Access and that no Coolify response + body reached the caller; +5. snapshot the AgoraHub deployment inventory, then POST the controller's exact + JSON shape (`ref`, matching `after`, `repository.full_name`, and empty + `commits`) with its exact content type and event/delivery headers but an + intentionally invalid 64-hex HMAC; require Coolify's exact failed + `Invalid signature.` acknowledgement and prove the deployment inventory did + not change; +6. separately verify on the application and deployment paths that the service- + token headers reach Coolify and receive the expected bearer-layer `401` + without an Authorization header; +7. POST to exact `/api/v1/deploy` with only the new service-token headers and no + Coolify bearer or request body; require an Access-originated denial and prove + that no Coolify response body or deployment reaches the caller. Do not expose either one-time value in tool output or terminal history. -- [ ] **Step 4: Create the least-privilege Coolify API token** +- [ ] **Step 4: Create separate least-privilege Coolify credentials** -In Coolify API Tokens, create `GitHub Actions - AgoraHub deploy` with exactly -`read` and `deploy`. Verify the UI lists those two permissions and neither -`write` nor `root`. +On the AgoraHub application, create a high-entropy +`manual_webhook_secret_github` value. In Coolify API Tokens, create +`GitHub Actions - AgoraHub read` with exactly `read`. Verify the UI lists `read` +and neither `deploy`, `write`, nor `root`. Transfer both values only through the +authenticated browser into GitHub's secret form; never place either value in a +shell argument, file, log, clipboard history, or Brain note. - [ ] **Step 5: Transfer API credentials directly into GitHub and set variables** Create these GitHub secrets through the authenticated GitHub UI, transferring the one-time browser-held values directly: -- `COOLIFY_API_TOKEN` +- `COOLIFY_READ_TOKEN` +- `COOLIFY_WEBHOOK_SECRET` - `CF_ACCESS_CLIENT_ID` - `CF_ACCESS_CLIENT_SECRET` @@ -1255,28 +1773,96 @@ Set variables: ```bash gh variable set COOLIFY_API_BASE_URL --repo Codevena/AgoraHub --body https://panel.codevena.dev/api/v1 +gh variable set COOLIFY_WEBHOOK_URL --repo Codevena/AgoraHub --body https://panel.codevena.dev/source/github/events/manual gh variable set COOLIFY_APP_UUID --repo Codevena/AgoraHub --body z5eij4n8c4ubvxsmtpb507p2 gh variable set COOLIFY_DEPLOY_BRANCH --repo Codevena/AgoraHub --body coolify-production ``` Verify only names/presence with `gh secret list` and exact non-secret values with -`gh variable list`. The final secret-name set for this path is the three API -secrets above plus `DEPLOY_REF_APP_ID` and `DEPLOY_REF_APP_PRIVATE_KEY`. +`gh variable list`. The final secret-name set for this path is the four +Coolify/Access secrets above plus `DEPLOY_REF_APP_ID` and +`DEPLOY_REF_APP_PRIVATE_KEY`. + +#### Changed-main recovery + +If remote `main` no longer equals the frozen SHA at any point, do not replay the +old delivery or require the obsolete SHA. Abort the cutover and: + +1. set `COOLIFY_CD_ENABLED=false`, restore the Coolify application branch to + `main`, and set native auto-deploy enabled; +2. wait until every pre-existing AgoraHub deployment and every `Deploy to + Coolify` workflow run is terminal; stop without redelivery if quiescence + cannot be established; +3. resolve the new current remote `main` SHA, require its `CI` run to be + successful, and locate the normal GitHub App `push` delivery whose `after` + field equals that SHA without recording its payload; +4. snapshot deployment UUID/timestamp and native-delivery inventories, redeliver + that current-SHA native delivery, and require exactly one new deployment UUID + through a 120-second quiet window: terminal `finished`, commit equal to the + new current `main`, healthy container with 0 restarts, DB-connected + `/api/health`, and fresh 11/11 release smoke; +5. record that current-SHA delivery ID and the resulting finished deployment as + the new rollback baseline; +6. inspect the production-ref ruleset and App installation state. If neither + exists yet, require read-only proof that no active ruleset targets the ref, + then use the authenticated human GitHub identity to create a missing ref or + non-force fast-forward the existing ref to the new baseline; re-read it and + require exact equality. If the dedicated App and active single-App ruleset + both exist, dispatch `Recover Coolify production ref` on `main` with + `expected_main_sha` equal to the new baseline, require that single recovery + run to succeed, and re-read the ref at the exact SHA. If installation/ruleset + state is partial or ambiguous, stop instead of disabling protection or using + force; +7. restart Step 1 with the new frozen SHA and recorded baseline. + +If the current-SHA native delivery is unavailable or cannot be redelivered, +stop and ask Markus for explicit authorization for a replacement `main` push; +do not invent or silently perform one. + +The recovery workflow never calls Coolify and is dispatched only after branch +`main`, native auto-deploy enabled, CD false, native-deployment proof, and full +quiescence are established. Its `deploy-coolify` concurrency group prevents it +from overlapping the normal deploy workflow. + +#### Same-SHA post-switch rollback transaction + +For every failure or invariant drift at or after the branch switch where remote +`main` still equals the frozen SHA, perform and verify this complete transaction +before diagnosis or retry: + +1. set `COOLIFY_CD_ENABLED=false`; +2. restore the AgoraHub Coolify application branch to `main`; +3. set native auto-deploy enabled; +4. wait until every pre-existing AgoraHub deployment and every `Deploy to + Coolify` workflow run is terminal; stop without redelivery if quiescence + cannot be established; +5. require remote `main` still equals the frozen SHA, snapshot deployment + UUID/timestamp and native-delivery inventories, then redeliver the recorded + known-good normal GitHub App `main` push webhook from Step 1; +6. through a 120-second quiet window, require exactly one new deployment UUID + for that redelivery, terminal `finished`, the frozen commit, healthy container + with 0 restarts, DB-connected `/api/health`, and a fresh 11/11 release smoke. + +If the recorded delivery cannot be redelivered, stop and ask Markus for explicit +authorization for a replacement `main` push; do not invent or silently perform +one. - [ ] **Step 6: Point Coolify at the pinned branch — authority cutover** Change only the AgoraHub application branch from `main` to `coolify-production`. Keep `is_auto_deploy_enabled=true`. Verify persisted DB/API state and confirm no deployment was queued because the branch creation preceded -this setting change. +this setting change. Immediately before the switch, require remote `main`, +`coolify-production`, the live image, and the last finished deployment to equal +the frozen SHA, with no queued/in-progress main CI. Any mismatch runs the full +changed-main recovery above if remote `main` changed; otherwise it runs the +same-SHA rollback proof. This branch switch is the moment native `main` pushes stop advancing the source Coolify watches. From this point until Task 6 completes, every failure path must -run and verify the full rollback transaction: set `COOLIFY_CD_ENABLED=false`, -restore the Coolify application branch to `main`, set -`is_auto_deploy_enabled=true`, and prove the next controlled `main` push/native -webhook path can deploy. Re-enabling auto-deploy without restoring `main` is not -a rollback. +run and verify the applicable changed-main recovery or same-SHA rollback above. +Re-enabling auto-deploy without restoring `main`, quiescing in-flight work, and +replaying the delivery for the current `main` SHA is not a verified rollback. - [ ] **Step 7: Enable CD and trigger the controlled proof** @@ -1287,6 +1873,15 @@ gh variable set COOLIFY_CD_ENABLED --repo Codevena/AgoraHub --body true gh workflow run CI --repo Codevena/AgoraHub --ref main ``` +Immediately before setting the interlock to `true`, re-require remote `main == +coolify-production ==` the frozen live SHA and no queued/in-progress eligible +CI run. Also require every earlier `Deploy to Coolify` workflow run to be +terminal. Snapshot the deployment UUID/timestamp inventory and GitHub App native +webhook delivery IDs, record the UTC cutoff, then enable CD and dispatch exactly +one CI run. Record the resulting controlled `CI` run ID and, once created, its +single `Deploy to Coolify` `workflow_run` ID; reject any other deploy-workflow +run created after the cutoff. + Watch the CI run, then its `Deploy to Coolify` workflow. Require: - CI success for the current `main` SHA; @@ -1295,13 +1890,23 @@ Watch the CI run, then its `Deploy to Coolify` workflow. Require: bypass actor, and the workflow's ref token actor is that App; - the controller's final pre-queue ref read equals the CI SHA and Coolify's live application record reports `git_branch=coolify-production`; -- exactly one new AgoraHub deployment with `is_api=true`; +- exactly one new AgoraHub deployment acknowledged by the signed manual webhook; - terminal `finished`, recorded commit equals the CI SHA; - deploy workflow production smoke reports 11/11; -- no paired native webhook deployment for that CI event. - -On failure, execute the full post-switch rollback from Step 6, verify native -`main` deployment authority, and only then diagnose or retry. +- the pinned App-token action succeeded and the controller's protected-ref reads + succeeded, proving the dedicated App token can read the ref; +- after a 120-second quiet window, the Coolify database contains exactly one new + AgoraHub deployment UUID since the cutoff, equal to the controlled UUID with + `status=finished`, the frozen commit, `is_webhook=true`, and `is_api=false`; +- the GitHub App native webhook delivery inventory gained no new delivery, so no + second native deployment occurred; +- before ending the freeze, every `Deploy to Coolify` workflow run other than + the recorded controlled run is terminal, and the controlled run itself is + terminal success. + +On failure, execute changed-main recovery if `main` drifted; otherwise execute +the same-SHA post-switch rollback from Step 6. Verify native `main` deployment +authority before diagnosis or retry. --- @@ -1326,9 +1931,11 @@ Coolify database/API that: - application remains `running:healthy`; - running image/last finished deployment commit equals the controlled CI SHA. -If any post-disable verification fails, run the same full rollback transaction: -CD false, Coolify branch `main`, native auto-deploy enabled, then a verified -native main-push deployment. +If any post-disable verification fails, run changed-main recovery if remote +`main` drifted; otherwise run the same-SHA rollback transaction. Both paths +restore CD false, Coolify branch `main`, native auto-deploy enabled, quiesce +pre-existing work, and prove exactly one native main-push deployment through the +quiet window. - [ ] **Step 2: Run the final independent audit** @@ -1337,6 +1944,11 @@ CI and deploy conclusions, production branch SHA, Coolify settings, deployment UUID/status/commit, container health/restarts, `/api/health`, and a fresh `BASE_URL=https://agorahub.dev pnpm release:smoke` (11/11). +Re-run the Step-7 inventory check after a fresh 120-second quiet window: exactly +the one controlled UUID may exist since the recorded cutoff, and the native +GitHub App delivery inventory must still have no new delivery. Keep the `main` +change freeze in force until this audit and the Brain update finish. + Also verify the original annotated tag still dereferences to the frozen release: ```bash @@ -1354,10 +1966,12 @@ Update the three Brain notes with: - `coolify-production` exact-SHA ref; - two-layer Cloudflare/Coolify authentication and secret names only; - final deployment UUID/SHA/status and 11/11 smoke; -- rollback transaction: set CD false, restore Coolify branch `main`, re-enable - native auto-deploy, then verify the native main-push path; +- rollback transactions: set CD false, restore Coolify branch `main`, re-enable + native auto-deploy, quiesce work, then replay and verify the delivery matching + current `main`; use a new baseline rather than the old delivery after drift; - Cloudflare service-token expiry/rotation date 2027-08-11; -- Coolify token annual rotation paired with the Cloudflare token; +- Coolify read-token and application HMAC-secret annual rotation paired with the + Cloudflare token; - no token values. Report point 3 complete only with the fresh evidence above. The PostgreSQL backup @@ -1391,5 +2005,33 @@ this CD cutover plan. ### Round 3 — PASS No findings. Embedded TypeScript compiled with 0 diagnostics, embedded workflow -passed `actionlint`, mutation JSON parsed as 29 groups/70 individual mutations, +passed `actionlint`, mutation JSON parsed as 29 groups/88 individual mutations, and `git diff --check` passed. + +### Protected cutover runbook correction — Round 1 FAIL + +| Finding | Applied minimal fix | Plan task | +|---|---|---| +| Organization App/key transfer was not executable | Require App creation under `Codevena`; transfer the permission-restricted downloaded PEM to GitHub Secrets through stdin, verify names only, then remove it immediately | 5.2 | +| Setup-time App-token proof required prematurely enabling CD | Defer token-mint/read proof to the enabled controlled workflow and its successful protected-ref reads | 5.2, 5.7 | +| `main` drift could cause paired native and manual deployments | Freeze `main`, reject active/eligible CI, re-check main/ref/live equality before every authority mutation, and roll back on drift | 5.1, 5.6, 5.7 | +| Rollback had no deterministic deployment trigger | Record a known-good native GitHub App delivery and prescribe its redelivery with exact deployment/health/smoke evidence | 5.1, 5 rollback | +| Invalid-signature probe could stop before HMAC verification | Use the controller's exact payload/headers with an intentionally invalid signature, require `Invalid signature.`, and prove zero deployment delta | 5.3 | +| `/api/v1/deploy` exclusion was not tested | Require an Access-originated denial for the new service token on that exact path and zero deployment delta | 5.3 | +| Single-deployment claim lacked inventory and observation window | Snapshot DB/GitHub delivery inventories, require exactly one controlled UUID, and re-check after a 120-second quiet window | 5.7, 6.2 | + +### Protected cutover runbook correction — Round 2 FAIL + +| Finding | Applied minimal fix | Plan task | +|---|---|---| +| Changed `main` made the old-SHA rollback impossible | Split changed-main recovery from same-SHA rollback; recover native authority, quiesce work, prove the new current SHA via its matching delivery, record a new baseline, and restart | 5.1, 5 rollback, 5.6, 5.7 | +| Older deploy-workflow runs could race the controlled proof | Require prior deploy runs terminal, record the controlled deploy run ID, reject other post-cutoff runs, and recheck terminal state before ending the freeze | 5.7 | +| Rollback redelivery could race in-flight deployment work | Disable CD, restore native authority, wait for deployment and workflow quiescence, then snapshot and redeliver through a quiet window | 5 rollback | +| PEM transfer could bypass guards or leave key material on interruption | Use an error-stopping subshell with validation and EXIT/signal cleanup traps, plus explicit secure deletion on success | 5.2 | + +### Protected cutover runbook correction — Round 3 FAIL; Markus chose fix both + +| Finding | Applied minimal fix | Plan task | +|---|---|---| +| Active ruleset had no App-token recovery path while CD was false | Add a manual recovery-only workflow that validates disabled CD/current `main`, uses the dedicated App for non-force ref create/update, shares deploy concurrency, and never calls Coolify; retain a separately verified human non-force path before App/ruleset installation | 3A, 5 recovery | +| Cleanup could not see a PEM before filename/cardinality validation | Make EXIT/signal cleanup scan the exact private temp directory, securely delete every regular non-symlink file, unlink symlink entries without following them, and remove the directory | 5.2 | diff --git a/docs/superpowers/specs/2026-08-11-ci-gated-coolify-deploy-design.md b/docs/superpowers/specs/2026-08-11-ci-gated-coolify-deploy-design.md index 21c7667..4ab6583 100644 --- a/docs/superpowers/specs/2026-08-11-ci-gated-coolify-deploy-design.md +++ b/docs/superpowers/specs/2026-08-11-ci-gated-coolify-deploy-design.md @@ -1,6 +1,6 @@ # CI-Gated Coolify Deployment Design -**Status:** Approved direction; amended by pre-implementation plan-gate findings +**Status:** Approved and implemented; protected production cutover pending **Date:** 2026-08-11 **Scope:** AgoraHub continuous deployment only @@ -20,9 +20,10 @@ database schema, or introduce the later scheduled MCP/A2A synthetic monitor. - GitHub `CI` runs on pushes and pull requests targeting `main`, plus manual dispatches. Its two jobs cover quality/database/browser verification and the production audit/secret scan. -- `.github/workflows/deploy.yml` listens for completed `CI` runs on `main`, but - treats missing secrets as a successful skip. No Coolify secrets or variables - currently exist in the GitHub repository. +- `.github/workflows/deploy.yml` listens for completed successful `CI` runs on + current `main`. The merged workflow requires the explicit repository variable + `COOLIFY_CD_ENABLED`; it is currently set to `false`, so every secret-bearing + step is skipped before credentials are read. - Coolify application `z5eij4n8c4ubvxsmtpb507p2` deploys `Codevena/AgoraHub:main` and has `is_auto_deploy_enabled = true`. - Coolify `4.0.0-beta.470` protects `POST /api/v1/deploy` with a Sanctum token @@ -36,7 +37,7 @@ database schema, or introduce the later scheduled MCP/A2A synthetic monitor. ## Chosen Architecture -### Binding post-checkpoint correction (2026-08-11) +### As-built security correction (2026-08-11) The Task-2 security review exposed a remaining race in the original API design: Coolify's current `POST /api/v1/deploy` accepts a resource UUID but no commit @@ -47,9 +48,9 @@ successful CI SHA in `after`; Coolify passes that value as the explicit `commit` argument to `queue_application_deployment`, which stores it in the deployment queue before dispatch. -This correction supersedes every later reference in this document to -`POST /api/v1/deploy`, a Coolify `deploy` token, `COOLIFY_API_TOKEN`, or four -fixed target values: +The correction below is incorporated throughout this document. The direct +`POST /api/v1/deploy` design, a Coolify token with `deploy`, and the old +`COOLIFY_API_TOKEN` name are rejected architecture, not cutover options: - the controller still pins and verifies `coolify-production`, then preflights the live Coolify application UUID, branch, GitHub repository, and an explicit @@ -98,7 +99,8 @@ actor. For each eligible successful CI run the workflow: 5. re-reads the final production ref and requires the exact verified SHA; 6. reads the Coolify application and requires its configured branch to remain `coolify-production`; -7. calls Coolify for the AgoraHub resource; +7. sends the exact-SHA push payload to Coolify's HMAC-verified manual GitHub + webhook and accepts exactly one safe success acknowledgement; 8. waits for the returned deployment UUID to reach a terminal state; 9. requires terminal state `finished` and Coolify's recorded `commit` to equal the verified SHA; @@ -115,7 +117,7 @@ flowchart LR B -->|failure| X[No deployment] B -->|success for current main SHA| C[coolify-production ref] C --> D[Cloudflare Access service auth] - D --> E[Coolify deploy + read token] + D --> E[Signed webhook + read-only status API] E --> F[Coolify builds pinned ref] F --> G[Commit/status verification] G --> H[11-check production smoke] @@ -153,19 +155,22 @@ boundary. ### Two-layer API protection -Cloudflare Access gets two path-scoped self-hosted applications: one for -`panel.codevena.dev/api/v1/deploy*`, covering queue and deployment polling, and -one exact path for -`panel.codevena.dev/api/v1/applications/z5eij4n8c4ubvxsmtpb507p2`, covering the -pre-queue branch check. Neither grants the service token broader `/api/v1/*` -access. Their only automation policy is Service Auth bound to a new token named +Cloudflare Access gets three path-scoped self-hosted applications: the exact +manual webhook path `panel.codevena.dev/source/github/events/manual`, the exact +application path +`panel.codevena.dev/api/v1/applications/z5eij4n8c4ubvxsmtpb507p2`, and +`panel.codevena.dev/api/v1/deployments/*` for status polling. None grants the +service token broader `/api/v1/*` access or exposes `/api/v1/deploy`. Their only +automation policy is Service Auth bound to a new token named `GitHub Actions - AgoraHub Coolify deploy`. The existing human login policy for the rest of `panel.codevena.dev` stays unchanged. -Coolify gets a new team-scoped API token named -`GitHub Actions - AgoraHub deploy`. It carries exactly `read` and `deploy`, not -`write` or `root`. `read` is required to poll the deployment record; `deploy` -is required to queue it. +The AgoraHub application gets a new high-entropy +`manual_webhook_secret_github`; the same one-time value is stored in GitHub as +`COOLIFY_WEBHOOK_SECRET`. Coolify also gets a separate team-scoped API token +named `GitHub Actions - AgoraHub read`. It carries exactly `read`, never +`deploy`, `write`, or `root`; queue authority belongs only to the +application-specific HMAC secret. The GitHub repository stores: @@ -173,19 +178,21 @@ The GitHub repository stores: |---|---|---| | Variable | `COOLIFY_CD_ENABLED` | `false` during cutover, then permanently `true` | | Variable | `COOLIFY_API_BASE_URL` | `https://panel.codevena.dev/api/v1` | +| Variable | `COOLIFY_WEBHOOK_URL` | `https://panel.codevena.dev/source/github/events/manual` | | Variable | `COOLIFY_APP_UUID` | `z5eij4n8c4ubvxsmtpb507p2` | | Variable | `COOLIFY_DEPLOY_BRANCH` | `coolify-production` | -| Secret | `COOLIFY_API_TOKEN` | new Coolify `read` + `deploy` token | +| Secret | `COOLIFY_READ_TOKEN` | new Coolify read-only token | +| Secret | `COOLIFY_WEBHOOK_SECRET` | application-specific manual-webhook HMAC secret | | Secret | `CF_ACCESS_CLIENT_ID` | new Access service-token client ID | | Secret | `CF_ACCESS_CLIENT_SECRET` | new Access service-token secret | | Secret | `DEPLOY_REF_APP_ID` | dedicated ref-writer GitHub App ID | | Secret | `DEPLOY_REF_APP_PRIVATE_KEY` | dedicated ref-writer private key | -Every Coolify request supplies all three authentication headers: - -- `Authorization: Bearer $COOLIFY_API_TOKEN` -- `CF-Access-Client-Id: $CF_ACCESS_CLIENT_ID` -- `CF-Access-Client-Secret: $CF_ACCESS_CLIENT_SECRET` +Every Coolify request supplies both Cloudflare Access headers. Application and +deployment GETs additionally supply +`Authorization: Bearer $COOLIFY_READ_TOKEN`; the webhook POST instead supplies +`X-GitHub-Event: push` and `X-Hub-Signature-256` computed with +`COOLIFY_WEBHOOK_SECRET`, and never receives the Coolify bearer token. `COOLIFY_CD_ENABLED` is the explicit cutover interlock. It accepts only `false` or `true`; missing or any other value fails. `false` produces an intentional, @@ -195,17 +202,18 @@ configuration errors. The workflow must never turn missing production credentials into a green skip. Before any enabled-mode network call, the controller also fails closed unless -the repository, Coolify API base URL, application UUID, and deploy branch equal -the four approved AgoraHub values in this specification. This prevents variable -drift from sending bearer/service credentials to another host or deploying a -different target. +the repository, Coolify API base URL, webhook URL, application UUID, and deploy +branch equal the five approved AgoraHub values in this specification. This +prevents variable drift from sending bearer/service credentials to another host +or deploying a different target. ## Deployment Result Handling -The queue request must return exactly one deployment for the configured AgoraHub -resource and expose a non-empty `deployment_uuid`. Any malformed response, -unexpected resource UUID, HTTP error, Cloudflare Access redirect, authentication -failure, or Coolify rate limit fails the workflow. +The signed webhook must return an array containing exactly one `success` +acknowledgement for the configured AgoraHub application and a safe non-empty +`deployment_uuid`. Any malformed response, unexpected application UUID, +non-success acknowledgement, HTTP error, Cloudflare Access redirect, +authentication failure, or Coolify rate limit fails the workflow. The enabled controller uses one injected monotonic absolute deadline of at most 20 minutes across preflight requests, queueing, polling, request timeouts, and @@ -246,38 +254,128 @@ The cutover preserves a working rollback path at every step: The merge SHA receives normal CI and becomes the final native deployment. The new deploy workflow records its explicit disabled state and does not call the API. -3. Verify that the final native deployment is healthy and then create - `coolify-production` at that exact green/deployed `main` SHA. -4. Create and install the dedicated ref-writer GitHub App, store its App ID and - private key directly in GitHub, and activate/audit the single-App production - ref ruleset. -5. Create the two path-scoped Cloudflare Access applications and service token. - Anonymous denial may correctly be `3xx`, `401`, or `403`; separately prove - the service token reaches Coolify and receives Coolify's bearer-layer `401`. -6. Create the Coolify `read` + `deploy` API token. -7. Store the three API credential secrets and the remaining API/app/branch - variables in GitHub. +3. After the final native deployment is healthy, freeze AgoraHub `main` through + the final audit: no pushes, merges, tag-moving actions, Dependabot merges, or + manual CI dispatches, and no queued/in-progress main CI or deploy workflow. + Record the current deployment inventory and the known-good native GitHub App + delivery ID, then create `coolify-production` at that exact green/deployed + `main` SHA. +4. Under the Codevena organization, create and install the dedicated ref-writer + GitHub App. Transfer its unavoidable downloaded private-key PEM to GitHub + Secrets through stdin from a permission-restricted temporary directory, + verify names only, remove the PEM immediately, and activate/audit the + single-App production ref ruleset. Do not mint an ad-hoc setup token: the + enabled deploy proof establishes the normal path, while the separately + reviewed disabled-mode recovery workflow exists only for proved drift. +5. Create the three path-scoped Cloudflare Access applications and service token. + Anonymous denial may correctly be `3xx`, `401`, or `403`. With the service + token, separately prove the exact manual-webhook request reaches HMAC + verification and returns `Invalid signature.` for an intentionally invalid + signature without changing deployment inventory; prove application/deployment + reads reach Coolify's bearer-layer `401`; and prove `/api/v1/deploy` remains + denied by Access with zero deployment delta. +6. Create the application-specific manual-webhook HMAC secret and the separate + Coolify read-only API token. +7. Store `COOLIFY_READ_TOKEN`, `COOLIFY_WEBHOOK_SECRET`, both Cloudflare Access + service-token values, and the remaining API/webhook/app/branch variables in + GitHub without exposing their values. 8. Point the Coolify application at `coolify-production` while leaving native auto-deploy enabled. Because the branch already points at the running SHA, this configuration change does not introduce new code or emit a branch push. This is the authority cutover: native `main` pushes no longer advance the branch Coolify watches. -9. Set `COOLIFY_CD_ENABLED=true`, then manually dispatch `CI` on `main`. Its - successful completion triggers CD. The production ref is already on the same - SHA, so the workflow performs no ref write and native branch auto-deploy does - not race the API call. Require the single API deployment, recorded commit, and - full production smoke to succeed. +9. Re-require remote `main`, `coolify-production`, the live image, and the last + finished deployment to equal the frozen SHA, with no eligible active CI. + Require every prior deploy workflow to be terminal. Snapshot deployment and + native GitHub App delivery inventories, set `COOLIFY_CD_ENABLED=true`, then + manually dispatch exactly one `CI` on `main`. Record that CI run and its + single resulting deploy-workflow run and reject any other post-cutoff deploy + run. The production ref is already on the same SHA, so the workflow performs + no ref write and native branch auto-deploy does not race the signed webhook. + Require the single acknowledged deployment, recorded commit, and full + production smoke to succeed; after a 120-second quiet window require exactly + that one new deployment UUID, no new native GitHub App delivery, and all + other deploy-workflow runs terminal. 10. Disable Coolify native auto-deploy for AgoraHub. 11. Verify through Coolify's persisted settings that auto-deploy is false, CD is - enabled, and the application is healthy with the expected SHA and no paired - native/API deployment for the controlled CI event. + enabled, and the application is healthy with the expected SHA and no second + native deployment for the controlled CI event. At or after step 8, re-enabling native auto-deploy alone is not a rollback because -Coolify no longer watches `main`. Every failure from that point uses one complete -transaction: set `COOLIFY_CD_ENABLED=false`, restore the Coolify application -branch to `main`, set native auto-deploy enabled, and verify the native main-push -deployment path before diagnosing or retrying. Failures before step 8 do not -change deployment authority. +Coolify no longer watches `main`. For a same-SHA failure, set +`COOLIFY_CD_ENABLED=false`, restore branch `main`, enable native auto-deploy, +wait for all pre-existing deployments and deploy workflows to become terminal, +snapshot both inventories, redeliver the recorded known-good delivery, and prove +its single new finished UUID through a 120-second quiet window plus app/database +health and 11/11 smoke. If `main` changed, never replay the obsolete delivery: +restore those three settings, wait for quiescence, require successful CI for the +new current SHA, locate and redeliver that SHA's normal GitHub App push delivery, +prove the same single-deployment evidence, record it as a new baseline, and +restart the cutover. If the matching delivery is unavailable, stop for explicit +authorization before any replacement push. The `main` change freeze ends only +after the final audit and Brain update. + +### Recovery-only production-ref authority + +The protected ref must remain recoverable while CD is disabled. A small +dependency-free `scripts/recover-coolify-production-ref.ts` controller owns the +GitHub ref state machine, and a separate +`.github/workflows/recover-coolify-production-ref.yml` workflow invokes it +without calling Coolify or weakening the production ruleset: + +- it has only `workflow_dispatch` with one required `expected_main_sha` input; +- it shares the `deploy-coolify` concurrency group with the normal deploy + workflow and never cancels an in-progress run; +- its ordinary workflow token has `contents: read` only; +- before any GitHub App secret is read, it requires the repository to be exactly + `Codevena/AgoraHub`, the dispatch ref to be `refs/heads/main`, + `COOLIFY_CD_ENABLED` to be exactly `false`, the input to be a lowercase + 40-character commit SHA, and GitHub's current `main` ref to equal that input; +- it checks out only that verified SHA without persisted credentials and uses + the repository's pinned Node 22 runtime without installing dependencies; +- only after those checks does the pinned App-token action mint a short-lived + token for `AgoraHub CD Ref Writer` with repository contents write; +- using that App token, the controller independently re-reads current `main`, + creates `coolify-production` only on an exact 404, or performs a non-force + update of the existing ref to `expected_main_sha`, then re-reads the ref and + requires exact equality; +- a non-fast-forward update fails closed; the workflow never forces, changes a + ruleset, reads Coolify/Cloudflare credentials, calls Coolify, or deploys. + +`scripts/recover-coolify-production-ref.test.ts` exercises the state machine +against controlled GitHub API responses: already-current no-op, non-force +update, exact-404 creation, changed-main rejection, update failure without +creation fallback, and final-ref mismatch. Workflow source checks are limited to +GitHub-specific wiring that cannot be executed locally: manual trigger, +concurrency, validation-before-secrets ordering, pinned actions, and absence of +deployment credentials. + +The runbook may dispatch this workflow only after it has restored the Coolify +application to branch `main`, enabled native auto-deploy, disabled CD, quiesced +all existing deployment/deploy-workflow activity, and proved the new current +`main` SHA through its matching native delivery. The recovery run ID and final +ref SHA become part of the new baseline evidence. + +If drift occurs before the dedicated App and active ruleset exist, the operator +instead uses the authenticated human GitHub identity for one non-force create or +fast-forward of `coolify-production` to the newly proved current `main` SHA. The +operator must first verify that no active production-ref ruleset exists. A +non-fast-forward response, unexpected existing ruleset, or ambiguous partial App +installation stops recovery rather than falling back to a force update or +temporarily disabling protection. + +### Ephemeral GitHub App key handling + +GitHub's unavoidable App private-key download is isolated in a new mode-`0700` +directory on the local FileVault volume. The transfer shell runs with +error-stopping and unset-variable checks. Its EXIT and HUP/INT/TERM paths scan +that exact dedicated directory and securely remove every regular non-symlink +file with macOS `rm -P`, including files downloaded before the expected filename +or cardinality has been validated; symlink directory entries are unlinked +without following them, and the empty directory is removed. The successful path +performs the same secure deletion before clearing the traps. No private-key +content appears in an argument, clipboard, log, repository, Brain note, or +review artifact. ## Testing and Review @@ -292,15 +390,17 @@ workflow and then enforce at least these properties: - the ref update uses GitHub's plural `/git/refs/{ref}` PATCH contract, the final ref is re-read before queueing, and Coolify's live application branch is checked; -- enabled configuration is pinned to the four approved AgoraHub target values; +- enabled configuration is pinned to the five approved AgoraHub target values; - only the dedicated App token can write the ruleset-protected production ref, while checkout credentials are not persisted; - deployment concurrency is serialized rather than canceled; - the mode variable is always required; when enabled, every API variable/secret is required and missing credentials fail; -- Coolify requests contain both Cloudflare Access headers and the Coolify bearer - header; -- the queue response is validated and the returned deployment is polled; +- application/deployment reads contain both Cloudflare Access headers and the + read-only Coolify bearer header, while the webhook contains Access headers and + the exact HMAC signature but no bearer; +- the signed webhook acknowledgement is validated and the returned deployment + is polled; - terminal status and deployed commit SHA are fail-closed; - the monotonic absolute deadline cannot exceed 20 minutes, including request and sleep time; @@ -331,16 +431,22 @@ The work is complete only when all of the following are true: exactly its `head_sha` and no other SHA. 4. An active production-ref ruleset allows exactly the dedicated GitHub App to bypass update restrictions; the ordinary workflow token is read-only. -5. GitHub reaches Coolify through Cloudflare Service Auth and a least-privilege - Coolify token; neither token appears in files or logs. +5. GitHub reaches Coolify through Cloudflare Service Auth, an application- + specific webhook HMAC secret, and a separate read-only Coolify token; none + appears in files or logs. 6. GitHub waits for the matching Coolify deployment and fails unless status is `finished` and the recorded commit is the verified SHA. 7. The deployed production origin passes the existing 11-check release smoke. 8. Coolify native auto-deploy is disabled only after the controlled replacement deployment succeeds. 9. A read-only final audit shows one enabled deployment authority, the expected - running SHA, a healthy app/database, `COOLIFY_CD_ENABLED=true`, and no paired - native/API deployment for the controlled CI event. -10. The AgoraHub project note, infrastructure resource note, and daily note record - the final architecture, credential rotation responsibility, verification - evidence, and rollback procedure without recording secret values. + running SHA, a healthy app/database, `COOLIFY_CD_ENABLED=true`, exactly one + new controlled UUID since the recorded cutoff after a 120-second quiet + window, and no new native GitHub App delivery for the controlled CI event. +10. With `COOLIFY_CD_ENABLED=false`, the recovery-only workflow can non-force + create or advance `coolify-production` to the exact current `main` SHA using + only the dedicated App, without reading deployment credentials or calling + Coolify. +11. The AgoraHub project note, infrastructure resource note, and daily note record + the final architecture, credential rotation responsibility, verification + evidence, and rollback procedure without recording secret values. diff --git a/scripts/ci-workflow.test.ts b/scripts/ci-workflow.test.ts index 814637f..7ef18e3 100644 --- a/scripts/ci-workflow.test.ts +++ b/scripts/ci-workflow.test.ts @@ -1,4 +1,4 @@ -import { readFileSync } from "node:fs"; +import { existsSync, readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; function readJob(workflow: string, jobName: string): string { @@ -99,3 +99,84 @@ describe("Deploy workflow", () => { expect(workflow).toContain("run: pnpm release:smoke"); }); }); + +describe("Production-ref recovery workflow", () => { + const recoveryUrl = new URL( + "../.github/workflows/recover-coolify-production-ref.yml", + import.meta.url + ); + const workflow = existsSync(recoveryUrl) + ? readFileSync(recoveryUrl, "utf8") + : ""; + + it("is manual-only and serialized with production deployments", () => { + expect(workflow).toContain("workflow_dispatch:"); + expect(workflow).toContain("expected_main_sha:"); + expect(workflow).not.toContain("workflow_run:"); + expect(workflow).toContain("group: deploy-coolify"); + expect(workflow).toContain("cancel-in-progress: false"); + expect(workflow).toContain("contents: read"); + }); + + it("validates the disabled exact-main request before reading App secrets", () => { + const validationIndex = workflow.indexOf( + "- name: Validate disabled recovery request" + ); + const firstSecretIndex = workflow.indexOf("secrets."); + expect(validationIndex).toBeGreaterThanOrEqual(0); + expect(firstSecretIndex).toBeGreaterThan(validationIndex); + const validation = workflow.slice(validationIndex, firstSecretIndex); + expect(validation).toContain( + 'test "$GITHUB_REPOSITORY" = "Codevena/AgoraHub"' + ); + expect(validation).toContain('test "$GITHUB_REF" = "refs/heads/main"'); + expect(validation).toContain('test "$COOLIFY_CD_ENABLED" = "false"'); + expect(validation).toContain( + '[[ "$EXPECTED_MAIN_SHA" =~ ^[0-9a-f]{40}$ ]]' + ); + expect(validation).toContain("git/ref/heads/main"); + expect(validation).toContain( + 'test "$CURRENT_MAIN_SHA" = "$EXPECTED_MAIN_SHA"' + ); + }); + + it("pins trusted code and gives only the dedicated App write authority", () => { + expect(workflow).toContain( + "actions/checkout@11d5960a326750d5838078e36cf38b85af677262" + ); + expect(workflow).toContain("persist-credentials: false"); + expect(workflow).toContain( + "actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020" + ); + expect(workflow).toContain("node-version: 22.19.0"); + expect(workflow).toContain( + "actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1" + ); + expect(workflow).toContain("DEPLOY_REF_APP_ID"); + expect(workflow).toContain("DEPLOY_REF_APP_PRIVATE_KEY"); + expect(workflow).toContain("permission-contents: write"); + expect(workflow).toMatch( + /node --no-warnings --experimental-strip-types\s+scripts\/recover-coolify-production-ref\.ts/ + ); + expect(workflow).toContain( + "GITHUB_TOKEN: ${{ steps.deploy-ref-token.outputs.token }}" + ); + }); + + it("cannot receive deployment credentials or call Coolify", () => { + expect(workflow).not.toBe(""); + for (const forbidden of [ + "deploy-coolify.ts", + "COOLIFY_API_BASE_URL", + "COOLIFY_WEBHOOK_URL", + "COOLIFY_READ_TOKEN", + "COOLIFY_WEBHOOK_SECRET", + "CF_ACCESS_CLIENT_ID", + "CF_ACCESS_CLIENT_SECRET", + "release:smoke", + "panel.codevena.dev", + ]) { + expect(workflow).not.toContain(forbidden); + } + }); +}); diff --git a/scripts/recover-coolify-production-ref.test.ts b/scripts/recover-coolify-production-ref.test.ts new file mode 100644 index 0000000..e653ac7 --- /dev/null +++ b/scripts/recover-coolify-production-ref.test.ts @@ -0,0 +1,246 @@ +import { describe, expect, it } from "vitest"; + +const EXPECTED_SHA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const OLD_SHA = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; +const DRIFTED_SHA = "cccccccccccccccccccccccccccccccccccccccc"; +const REPOSITORY = "Codevena/AgoraHub"; +const API_BASE = `https://api.github.com/repos/${REPOSITORY}`; + +interface FetchCall { + url: string; + init: RequestInit; +} + +function refPayload(branch: string, sha: string): Record { + return { + ref: `refs/heads/${branch}`, + node_id: `REF_${branch}`, + url: `${API_BASE}/git/refs/heads/${branch}`, + object: { + type: "commit", + sha, + url: `${API_BASE}/git/commits/${sha}`, + }, + }; +} + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { "Content-Type": "application/json" }, + }); +} + +function fetchQueue(responses: Array): { + calls: FetchCall[]; + fetchImpl: typeof fetch; +} { + const calls: FetchCall[] = []; + const fetchImpl: typeof fetch = async (input, init = {}) => { + const url = + typeof input === "string" + ? input + : input instanceof URL + ? input.toString() + : input.url; + calls.push({ url, init }); + const next = responses.shift(); + if (!next) throw new Error(`Unexpected request: ${url}`); + if (next instanceof Error) throw next; + return next; + }; + return { calls, fetchImpl }; +} + +function validEnv(): Record { + return { + COOLIFY_CD_ENABLED: "false", + EXPECTED_MAIN_SHA: EXPECTED_SHA, + GITHUB_REPOSITORY: REPOSITORY, + GITHUB_TOKEN: "short-lived-app-token", + }; +} + +describe("production-ref recovery controller", () => { + it("accepts only disabled mode and the approved recovery target", async () => { + const { readRecoveryConfig } = await import( + "./recover-coolify-production-ref" + ); + + expect(readRecoveryConfig(validEnv())).toEqual({ + expectedMainSha: EXPECTED_SHA, + githubRepository: REPOSITORY, + githubToken: "short-lived-app-token", + }); + + for (const [name, value, message] of [ + ["COOLIFY_CD_ENABLED", "true", "must be exactly false"], + ["COOLIFY_CD_ENABLED", "", "must be exactly false"], + ["EXPECTED_MAIN_SHA", EXPECTED_SHA.toUpperCase(), "lowercase 40-character"], + ["GITHUB_REPOSITORY", "Codevena/Other", "approved AgoraHub repository"], + ["GITHUB_TOKEN", " ", "GITHUB_TOKEN is required"], + ] as const) { + expect(() => + readRecoveryConfig({ ...validEnv(), [name]: value }) + ).toThrow(message); + } + }); + + it("non-force updates an existing production ref and verifies the final ref", async () => { + const { readRecoveryConfig, recoverProductionRef } = await import( + "./recover-coolify-production-ref" + ); + const { calls, fetchImpl } = fetchQueue([ + jsonResponse(refPayload("main", EXPECTED_SHA)), + jsonResponse(refPayload("coolify-production", OLD_SHA)), + jsonResponse(refPayload("coolify-production", EXPECTED_SHA)), + jsonResponse(refPayload("coolify-production", EXPECTED_SHA)), + ]); + + await expect( + recoverProductionRef(readRecoveryConfig(validEnv()), { fetchImpl }) + ).resolves.toEqual({ status: "recovered", previousSha: OLD_SHA }); + + expect(calls.map(({ url }) => url)).toEqual([ + `${API_BASE}/git/ref/heads/main`, + `${API_BASE}/git/ref/heads/coolify-production`, + `${API_BASE}/git/refs/heads/coolify-production`, + `${API_BASE}/git/ref/heads/coolify-production`, + ]); + expect(calls[2].init.method).toBe("PATCH"); + expect(JSON.parse(String(calls[2].init.body))).toEqual({ + sha: EXPECTED_SHA, + force: false, + }); + expect(new Headers(calls[2].init.headers).get("Authorization")).toBe( + "Bearer short-lived-app-token" + ); + }); + + it("creates only an exactly missing production ref and verifies it", async () => { + const { readRecoveryConfig, recoverProductionRef } = await import( + "./recover-coolify-production-ref" + ); + const { calls, fetchImpl } = fetchQueue([ + jsonResponse(refPayload("main", EXPECTED_SHA)), + jsonResponse( + { + message: "Not Found", + documentation_url: "https://docs.github.com/rest/git/refs#get-a-reference", + status: "404", + }, + 404 + ), + jsonResponse(refPayload("coolify-production", EXPECTED_SHA), 201), + jsonResponse(refPayload("coolify-production", EXPECTED_SHA)), + ]); + + await expect( + recoverProductionRef(readRecoveryConfig(validEnv()), { fetchImpl }) + ).resolves.toEqual({ status: "recovered", previousSha: null }); + expect(calls[2].url).toBe(`${API_BASE}/git/refs`); + expect(calls[2].init.method).toBe("POST"); + expect(JSON.parse(String(calls[2].init.body))).toEqual({ + ref: "refs/heads/coolify-production", + sha: EXPECTED_SHA, + }); + }); + + it("does not write when the production ref already matches", async () => { + const { readRecoveryConfig, recoverProductionRef } = await import( + "./recover-coolify-production-ref" + ); + const { calls, fetchImpl } = fetchQueue([ + jsonResponse(refPayload("main", EXPECTED_SHA)), + jsonResponse(refPayload("coolify-production", EXPECTED_SHA)), + jsonResponse(refPayload("coolify-production", EXPECTED_SHA)), + ]); + + await expect( + recoverProductionRef(readRecoveryConfig(validEnv()), { fetchImpl }) + ).resolves.toEqual({ status: "recovered", previousSha: EXPECTED_SHA }); + expect(calls).toHaveLength(3); + expect(calls.every(({ init }) => init.method === undefined)).toBe(true); + }); + + it("rejects changed main before reading the production ref", async () => { + const { readRecoveryConfig, recoverProductionRef } = await import( + "./recover-coolify-production-ref" + ); + const { calls, fetchImpl } = fetchQueue([ + jsonResponse(refPayload("main", DRIFTED_SHA)), + ]); + + await expect( + recoverProductionRef(readRecoveryConfig(validEnv()), { fetchImpl }) + ).rejects.toThrow("GitHub main ref changed during recovery"); + expect(calls).toHaveLength(1); + }); + + it("does not treat a forbidden production-ref read as missing", async () => { + const { readRecoveryConfig, recoverProductionRef } = await import( + "./recover-coolify-production-ref" + ); + const { calls, fetchImpl } = fetchQueue([ + jsonResponse(refPayload("main", EXPECTED_SHA)), + jsonResponse({ message: "Resource not accessible by integration" }, 403), + ]); + + await expect( + recoverProductionRef(readRecoveryConfig(validEnv()), { fetchImpl }) + ).rejects.toThrow("GitHub production ref read failed with HTTP 403"); + expect(calls).toHaveLength(2); + }); + + it("does not treat a successful null production-ref payload as missing", async () => { + const { readRecoveryConfig, recoverProductionRef } = await import( + "./recover-coolify-production-ref" + ); + const { calls, fetchImpl } = fetchQueue([ + jsonResponse(refPayload("main", EXPECTED_SHA)), + jsonResponse(null), + ]); + + await expect( + recoverProductionRef(readRecoveryConfig(validEnv()), { fetchImpl }) + ).rejects.toThrow( + "GitHub production ref read returned no valid lowercase SHA" + ); + expect(calls).toHaveLength(2); + }); + + it("does not fall back to creation after a failed non-force update", async () => { + const { readRecoveryConfig, recoverProductionRef } = await import( + "./recover-coolify-production-ref" + ); + const { calls, fetchImpl } = fetchQueue([ + jsonResponse(refPayload("main", EXPECTED_SHA)), + jsonResponse(refPayload("coolify-production", OLD_SHA)), + jsonResponse({ message: "Update is not a fast forward" }, 422), + ]); + + await expect( + recoverProductionRef(readRecoveryConfig(validEnv()), { fetchImpl }) + ).rejects.toThrow("GitHub production ref update failed with HTTP 422"); + expect(calls).toHaveLength(3); + expect(calls.some(({ init }) => init.method === "POST")).toBe(false); + }); + + it("rejects a mismatched final production ref", async () => { + const { readRecoveryConfig, recoverProductionRef } = await import( + "./recover-coolify-production-ref" + ); + const { fetchImpl } = fetchQueue([ + jsonResponse(refPayload("main", EXPECTED_SHA)), + jsonResponse(refPayload("coolify-production", OLD_SHA)), + jsonResponse(refPayload("coolify-production", EXPECTED_SHA)), + jsonResponse(refPayload("coolify-production", OLD_SHA)), + ]); + + await expect( + recoverProductionRef(readRecoveryConfig(validEnv()), { fetchImpl }) + ).rejects.toThrow( + "GitHub final production ref does not match expected main SHA" + ); + }); +}); diff --git a/scripts/recover-coolify-production-ref.ts b/scripts/recover-coolify-production-ref.ts new file mode 100644 index 0000000..b26fabd --- /dev/null +++ b/scripts/recover-coolify-production-ref.ts @@ -0,0 +1,183 @@ +import { pathToFileURL } from "node:url"; + +export type FetchLike = ( + input: string | URL | Request, + init?: RequestInit +) => Promise; + +export interface RecoveryConfig { + expectedMainSha: string; + githubRepository: string; + githubToken: string; +} + +interface RecoveryDependencies { + fetchImpl?: FetchLike; +} + +export interface RecoveryOutcome { + status: "recovered"; + previousSha: string | null; +} + +const APPROVED_REPOSITORY = "Codevena/AgoraHub"; +const PRODUCTION_BRANCH = "coolify-production"; +const SHA_PATTERN = /^[0-9a-f]{40}$/; +const NOT_FOUND = Symbol("not-found"); + +function required( + env: Record, + name: string +): string { + const value = env[name]?.trim(); + if (!value) throw new Error(`${name} is required for production-ref recovery`); + return value; +} + +export function readRecoveryConfig( + env: Record = process.env +): RecoveryConfig { + if (env.COOLIFY_CD_ENABLED?.trim() !== "false") { + throw new Error("COOLIFY_CD_ENABLED must be exactly false for recovery"); + } + + const expectedMainSha = required(env, "EXPECTED_MAIN_SHA"); + const githubRepository = required(env, "GITHUB_REPOSITORY"); + const githubToken = required(env, "GITHUB_TOKEN"); + + if (!SHA_PATTERN.test(expectedMainSha)) { + throw new Error("EXPECTED_MAIN_SHA must be a lowercase 40-character Git SHA"); + } + if (githubRepository !== APPROVED_REPOSITORY) { + throw new Error("GITHUB_REPOSITORY does not match the approved AgoraHub repository"); + } + + return { expectedMainSha, githubRepository, githubToken }; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function readRefSha(value: unknown, label: string): string { + const sha = isRecord(value) && isRecord(value.object) + ? value.object.sha + : undefined; + if (typeof sha !== "string" || !SHA_PATTERN.test(sha)) { + throw new Error(`${label} returned no valid lowercase SHA`); + } + return sha; +} + +async function requestJson( + fetchImpl: FetchLike, + input: string, + init: RequestInit, + label: string, + allowNotFound = false +): Promise { + let response: Response; + try { + response = await fetchImpl(input, { ...init, redirect: "manual" }); + } catch (error) { + const detail = error instanceof Error ? `: ${error.message}` : ""; + throw new Error(`${label} request failed${detail}`, { cause: error }); + } + + if (allowNotFound && response.status === 404) return NOT_FOUND; + if (response.status >= 300 && response.status < 400) { + throw new Error(`${label} returned an unexpected redirect (${response.status})`); + } + if (!response.ok) throw new Error(`${label} failed with HTTP ${response.status}`); + + try { + return await response.json(); + } catch { + throw new Error(`${label} returned invalid JSON`); + } +} + +export async function recoverProductionRef( + config: RecoveryConfig, + dependencies: RecoveryDependencies = {} +): Promise { + const fetchImpl = dependencies.fetchImpl ?? fetch; + const apiBase = `https://api.github.com/repos/${config.githubRepository}`; + const headers = { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${config.githubToken}`, + "Content-Type": "application/json", + "X-GitHub-Api-Version": "2022-11-28", + }; + const get = (input: string, label: string, allowNotFound = false) => + requestJson(fetchImpl, input, { headers }, label, allowNotFound); + + const mainValue = await get(`${apiBase}/git/ref/heads/main`, "GitHub main ref"); + if (readRefSha(mainValue, "GitHub main ref") !== config.expectedMainSha) { + throw new Error("GitHub main ref changed during recovery"); + } + + const productionReadUrl = `${apiBase}/git/ref/heads/${PRODUCTION_BRANCH}`; + const productionValue = await get( + productionReadUrl, + "GitHub production ref read", + true + ); + const previousSha = productionValue === NOT_FOUND + ? null + : readRefSha(productionValue, "GitHub production ref read"); + + if (previousSha === null) { + await requestJson( + fetchImpl, + `${apiBase}/git/refs`, + { + method: "POST", + headers, + body: JSON.stringify({ + ref: `refs/heads/${PRODUCTION_BRANCH}`, + sha: config.expectedMainSha, + }), + }, + "GitHub production ref creation" + ); + } else if (previousSha !== config.expectedMainSha) { + await requestJson( + fetchImpl, + `${apiBase}/git/refs/heads/${PRODUCTION_BRANCH}`, + { + method: "PATCH", + headers, + body: JSON.stringify({ sha: config.expectedMainSha, force: false }), + }, + "GitHub production ref update" + ); + } + + const finalValue = await get( + productionReadUrl, + "GitHub final production ref" + ); + if ( + readRefSha(finalValue, "GitHub final production ref") !== + config.expectedMainSha + ) { + throw new Error("GitHub final production ref does not match expected main SHA"); + } + + return { status: "recovered", previousSha }; +} + +async function main(): Promise { + const config = readRecoveryConfig(); + await recoverProductionRef(config); + console.log(`Production ref recovered at ${config.expectedMainSha}`); +} + +const entrypoint = process.argv[1]; +if (entrypoint && import.meta.url === pathToFileURL(entrypoint).href) { + void main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +}