fix(agentic): make phase-eval status cleanup race-safe

.github/scripts/phase-eval-status.mjs

@@ -0,0 +1,59 @@
+/** Prefix reserved for the single lifecycle status label. */
+export const STATUS_PREFIX = 'status:';
+
+/** Terminal status entered before dispatching IMPL-EVAL. */
+export const IMPL_EVAL_STATUS = 'status:impl-eval';
+
+/** GitHub's exact response message when a label is absent from an issue. */
+export const MISSING_LABEL_MESSAGE = 'Label does not exist';
+
+/**
+ * Decide the idempotent status-label transition from a live issue-label set.
+ *
+ * @param {readonly string[]} liveLabels
+ * @returns {{ remove: string[], add: string[] }}
+ */
+export function decideImplEvalStatusTransition(liveLabels) {
+  return {
+    remove: liveLabels.filter((label) => label.startsWith(STATUS_PREFIX)),
+    add: [IMPL_EVAL_STATUS],
+  };
+}
+
+/**
+ * Apply the IMPL-EVAL transition through injected GitHub label operations.
+ *
+ * @param {{
+ *   listLabelsOnIssue: () => Promise<string[]>,
+ *   removeLabel: (label: string) => Promise<void>,
+ *   addLabels: (labels: string[]) => Promise<void>,
+ * }} operations
+ */
+export async function applyImplEvalStatusTransition(operations) {
+  const liveLabels = await operations.listLabelsOnIssue();
+  const decision = decideImplEvalStatusTransition(liveLabels);
+
+  for (const label of decision.remove) {
+    try {
+      await operations.removeLabel(label);
+    } catch (error) {
+      if (!isMissingLabelError(error)) throw error;
+    }
+  }
+
+  await operations.addLabels(decision.add);
+}
+
+/** @param {unknown} error */
+function isMissingLabelError(error) {
+  if (!isRecord(error) || error.status !== 404) return false;
+  const response = error.response;
+  if (!isRecord(response)) return false;
+  const data = response.data;
+  return isRecord(data) && data.message === MISSING_LABEL_MESSAGE;
+}
+
+/** @param {unknown} value */
+function isRecord(value) {
+  return typeof value === 'object' && value !== null;
+}

.github/scripts/phase-eval-status.test.ts

@@ -0,0 +1,166 @@
+import { assertEquals, assertRejects, assertStringIncludes } from '@std/assert';
+import {
+  applyImplEvalStatusTransition,
+  decideImplEvalStatusTransition,
+  IMPL_EVAL_STATUS,
+  MISSING_LABEL_MESSAGE,
+} from './phase-eval-status.mjs';
+
+interface IssueLabelOperations {
+  listLabelsOnIssue(): Promise<string[]>;
+  removeLabel(label: string): Promise<void>;
+  addLabels(labels: string[]): Promise<void>;
+}
+
+function workflowStep(source: string, name: string): string {
+  const lines = source.split('\n');
+  const start = lines.indexOf(`      - name: ${name}`);
+  if (start < 0) throw new Error(`Missing workflow step: ${name}`);
+  let end = lines.length;
+  for (let index = start + 1; index < lines.length; index += 1) {
+    if (lines[index].startsWith('      - name: ')) {
+      end = index;
+      break;
+    }
+  }
+  return lines.slice(start, end).join('\n');
+}
+
+function operations(
+  labels: string[],
+  removeError?: (label: string) => unknown,
+) {
+  const removed: string[] = [];
+  const added: string[][] = [];
+  const client: IssueLabelOperations = {
+    listLabelsOnIssue: () => Promise.resolve(labels),
+    removeLabel: (label: string) => {
+      const error = removeError?.(label);
+      if (error !== undefined) return Promise.reject(error);
+      removed.push(label);
+      return Promise.resolve();
+    },
+    addLabels: (next: string[]) => {
+      added.push(next);
+      return Promise.resolve();
+    },
+  };
+  return { client, removed, added };
+}
+
+Deno.test('race regression: a concurrently removed status label does not fail cleanup', async () => {
+  const { client, removed, added } = operations(
+    ['status:impl', 'area:tooling'],
+    () => ({ status: 404, response: { data: { message: 'Label does not exist' } } }),
+  );
+
+  await applyImplEvalStatusTransition(client);
+
+  assertEquals(removed, []);
+  assertEquals(added, [['status:impl-eval']]);
+});
+
+Deno.test('narrow tolerance: permission failures still fail cleanup', async () => {
+  const { client } = operations(
+    ['status:impl'],
+    () => ({
+      status: 403,
+      response: { data: { message: 'Resource not accessible by integration' } },
+    }),
+  );
+
+  await assertRejects(() => applyImplEvalStatusTransition(client));
+});
+
+Deno.test('narrow tolerance: an unrelated 404 still fails cleanup', async () => {
+  const { client } = operations(
+    ['status:impl'],
+    () => ({ status: 404, response: { data: { message: 'Not Found' } } }),
+  );
+
+  await assertRejects(() => applyImplEvalStatusTransition(client));
+});
+
+Deno.test('terminal decision contains exactly one status label', () => {
+  const decision = decideImplEvalStatusTransition([
+    'type:fix',
+    'status:impl',
+    'status:plan-eval',
+    'area:tooling',
+  ]);
+
+  assertEquals(decision, {
+    remove: ['status:impl', 'status:plan-eval'],
+    add: ['status:impl-eval'],
+  });
+  const terminal = [
+    'type:fix',
+    'area:tooling',
+    ...decision.add,
+  ];
+  assertEquals(terminal.filter((label) => label.startsWith('status:')), ['status:impl-eval']);
+});
+
+Deno.test('generation deduplication remains before trigger creation', async () => {
+  const workflow = await Deno.readTextFile('.github/workflows/openhands-phase-eval.yml');
+  const marker =
+    'const marker = `<!-- openhands-phase-eval generation=${generationEvent.id} phase=${phase} head=${pr.head.sha} -->`;';
+  const claim = "String(comment.body ?? '').includes(marker)";
+  const earlyReturn = 'if (existing) {';
+  const create = 'github.rest.issues.createComment({';
+
+  assertStringIncludes(workflow, marker);
+  assertStringIncludes(workflow, claim);
+  assertStringIncludes(workflow, earlyReturn);
+  assertStringIncludes(workflow, create);
+  assertEquals(workflow.indexOf(marker) < workflow.indexOf(claim), true);
+  assertEquals(workflow.indexOf(claim) < workflow.indexOf(earlyReturn), true);
+  assertEquals(workflow.indexOf(earlyReturn) < workflow.indexOf(create), true);
+});
+
+Deno.test('status bookkeeping failures are attributed and dispatch remains conditionally eligible', async () => {
+  const workflow = await Deno.readTextFile('.github/workflows/openhands-phase-eval.yml');
+  const transition = workflowStep(workflow, 'Enter IMPL-EVAL status on ready transition');
+  const diagnostic = workflowStep(
+    workflow,
+    'Record attributed IMPL-EVAL status-transition failure',
+  );
+  const dispatch = workflowStep(workflow, 'Resolve and dispatch exactly one evaluator');
+
+  assertStringIncludes(transition, 'id: enter_impl_eval_status');
+  assertStringIncludes(transition, 'continue-on-error: true');
+  assertStringIncludes(transition, 'core.setOutput(');
+  assertStringIncludes(transition, "'failure_reason'");
+  assertStringIncludes(diagnostic, "steps.enter_impl_eval_status.outcome == 'failure'");
+  assertStringIncludes(diagnostic, 'evaluator dispatch attempt continues');
+  assertStringIncludes(diagnostic, 'REQUEST_ACTOR: ${{ github.actor }}');
+  assertStringIncludes(diagnostic, 'FAILURE_REASON:');
+  assertStringIncludes(dispatch, '!cancelled()');
+  assertStringIncludes(
+    dispatch,
+    "steps.require_chainable_trigger_token.outcome == 'success'",
+  );
+  assertEquals(dispatch.includes('enter_impl_eval_status.outcome'), false);
+});
+
+Deno.test('inline cleanup transcription matches helper contract literals', async () => {
+  const workflow = await Deno.readTextFile('.github/workflows/openhands-phase-eval.yml');
+  const transition = workflowStep(workflow, 'Enter IMPL-EVAL status on ready transition');
+
+  assertEquals(workflow.includes('Check out trusted phase-eval scripts'), false);
+  assertEquals(transition.includes('await import('), false);
+  assertStringIncludes(transition, 'github.rest.issues.listLabelsOnIssue');
+  assertStringIncludes(
+    transition,
+    `const IMPL_EVAL_STATUS = '${IMPL_EVAL_STATUS}';`,
+  );
+  assertStringIncludes(
+    transition,
+    `const MISSING_LABEL_MESSAGE = '${MISSING_LABEL_MESSAGE}';`,
+  );
+  assertStringIncludes(
+    transition,
+    'error?.response?.data?.message === MISSING_LABEL_MESSAGE',
+  );
+  assertStringIncludes(transition, 'labels: [IMPL_EVAL_STATUS]');
+});

.github/workflows/openhands-phase-eval.yml

@@ -57,6 +57,7 @@ jobs:
           } >> "$GITHUB_STEP_SUMMARY"
 
       - name: Require chainable trigger token
+        id: require_chainable_trigger_token
         if: env.SKIP_IMPL != 'true'
         env:
           CHAIN_TOKEN: ${{ secrets.PAT_TOKEN }}
@@ -67,24 +68,85 @@ jobs:
           fi
 
       - name: Enter IMPL-EVAL status on ready transition
-        if: env.SKIP_IMPL != 'true' && github.event.action == 'ready_for_review'
+        id: enter_impl_eval_status
+        if: >-
+          !cancelled() &&
+          env.SKIP_IMPL != 'true' &&
+          github.event.action == 'ready_for_review' &&
+          steps.require_chainable_trigger_token.outcome == 'success'
+        continue-on-error: true
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
         with:
           # This transition must use the same chainable token as dispatch. Repository Actions
           # policy may make GITHUB_TOKEN read-only even when workflow permissions request writes.
           github-token: ${{ secrets.PAT_TOKEN }}
           script: |
-            const owner = context.repo.owner;
-            const repo = context.repo.repo;
-            const issue_number = context.payload.pull_request.number;
-            const labels = context.payload.pull_request.labels.map((label) => label.name);
-            for (const label of labels.filter((name) => name.startsWith('status:'))) {
-              await github.rest.issues.removeLabel({ owner, repo, issue_number, name: label });
+            try {
+              const owner = context.repo.owner;
+              const repo = context.repo.repo;
+              const issue_number = context.payload.pull_request.number;
+              const STATUS_PREFIX = 'status:';
+              const IMPL_EVAL_STATUS = 'status:impl-eval';
+              const MISSING_LABEL_MESSAGE = 'Label does not exist';
+              const liveLabels = await github.paginate(
+                github.rest.issues.listLabelsOnIssue,
+                { owner, repo, issue_number, per_page: 100 },
+              );
+              for (const { name } of liveLabels) {
+                if (!name.startsWith(STATUS_PREFIX)) continue;
+                try {
+                  await github.rest.issues.removeLabel({ owner, repo, issue_number, name });
+                } catch (error) {
+                  const missingLabel = error?.status === 404 &&
+                    error?.response?.data?.message === MISSING_LABEL_MESSAGE;
+                  if (!missingLabel) throw error;
+                }
+              }
+              await github.rest.issues.addLabels({
+                owner,
+                repo,
+                issue_number,
+                labels: [IMPL_EVAL_STATUS],
+              });
+            } catch (error) {
+              const reason = (error instanceof Error ? error.message : String(error))
+                .replace(/[\r\n\0]+/g, ' ')
+                .slice(0, 500);
+              core.setOutput(
+                'failure_reason',
+                reason,
+              );
+              throw error;
             }
-            await github.rest.issues.addLabels({ owner, repo, issue_number, labels: ['status:impl-eval'] });
+
+      - name: Record attributed IMPL-EVAL status-transition failure
+        if: >-
+          !cancelled() &&
+          env.SKIP_IMPL != 'true' &&
+          github.event.action == 'ready_for_review' &&
+          steps.enter_impl_eval_status.outcome == 'failure'
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+          REQUEST_ACTOR: ${{ github.actor }}
+          FAILURE_REASON: ${{ steps.enter_impl_eval_status.outputs.failure_reason }}
+        run: |
+          {
+            echo '## OpenHands phase evaluation'
+            echo
+            echo '**Status:** IMPL-EVAL status transition failed; evaluator dispatch attempt continues'
+            echo
+            printf -- '- Who: `@%s`\n' "$REQUEST_ACTOR"
+            printf -- '- PR: `#%s`\n' "$PR_NUMBER"
+            printf -- '- Head SHA: `%s`\n' "$HEAD_SHA"
+            printf -- '- Reason: `%s`\n' "$FAILURE_REASON"
+          } >> "$GITHUB_STEP_SUMMARY"
 
       - name: Resolve and dispatch exactly one evaluator
-        if: env.SKIP_IMPL != 'true'
+        if: >-
+          !cancelled() &&
+          env.SKIP_IMPL != 'true' &&
+          steps.require_chainable_trigger_token.outcome == 'success'
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
         with:
           github-token: ${{ secrets.PAT_TOKEN }}

.llm/runs/release-0.0.6-internals--orchestration/slices/pr-f-1566/context-pack.md

@@ -0,0 +1,44 @@
+# PR-F #1566 Context Pack
+
+## Objective
+
+Make the ready-for-review IMPL-EVAL status cleanup read live labels and tolerate only the specific
+missing-label 404 race, while preserving generation deduplication and the single-status taxonomy.
+
+## Current state
+
+- Branch is based on `e67c1ba13` (`origin/main`).
+- The supplied implementation brief is committed at `fe1d3b5e8` and pushed.
+- Live issue #1566 has six acceptance boxes; PR evidence must map them with `box-index: 1..6`.
+- Draft PR #1567 is open with `type:fix`, `area:tooling`, `priority:p2`, `status:impl`, and milestone
+  `0.0.6`; it remains draft.
+- The S1 test file defines the extracted module contract, race regression, narrow 403/unrelated-404
+  failures, terminal single-status state, and a guard for the unchanged generation-dedup ordering.
+- S2 implementation and all functional/static gates are green. The asset generator produced no
+  generated-file drift; a final post-commit clean status remains to capture before handoff.
+- Orchestrator review found that bookkeeping failures could still abort dispatch and that this PR
+  cannot bootstrap the trusted-base module on its own ready event. The labeled evaluation path is
+  orchestrator-owned; the review-fix slice makes checkout/transition failures non-blocking and
+  attributed without weakening the trusted-base boundary.
+- Review-fix commit `7170d574b3` is pushed. Gates are green: 66 script tests plus scoped
+  check/lint/format, YAML parsing, and post-commit asset generation with an empty working tree.
+- The PR body and S3 phase comment state the box-1 interpretation and bootstrap limitation. PR
+  #1567 remains draft with exactly `status:impl` and milestone `0.0.6`.
+- Run `31598386001` showed the hidden event-history dependency: dispatch ran after the non-fatal
+  transition failure, then failed because no `status:impl-eval` labeled-event generation existed.
+  The owner-directed next landing removes checkout/import and transcribes the tested cleanup inline;
+  the helper and unit tests remain.
+- Self-contained implementation commit `d7ea38f1cd` is pushed. All six local gates are green: 67
+  script tests, scoped check/lint/format, YAML parsing, and post-commit asset generation followed by
+  an empty working-tree proof.
+
+## Locked boundaries
+
+Only the phase-eval workflow, `.github/scripts/`, and this slice directory may change. Do not alter
+dispatch deduplication, triggers, conditions, model/trusted-base logic, #1564, or PR #1541. Do not
+merge or mark the PR ready.
+
+## Next action
+
+Update the PR body/evidence and phase comment, then stop. The orchestrator owns the ready flip and
+automatic DeepSeek retry; this agent must not trigger or merge.

.llm/runs/release-0.0.6-internals--orchestration/slices/pr-f-1566/drift.md

@@ -0,0 +1,23 @@
+# PR-F #1566 Drift Log
+
+## Entries
+
+- 2026-08-12: The orchestrator-provided first bootstrap commit contained `implement.md` only.
+  Completed the mandatory harness artifact set in an immediate bootstrap follow-up before tests or
+  implementation; no product scope changed.
+- 2026-08-12: Orchestrator review identified a bootstrap limitation and widened the reliability
+  invariant from one known 404 to all status-bookkeeping failures. `phase-eval-status.mjs` is absent
+  on `origin/main`, so PR #1567 cannot import it from the trusted base during its own ready event;
+  the orchestrator will evaluate this PR through the labeled path. The durable fix keeps trusted
+  base execution and makes checkout/transition failures attributed but non-blocking for dispatch.
+  The new independence test statically validates named workflow step contracts and dependencies;
+  it cannot simulate GitHub Actions runner status semantics locally, so its evidence is policy
+  structure plus YAML parsing rather than an end-to-end Actions execution.
+- 2026-08-12: Run `31598386001` corrected the prior interpretation. Dispatch was conditionally
+  eligible and did run, but its data dependency on a `status:impl-eval` labeled-event generation
+  made successful dispatch impossible after transition failure. The static policy test's recorded
+  limitation was decisive; it is retained but no longer cited as end-to-end independence evidence.
+  Owner directed a self-contained first landing: inline the tested cleanup in the workflow and keep
+  the helper as its independently tested contract. Importing the helper is deferred to a follow-up
+  only after this PR merges and the helper is reachable from trusted `main`; no PR-head fallback is
+  permitted.