Skip to content

feat(brief): add standing test-selection contract to ship and scout scaffolds #13

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pruge merged 1 commit into from
Aug 25, 2026
Merged

feat(brief): add standing test-selection contract to ship and scout scaffolds #13

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions bin/fm-brief.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,10 @@
# affected-based test selection before done, a learning loop into the project's
# docs/agents/codegraph/README.md with an AGENTS.md pointer, and a one-line note
# to continue without codegraph when the binary is missing.
# They also carry a standing test-selection contract: no full-suite first run,
# changed-first selection (fm-test-run.sh --changed here, the project runner's
# native equivalent elsewhere), failed-family-only reruns, GitHub CI as the PR
# merge verdict, and full output kept in a file so failures need no rerun.
# Ship briefs begin with a worktree-isolation assertion before the branch step.
# --mode is refused on scout and secondmate scaffolds: a scout's deliverable is a
# report rather than a merge, and a charter is not a delivery contract.
Expand Down Expand Up @@ -317,6 +321,16 @@ CODEGRAPH_SECTION=$(printf '%s\n' \
" Keep an \`AGENTS.md\` pointer section to that file via '$FM_ROOT/bin/fm-ensure-agents-md.sh .'" \
'If the `codegraph` binary is missing, append one status line noting it and continue with ordinary tools; nothing else changes.')

# Standing test-selection contract shared by the ship and scout scaffolds.
# shellcheck disable=SC2016 # single quotes are deliberate: backtick-wrapped command names must reach the reading agent verbatim, not expand at scaffold time.
TEST_SELECTION_SECTION=$(printf '%s\n' \
'# Test selection contract' \
'Verify efficiently: never make a full local suite run your first verification step.' \
"1. Changed-first selection: when the repo under test is the firstmate repo itself, select tests with \`bin/fm-test-run.sh --changed\` (add \`--base <ref>\` when your working base is not origin/main); on other projects use the runner's native changed/related feature, e.g. \`vitest --changed\`." \
'2. On failure, re-run only the failed family or single script (`bin/fm-test-run.sh --family <name>` or that one script path), never the whole suite.' \
'3. For PR-based deliveries, GitHub CI owns the final merge verdict; local runs are a fast pre-check, not the authority.' \
'4. Keep full test output in a file: never pipe it through `tail` or any other filter that discards per-test result lines, so failures stay identifiable without rerunning.')

if [ "$KIND" = scout ]; then
cat > "$BRIEF" <<EOF
You are a crewmate: an autonomous worker agent managed by firstmate. Work on your own; do not wait for a human.
Expand Down Expand Up @@ -357,6 +371,8 @@ The report is the only thing that survives, so anything worth keeping must be in

$CODEGRAPH_SECTION

$TEST_SELECTION_SECTION

# Definition of done
Write your findings to \`$DATA/$ID/report.md\`.
The report must stand alone: what you did, what you found, the evidence (commands run, output, file:line references), and what you recommend.
Expand Down Expand Up @@ -476,6 +492,8 @@ $RULE1

$CODEGRAPH_SECTION

$TEST_SELECTION_SECTION

# Project memory
If \`AGENTS.md\` or \`CLAUDE.md\` already exists, or if this task produced durable project-intrinsic knowledge, run \`$FM_ROOT/bin/fm-ensure-agents-md.sh .\` in the worktree.
Record only project knowledge useful to almost every future session.
Expand Down
41 changes: 41 additions & 0 deletions tests/fm-brief.test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,46 @@ test_codegraph_contract_in_ship_and_scout_briefs() {
pass "fm-brief.sh: ship and scout scaffolds carry the standing CodeGraph usage contract"
}

# Ship and scout scaffolds carry a standing test-selection contract; a
# secondmate charter supervises its own crews and must not carry this worker
# contract.
test_test_selection_contract_in_ship_and_scout_briefs() {
local home id kind brief
home="$TMP_ROOT/testsel-home"
mkdir -p "$home/data"
for kind in ship scout; do
id="brief-testsel-$kind"
if [ "$kind" = scout ]; then
FM_HOME="$home" FM_ROOT_OVERRIDE="$ROOT" "$ROOT/bin/fm-brief.sh" "$id" some-proj --scout >/dev/null 2>&1
else
FM_HOME="$home" FM_ROOT_OVERRIDE="$ROOT" "$ROOT/bin/fm-brief.sh" "$id" some-proj --mode direct-PR >/dev/null 2>&1
fi
brief="$home/data/$id/brief.md"
assert_grep "# Test selection contract" "$brief" \
"$kind brief missing the test-selection contract heading"
assert_grep "never make a full local suite run your first verification step" "$brief" \
"$kind brief missing the no-full-suite-first rule"
assert_grep "select tests with \`bin/fm-test-run.sh --changed\` (add \`--base <ref>\` when your working base is not origin/main)" "$brief" \
"$kind brief missing changed-first selection for the firstmate repo"
assert_grep "on other projects use the runner's native changed/related feature" "$brief" \
"$kind brief missing the other-projects changed-selection rule"
assert_grep "re-run only the failed family or single script (\`bin/fm-test-run.sh --family <name>\` or that one script path), never the whole suite" "$brief" \
"$kind brief missing the failed-family-only rerun rule"
assert_grep "GitHub CI owns the final merge verdict; local runs are a fast pre-check, not the authority" "$brief" \
"$kind brief missing the CI-owns-the-verdict rule"
assert_grep "never pipe it through \`tail\` or any other filter that discards per-test result lines" "$brief" \
"$kind brief missing the keep-full-output rule"
assert_no_grep "EOF" "$brief" \
"$kind brief leaked a heredoc EOF marker (unterminated heredoc) in the test-selection section"
done

FM_HOME="$home" FM_SECONDMATE_CHARTER='sample domain' \
"$ROOT/bin/fm-brief.sh" brief-testsel-mate --secondmate --no-projects >/dev/null 2>&1
assert_no_grep "# Test selection contract" "$home/data/brief-testsel-mate/brief.md" \
"secondmate charter must not carry the worker-level test-selection contract"
pass "fm-brief.sh: ship and scout scaffolds carry the standing test-selection contract"
}

test_script_parses() {
local out rc
out=$(bash -n "$ROOT/bin/fm-brief.sh" 2>&1); rc=$?
Expand Down Expand Up @@ -771,6 +811,7 @@ test_herdr_lab_contract_quotes_foreign_firstmate_path
test_herdr_lab_omission_is_loud_for_ship_and_scout
test_herdr_lab_contract_applies_to_scouts_but_not_secondmates
test_codegraph_contract_in_ship_and_scout_briefs
test_test_selection_contract_in_ship_and_scout_briefs
test_secondmate_no_projects_charter
test_secondmate_marked_request_reporting_contract
test_secondmate_directory_paths_are_absolute_and_output_is_stable
Expand Down
Loading