diff --git a/.github/workflows/retrain.yml b/.github/workflows/retrain.yml index 1de3293..1ccdfa5 100644 --- a/.github/workflows/retrain.yml +++ b/.github/workflows/retrain.yml @@ -55,14 +55,39 @@ jobs: p.write_text(json.dumps(d, indent=2, ensure_ascii=False) + "\n") PY - - name: Verify the fitted weights still pass the suite + # Sanity gate: the fitted weights must still separate clear AI from clear + # human. This is deliberately NOT the full pytest suite — the frozen golden + # snapshots pin the *exact* current output, so any weight change trips them + # by design; updating those snapshots is part of a human accepting the PR. + # A failure here is a soft "candidate rejected", not a red workflow. + - name: Sanity-check the fitted weights + id: sane + continue-on-error: true run: | - pip install -e ".[dev]" || pip install pytest - pytest tests/ -k "detect or golden or snapshot" -q --timeout=120 + python - <<'PY' + import sys + from texthumanize.detectors import AIDetector + ai = ("In today's rapidly evolving digital landscape, it is important to note " + "that leveraging synergistic solutions can significantly enhance productivity. " + "Furthermore, organizations must carefully consider the multifaceted implications " + "of these transformative technologies. In conclusion, robust frameworks play a " + "crucial role in navigating modern business environments.") + human = ("So I finally got around to fixing that flaky test on Tuesday. Took me three " + "hours. Turns out the whole thing was a race condition in the mock server. " + "My teammate Dave had bet me five bucks it was a timezone bug. The fix was " + "literally two lines. Two! QA found nothing after we shipped it Thursday.") + AIDetector._fitted_weights_cache = None + a = AIDetector().detect(ai, "en").ai_probability + h = AIDetector().detect(human, "en").ai_probability + print(f"sanity: ai={a:.2f} human={h:.2f}") + sys.exit(0 if (a >= 0.55 and h <= 0.45 and a - h >= 0.2) else 1) + PY # Open a PR with the gh CLI (pre-installed on the runner) rather than a - # third-party action — nothing to pin, no supply-chain surface. + # third-party action — nothing to pin, no supply-chain surface. Only when + # the sanity check passed; otherwise the candidate is silently dropped. - name: Open a pull request if the weights changed + if: steps.sane.outcome == 'success' env: GH_TOKEN: ${{ github.token }} run: | @@ -71,6 +96,16 @@ jobs: echo "weights unchanged — nothing to propose." exit 0 fi + # Note for the reviewer: any weight change alters the detector's exact + # output, so the frozen golden snapshot (tests/test_golden.py) will fail + # on this PR by design — refresh it as part of accepting the new weights. + { + echo + echo "---" + echo "⚠️ Accepting this PR changes the detector's output, so the frozen" + echo "golden snapshot in tests/test_golden.py will fail — update" + echo "EXPECTED_EN_HASH to the new value as part of merging." + } >> /tmp/train_report.txt branch="auto/retrain-weights" git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com"