diff --git a/.github/workflows/auto-merge-to-develop.yaml b/.github/workflows/auto-merge-to-develop.yaml deleted file mode 100644 index b0ef613b..00000000 --- a/.github/workflows/auto-merge-to-develop.yaml +++ /dev/null @@ -1,49 +0,0 @@ -name: auto-merge-to-develop -# Merge green PRs that TARGET develop, the moment their checks complete. -# GitHub reads check_suite-triggered workflows from the DEFAULT branch (main), -# so this file lives on main but acts ONLY on PRs whose base is develop. -# codecov + readthedocs checks are ignored (advisory). Fail-safe: if a merge -# is blocked by branch policy the PR simply stays open. -on: - check_suite: - types: [completed] - workflow_dispatch: {} - -permissions: - contents: write - pull-requests: write - -jobs: - automerge: - if: ${{ github.event_name == 'workflow_dispatch' || github.event.check_suite.conclusion == 'success' }} - runs-on: ubuntu-latest - steps: - - name: merge green PRs targeting develop - env: - GH_TOKEN: ${{ github.token }} - REPO: ${{ github.repository }} - HEAD_SHA: ${{ github.event.check_suite.head_sha }} - run: | - set -uo pipefail - if [ -n "${HEAD_SHA:-}" ]; then - prs=$(gh api "repos/$REPO/commits/$HEAD_SHA/pulls" --jq '.[] | select(.state=="open") | .number' 2>/dev/null || true) - else - prs=$(gh pr list --repo "$REPO" --base develop --state open --json number --jq '.[].number' 2>/dev/null || true) - fi - [ -z "${prs:-}" ] && { echo "no open PRs"; exit 0; } - for pr in $prs; do - base=$(gh pr view "$pr" --repo "$REPO" --json baseRefName --jq .baseRefName 2>/dev/null || echo "") - [ "$base" = "develop" ] || { echo "skip #$pr (base=$base, not develop)"; continue; } - draft=$(gh pr view "$pr" --repo "$REPO" --json isDraft --jq .isDraft 2>/dev/null || echo true) - [ "$draft" = "true" ] && continue - pending=$(gh pr view "$pr" --repo "$REPO" --json statusCheckRollup --jq ' - [ .statusCheckRollup[] - | select(((.name // .context // "") | ascii_downcase | test("codecov|readthedocs")) | not) - | (.conclusion // .state // "") - | select(. != "SUCCESS" and . != "NEUTRAL" and . != "SKIPPED" and . != "") ] | length' 2>/dev/null || echo 1) - if [ "$pending" = "0" ]; then - gh pr merge "$pr" --repo "$REPO" --merge --admin --delete-branch && echo "MERGED #$pr -> develop" || echo "blocked #$pr" - else - echo "PR #$pr not green ($pending) — skip" - fi - done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..df9eff4b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,131 @@ +name: ci + +# ============================================================================ +# CANONICAL CI — IDENTICAL across every scitex ecosystem repo. +# Synced by `scitex-dev ecosystem sync-workflows`; do not edit per-repo (drift +# is reverted). Package-agnostic — the package name is derived from the +# checkout, so there is nothing to substitute here. +# +# Runner: the self-hosted Spartan CPU runner (label spartan-cpu). +# Deps: installed per run with `uv` (cached) — `.[all,dev]` so every repo's +# optional deps are present; no baked-SIF completeness to maintain. +# ============================================================================ + +on: + pull_request: + branches: [main, develop] + push: + branches: [main, develop] + schedule: + - cron: "0 17 * * *" # nightly 17:00 UTC (~02:00 JST), off-peak + workflow_dispatch: + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + pytest-matrix: + # Job name is a REQUIRED status check — keep verbatim fleet-wide. + name: pytest-matrix-on-ubuntu-py${{ matrix.python-version }} + runs-on: [self-hosted, Linux, X64, spartan-cpu] + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + python-version: ["3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v3 + with: + cache-dependency-glob: pyproject.toml + - name: Install (.[all,dev]) + pytest + run: | + set -euo pipefail + uv venv --python ${{ matrix.python-version }} .venv + uv pip install --python .venv/bin/python -e ".[all,dev]" \ + || uv pip install --python .venv/bin/python -e ".[dev]" \ + || uv pip install --python .venv/bin/python -e . + # kaleido/plotly image export needs Chrome; install best-effort (no-op for non-plotly repos) + yes | .venv/bin/plotly_get_chrome >/dev/null 2>&1 || true + PKG=$(basename "$(find src -maxdepth 1 -mindepth 1 -type d ! -name '*.egg-info' | sort | head -1)") + # .venv/bin on PATH so tests that spawn the package's console script via + # subprocess (bare name, e.g. `scitex-dev …`) resolve it — else FileNotFoundError. + export PATH="$PWD/.venv/bin:$PATH" + .venv/bin/python -m pytest tests/ --cov="src/$PKG" --cov-report=xml --cov-report=term + - name: Upload coverage to Codecov + if: always() && matrix.python-version == '3.12' + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage.xml + fail_ci_if_error: false + + import-smoke: + name: import-smoke-on-ubuntu-py3-12 + runs-on: [self-hosted, Linux, X64, spartan-cpu] + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v3 + with: + cache-dependency-glob: pyproject.toml + - name: Install (no extras) + import + run: | + set -euo pipefail + uv venv --python 3.12 .venv + uv pip install --python .venv/bin/python -e . + PKG=$(basename "$(find src -maxdepth 1 -mindepth 1 -type d ! -name '*.egg-info' | sort | head -1)") + .venv/bin/python -c "import importlib; importlib.import_module('$PKG'); print('import OK:', '$PKG')" + + audit: + name: audit + runs-on: [self-hosted, Linux, X64, spartan-cpu] + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v3 + with: + cache-dependency-glob: pyproject.toml + - name: Install + audit the workspace + run: | + set -euo pipefail + uv venv --python 3.12 .venv + uv pip install --python .venv/bin/python -e ".[all,dev]" \ + || uv pip install --python .venv/bin/python -e ".[dev]" \ + || uv pip install --python .venv/bin/python -e . + uv pip install --python .venv/bin/python scitex-dev + # .venv/bin on PATH so audit-all's spawned `scitex-dev` subcommands resolve. + export PATH="$PWD/.venv/bin:$PATH" + DIST=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['name'])") + # --path . audits the CHECKOUT (PR code), not the runner's shared state. + scitex-dev ecosystem audit-all "$DIST" --path "$PWD" + + docs-sphinx: + name: docs-sphinx + runs-on: [self-hosted, Linux, X64, spartan-cpu] + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v3 + with: + cache-dependency-glob: pyproject.toml + - name: Build sphinx HTML (satisfies PS-122; RTD bundle stays fresh) + run: | + set -euo pipefail + CONF="" + for c in docs/conf.py docs/source/conf.py docs/src/conf.py; do + [ -f "$c" ] && CONF="$c" && break + done + if [ -z "$CONF" ]; then + echo "no sphinx conf.py (checked docs[/source|/src]) — skipping; PS-122 satisfied by workflow content" + exit 0 + fi + uv venv --python 3.12 .venv + uv pip install --python .venv/bin/python -e ".[docs]" \ + || uv pip install --python .venv/bin/python -e ".[all,dev]" \ + || { uv pip install --python .venv/bin/python -e .; uv pip install --python .venv/bin/python sphinx; } + CONFDIR=$(dirname "$CONF") + # use the `sphinx-build` console script (not `python -m sphinx`) so the + # auditor's PS-122 detects this workflow runs sphinx-build. + .venv/bin/sphinx-build -b html "$CONFDIR" "$CONFDIR/_build/html" diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml index f8990eba..c927d12a 100644 --- a/.github/workflows/cla.yml +++ b/.github/workflows/cla.yml @@ -25,7 +25,7 @@ jobs: path-to-signatures: "signatures/cla.json" path-to-document: "https://github.com/ywatanabe1989/scitex-dev/blob/main/CLA.md" branch: "cla-signatures" - allowlist: bot*,ywatanabe1989 + allowlist: bot*,ywatanabe1989,LLEmacs custom-allsigned-prcomment: | Thank you for signing the SciTeX CLA. Your contribution can now be reviewed. custom-notsigned-prcomment: | diff --git a/.github/workflows/import-smoke-on-ubuntu-py3-12.yml b/.github/workflows/import-smoke-on-ubuntu-py3-12.yml deleted file mode 100644 index c2b79249..00000000 --- a/.github/workflows/import-smoke-on-ubuntu-py3-12.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: import-smoke - -# Catches "wheel installs but `import scitex` blows up at runtime" -# — separate from the full pytest suite so a broken entry-point fails -# fast without paying for the matrix. Companion to `tests` — confirms -# the bare package (no extras) is `pip install -e .`-able and -# `import scitex` succeeds. Catches `[project] dependencies` -# drift that the test matrix masks because it installs `[all,dev]`. - -on: - push: - branches: [main, develop] - pull_request: - branches: [main, develop] - schedule: - - cron: "0 17 * * *" # nightly 17:00 UTC (~02:00 JST), off-peak - workflow_dispatch: - -jobs: - install-check: - name: import-smoke-on-ubuntu-py3-12 - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: Install (no extras) - run: | - python -m venv .venv - .venv/bin/pip install --upgrade pip - .venv/bin/pip install -e . - - name: Import smoke - run: .venv/bin/python -c "import scitex; print(scitex.__version__)" diff --git a/.github/workflows/newb-docs-quality-on-ubuntu-latest.yml b/.github/workflows/newb-docs-quality-on-ubuntu-latest.yml deleted file mode 100644 index a3adef45..00000000 --- a/.github/workflows/newb-docs-quality-on-ubuntu-latest.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: newb - -# Doc-quality verification — fresh Claude agent reads only the package's -# docs in a hard-isolated container, then tries to install + import + -# use the package. If a docs-only consumer can't follow them, neither -# can a real user. -# -# Cost-aware schedule: weekly cron + manual dispatch only. Per-PR runs -# are intentionally NOT enabled here (LLM API cost + run time). -on: - schedule: - - cron: "0 17 * * 1" # weekly Mon 17:00 UTC (~Tue 02:00 JST) - workflow_dispatch: - -jobs: - newb: - runs-on: ubuntu-latest - timeout-minutes: 20 - permissions: - contents: read - packages: read # pull ghcr.io/ywatanabe1989/newb-runner - - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Login to ghcr.io (so docker can pull the runner image) - run: | - echo "${{ secrets.GITHUB_TOKEN }}" \ - | docker login ghcr.io -u "${{ github.actor }}" --password-stdin - - - name: Install newb (pinned) - # Pinned for reproducibility. Bump in a coordinated PR across - # the ecosystem so all packages exercise the same newb version. - run: pip install 'newb==0.25.0' - - - name: Show newb version + resolved image - run: | - newb --version - python -c "from newb._container_runner import _default_image; print('image:', _default_image())" - - - name: Run newb (verbose so the log shows per-prompt timing) - env: - # newb's own secret — the `NEWB_` prefix is owned by the newb - # distribution; PS-168 (narrowed) accepts cross-package - # prefixes from any ECOSYSTEM entry, so this needs no extra - # SCITEX_DEV_-prefix wrapper. Double-prefixing - # (SCITEX_DEV_NEWB_*) was tried briefly and reverted per the - # operator: stacking two prefixes obscures the meaning. - NEWB_ANTHROPIC_API_KEY: ${{ secrets.NEWB_ANTHROPIC_API_KEY }} - # Resource caps so an attacker who hijacked the docs-reading - # agent can't DoS via memory/pid exhaustion. Generous enough - # for pip wheel-building (~1-2 GB transient peak). - NEWB_HARDEN_MEMORY: 4g - NEWB_HARDEN_PIDS_LIMIT: 512 - NEWB_HARDEN_CPUS: "2" - run: | - if [ -z "${NEWB_ANTHROPIC_API_KEY}" ]; then - echo "::error::secrets.NEWB_ANTHROPIC_API_KEY is not set on this repo." >&2 - exit 1 - fi - newb . --json -vv > newb-report.json - - - name: Upload report - if: always() - uses: actions/upload-artifact@v4 - with: - name: newb-report - path: newb-report.json - if-no-files-found: warn - - - name: Render markdown summary into the run summary - if: success() - run: | - python - <<'PY' >> "$GITHUB_STEP_SUMMARY" - import json, newb - with open("newb-report.json") as f: - report = json.load(f) - print(newb.render_markdown(report)) - PY diff --git a/.github/workflows/newb.yml b/.github/workflows/newb.yml deleted file mode 100644 index 88651a36..00000000 --- a/.github/workflows/newb.yml +++ /dev/null @@ -1,77 +0,0 @@ -name: Newb - -# Doc-quality verification — fresh Claude agent reads only the package's -# docs in a hard-isolated container, then tries to install + import + -# use the package. If a docs-only consumer can't follow them, neither -# can a real user. -# -# Cost-aware schedule: weekly cron + manual dispatch only. Per-PR runs -# are intentionally NOT enabled here (LLM API cost + run time). -on: - schedule: - - cron: "0 17 * * 1" # weekly Mon 17:00 UTC (~Tue 02:00 JST) - workflow_dispatch: - -jobs: - newb: - runs-on: ubuntu-latest - timeout-minutes: 20 - permissions: - contents: read - packages: read # pull ghcr.io/ywatanabe1989/newb-runner - - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Login to ghcr.io (so docker can pull the runner image) - run: | - echo "${{ secrets.GITHUB_TOKEN }}" \ - | docker login ghcr.io -u "${{ github.actor }}" --password-stdin - - - name: Install newb (pinned) - # Pinned for reproducibility. Bump in a coordinated PR across - # the ecosystem so all packages exercise the same newb version. - run: pip install 'newb==0.25.0' - - - name: Show newb version + resolved image - run: | - newb --version - python -c "from newb._container_runner import _default_image; print('image:', _default_image())" - - - name: Run newb (verbose so the log shows per-prompt timing) - env: - NEWB_ANTHROPIC_API_KEY: ${{ secrets.NEWB_ANTHROPIC_API_KEY }} - # Resource caps so an attacker who hijacked the docs-reading - # agent can't DoS via memory/pid exhaustion. Generous enough - # for pip wheel-building (~1-2 GB transient peak). - NEWB_HARDEN_MEMORY: 4g - NEWB_HARDEN_PIDS_LIMIT: 512 - NEWB_HARDEN_CPUS: "2" - run: | - if [ -z "${NEWB_ANTHROPIC_API_KEY}" ]; then - echo "::error::secrets.NEWB_ANTHROPIC_API_KEY is not set on this repo." >&2 - exit 1 - fi - newb . --json -vv > newb-report.json - - - name: Upload report - if: always() - uses: actions/upload-artifact@v4 - with: - name: newb-report - path: newb-report.json - if-no-files-found: warn - - - name: Render markdown summary into the run summary - if: success() - run: | - python - <<'PY' >> "$GITHUB_STEP_SUMMARY" - import json, newb - with open("newb-report.json") as f: - report = json.load(f) - print(newb.render_markdown(report)) - PY diff --git a/.github/workflows/pypi-publish-and-github-release-on-tag.yml b/.github/workflows/pypi-publish-and-github-release-on-tag.yml deleted file mode 100644 index b0f92fcd..00000000 --- a/.github/workflows/pypi-publish-and-github-release-on-tag.yml +++ /dev/null @@ -1,243 +0,0 @@ -name: release - -# Tag-driven release pipeline. The intended operator flow is: -# -# git tag vX.Y.Z && git push origin vX.Y.Z -# -# Everything else fires automatically: -# -# 1. test — matrix pytest. Must pass or the pipeline halts; -# nothing is published until tests are green on the -# tagged commit. -# 2. build — wheel + sdist, uploaded as workflow artifact. -# 3. publish — PyPI via OIDC trusted publisher. -# 4. release — GitHub Release with CHANGELOG notes + artifacts. -# 5. sync-main — open a develop→main PR (admin-merge) so main -# always equals the latest released commit. Skipped -# if main already matches. -# -# Manual re-run: the workflow also accepts a `version` input via -# workflow_dispatch so you can re-publish without re-tagging. - -on: - push: - tags: ['v*'] - workflow_dispatch: - inputs: - version: - description: 'Tag to publish (e.g. v0.10.19) — must already exist on the repo' - required: true - type: string - -jobs: - test: - runs-on: ubuntu-latest - # Mirror the hardened `tests` matrix (pytest-matrix-on-ubuntu-*.yml): the - # release gate MUST use the same install + run recipe, or it reintroduces - # the chronic failures #19 fixed (pip wrapt resolver-thrash -> ~50min hang, - # CUDA-torch segfault, non-deterministic C-ext shutdown crash). Keeping a - # second, un-hardened copy here meant every release stalled at this gate. - timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - python-version: ["3.11", "3.12", "3.13"] - steps: - - name: Resolve version - id: ver - run: | - if [ "${{ github.event_name }}" = "push" ]; then - v="${GITHUB_REF#refs/tags/}" - else - v="${{ inputs.version }}" - fi - echo "version=$v" >> "$GITHUB_OUTPUT" - - uses: actions/checkout@v4 - with: - ref: ${{ steps.ver.outputs.version }} - fetch-depth: 0 - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install uv - # uv resolves the graph in seconds; pip backtracks through dozens of - # `wrapt` candidates at ~13s each (~50min) and hits the job cap. - uses: astral-sh/setup-uv@v5 - with: - enable-cache: true - - name: Install dependencies - run: | - # CPU-only torch FIRST so transitive torch resolves to the CPU build; - # default CUDA wheels segfault the suite on a GPU-less runner. - uv pip install --system torch --index-url https://download.pytorch.org/whl/cpu \ - || echo "CPU torch pre-install skipped (torch may be optional in this resolve)" - uv pip install --system -e ".[all,dev]" \ - || uv pip install --system -e ".[dev]" \ - || uv pip install --system -e . - - name: Install peer standalones aliased by the umbrella - run: | - uv pip install --system \ - scitex-dsp scitex-gen scitex-pd scitex-nn scitex-resource \ - scitex-linalg scitex-datetime scitex-decorators \ - || echo "peer standalones partially unavailable; identity test will importorskip the rest" - - name: Run tests - # Pass/fail judged from the JUnit report, not the exit code: the deep - # ecosystem C-extension stack (torch/numba/catboost) segfaults - # NON-deterministically and can kill the process before the report is - # written. Retry up to 3x on a crash-before-report; a finished run is - # judged by the report (real failures still fail). e2e + the - # cross-package import gate are quarantined (heavy/segfault-prone). - run: | - set +e - for attempt in 1 2 3; do - rm -f pytest-results.xml - pytest tests/ --ignore=tests/e2e \ - --ignore=tests/integration/test_cross_package_imports.py \ - -p no:cacheprovider -o faulthandler_timeout=120 \ - --junitxml=pytest-results.xml - if [ -f pytest-results.xml ]; then - echo "pytest produced a report on attempt $attempt" - break - fi - echo "::warning::attempt $attempt crashed before writing the report (C-extension shutdown/import segfault); retrying" - done - set -e - python - <<'PY' - import sys, xml.etree.ElementTree as ET - try: - root = ET.parse("pytest-results.xml").getroot() - except Exception as e: - print(f"::error::no JUnit report after retries ({e}) — pytest never " - "finished; failing") - sys.exit(1) - suites = list(root.iter("testsuite")) - bad = sum(int(s.get("failures", 0)) + int(s.get("errors", 0)) for s in suites) - total = sum(int(s.get("tests", 0)) for s in suites) - print(f"JUnit: {total} tests, {bad} failures+errors") - sys.exit(1 if bad else 0) - PY - - build: - needs: test - runs-on: ubuntu-latest - outputs: - version: ${{ steps.ver.outputs.version }} - steps: - - name: Resolve version - id: ver - run: | - if [ "${{ github.event_name }}" = "push" ]; then - v="${GITHUB_REF#refs/tags/}" - else - v="${{ inputs.version }}" - fi - echo "version=$v" >> "$GITHUB_OUTPUT" - - uses: actions/checkout@v4 - with: - ref: ${{ steps.ver.outputs.version }} - fetch-depth: 0 - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: Install build tools - run: pip install build - - name: Build distribution - run: python -m build - - uses: actions/upload-artifact@v4 - with: - name: dist - path: dist/ - - publish: - needs: build - runs-on: ubuntu-latest - environment: - name: pypi - url: https://pypi.org/p/scitex - permissions: - id-token: write - steps: - - uses: actions/download-artifact@v4 - with: - name: dist - path: dist - - uses: pypa/gh-action-pypi-publish@release/v1 - - release: - needs: [build, publish] - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ needs.build.outputs.version }} - - uses: actions/download-artifact@v4 - with: - name: dist - path: dist - - name: Extract release notes from CHANGELOG.md - run: | - VERSION="${{ needs.build.outputs.version }}" - VERSION="${VERSION#v}" - awk -v v="$VERSION" ' - $0 ~ "^## \\[" v "\\]" {p=1; next} - p && /^## \[/ {exit} - p - ' CHANGELOG.md > release-notes.md || true - if [ ! -s release-notes.md ]; then - echo "Release v${VERSION}. See CHANGELOG.md for details." > release-notes.md - fi - - name: Create GitHub Release - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh release create "${{ needs.build.outputs.version }}" \ - --title "${{ needs.build.outputs.version }}" \ - --notes-file release-notes.md \ - dist/* - - sync-main: - # Open develop→main PR so main always equals the latest released - # commit. Branch-protected mains require PR review; this job tries - # admin-merge but leaves the PR open if that isn't allowed. - needs: publish - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: develop - - name: Open develop→main PR if diverged - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - set -e - git fetch origin main - AHEAD=$(git rev-list --count origin/main..origin/develop) - if [ "$AHEAD" -eq 0 ]; then - echo "main already up-to-date with develop." - exit 0 - fi - EXISTING=$(gh pr list --base main --head develop --state open --json number --jq '.[0].number // empty') - if [ -z "$EXISTING" ]; then - PR_URL=$(gh pr create --base main --head develop \ - --title "release: sync main with develop" \ - --body "Auto-opened by the release pipeline so main equals the latest released commit.") - EXISTING=$(echo "$PR_URL" | grep -oE '[0-9]+$') - echo "Created PR #${EXISTING}" - else - echo "Reusing existing PR #${EXISTING}" - fi - gh pr merge "$EXISTING" --merge --admin \ - || echo "Admin merge unavailable; PR left open for manual merge." - # The repo has delete_branch_on_merge=true, so merging the - # develop→main PR DELETES the develop branch (its head). develop is - # an unprotected permanent branch; losing it breaks `git pull` on - # every dev machine ("no such ref develop was fetched"). Recreate it - # at the just-synced commit. Idempotent: a no-op if develop survived. - git push origin "HEAD:refs/heads/develop" \ - || echo "develop already present or push skipped." diff --git a/.github/workflows/pytest-matrix-on-ubuntu-py3-11-3-12-3-13.yml b/.github/workflows/pytest-matrix-on-ubuntu-py3-11-3-12-3-13.yml deleted file mode 100644 index a82717b8..00000000 --- a/.github/workflows/pytest-matrix-on-ubuntu-py3-11-3-12-3-13.yml +++ /dev/null @@ -1,149 +0,0 @@ -name: tests - -on: - push: - branches: [main, develop] - pull_request: - branches: [main, develop] - schedule: - - cron: "0 17 * * *" # nightly 17:00 UTC (~02:00 JST), off-peak - workflow_dispatch: - -jobs: - test: - name: pytest-matrix-on-ubuntu-py${{ matrix.python-version }} - runs-on: ubuntu-latest - # Ceiling well below the GitHub-Actions 6h maximum so a genuine hang - # (wedged subprocess / coverage deadlock — the old chronic "tests" red) - # fails fast and diagnosably. The suite itself is single-digit minutes - # locally; in CI the long pole is the cold install of the whole ecosystem - # (`.[all,dev]` pulls 60+ scitex-* wheels + torch/scientific stack), which - # alone can approach 40min uncached — hence pip caching below + a generous - # cap so the first (cache-priming) run still completes. - timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - python-version: ["3.11", "3.12", "3.13"] - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install uv - # uv's resolver is what makes this matrix viable: pip backtracks through - # dozens of `wrapt` candidates (deprecated->numcodecs->scitex) at ~13s - # each, turning the install into a ~50min resolver-thrash that hit the - # job cap. uv resolves the same graph in seconds and caches wheels. - uses: astral-sh/setup-uv@v5 - with: - enable-cache: true - - name: Install dependencies - run: | - # CPU-only torch FIRST so the transitive torch (via scitex-nn/dsp) - # resolves to the CPU build. The default CUDA wheels pull ~600MB of - # nvidia-cuda-* libs that, imported on a GPU-less runner, SEGFAULT the - # suite (exit 139). Pre-satisfying torch from the CPU index avoids both - # the bloat and the crash. - uv pip install --system torch --index-url https://download.pytorch.org/whl/cpu \ - || echo "CPU torch pre-install skipped (torch may be optional in this resolve)" - # Fallback chain: prefer [all,dev] so optional-dep code paths - # (yaml, fastmcp, textual) get exercised. Falls back to [dev] then - # plain editable so a packaging hiccup doesn't strand CI. - uv pip install --system -e ".[all,dev]" \ - || uv pip install --system -e ".[dev]" \ - || uv pip install --system -e . - - name: Install peer standalones aliased by the umbrella - # The umbrella aliases scitex. -> scitex_ for several peers that - # are mid-extraction and not yet declared as hard dependencies (open - # extract/* PRs do that wiring). Install them here, unpinned, so the - # peer-identity gate (tests/scitex/test_subset_identity_peers.py) - # actually exercises them instead of importorskip-ing. Unpinned and - # best-effort: a peer that isn't on PyPI yet must not strand CI, and - # the identity test importorskips anything still absent. - run: | - uv pip install --system \ - scitex-dsp scitex-gen scitex-pd scitex-nn scitex-resource \ - scitex-linalg scitex-datetime scitex-decorators \ - || echo "peer standalones partially unavailable; identity test will importorskip the rest" - - name: Audit umbrella pin freshness (PS-170) - # Fails CI if any pin in pyproject.toml lags PyPI latest. - # See scitex-dev v0.11.19+ for the rule. Runs once per matrix - # row so a flake on one Python version doesn't block all three. - run: | - scitex-dev ecosystem audit-umbrella-pins . || exit 1 - - name: Run tests - # Coverage is only uploaded for py3.12 (see the Codecov step), so only - # pay the instrumentation cost there. 3.11/3.13 run plain pytest, which - # is faster and side-steps the subprocess-coverage shim entirely. - # faulthandler dumps every thread's traceback if a single test wedges, - # turning a future hang into an actionable log instead of a silent - # timeout. - # - # tests/e2e is excluded from the unit matrix: those are heavy - # full-stack subprocess journeys (session→io→stats) that spin up the - # entire ecosystem and are environment-fragile in CI (a hard crash - # there segfaults the whole run rather than failing one test). They - # belong in a dedicated e2e gate, not the per-PR unit matrix. - # - # Pass/fail is judged from the JUnit report, NOT the process exit code. - # The full suite passes (100%), but a C-extension in the deep ecosystem - # stack segfaults during *interpreter finalization* (after the last test - # PASSED), so the process exits 139 even though every test passed. That - # shutdown crash is not a test failure; reading results.xml decouples the - # logical result from a corrupted exit code. A genuine test failure shows - # up as failures/errors in the report and still fails the job. - # - # No coverage: --cov instrumentation triggers the C-extension segfault - # MID-run (not just at shutdown), killing pytest before the JUnit report - # is written, on the coverage row only. Coverage is advisory (codecov is - # non-blocking), so run plain pytest on every row for a stable matrix; - # reinstate coverage in a dedicated isolated job if it's wanted. - # - # tests/integration/test_cross_package_imports.py is also quarantined: - # it imports the full torch/numba-backed peer stack (scitex.dsp.utils.*, - # scitex_stats.descriptive, ...) which segfaults NON-DETERMINISTICALLY in - # CI — sometimes at interpreter shutdown (tolerated by the JUnit verdict), - # sometimes MID-import, before the report is written (un-catchable; takes - # the whole job 139). It's a heavy cross-package import gate, not a unit - # test; run it in a dedicated forked/isolated job, not the per-PR matrix. - # - # Retry-on-segfault: the deep ecosystem C-extension stack (torch/numba/ - # catboost/cuda) segfaults NON-DETERMINISTICALLY when many extensions - # co-load in one long-lived process — it can kill *any* test's process - # mid-run, before the JUnit report is written (so the JUnit verdict can't - # see it). It's a random dice-roll, not tied to a specific test. So if a - # run crashes before producing a report, retry (up to 3x); a run that - # FINISHES writes the report, and the verdict below judges it. A genuine - # test failure produces a report with failures and still fails the job — - # retry only papers over the crash-before-report, never a real failure. - run: | - set +e - for attempt in 1 2 3; do - rm -f pytest-results.xml - pytest tests/ --ignore=tests/e2e \ - --ignore=tests/integration/test_cross_package_imports.py \ - -p no:cacheprovider -o faulthandler_timeout=120 \ - --junitxml=pytest-results.xml - if [ -f pytest-results.xml ]; then - echo "pytest produced a report on attempt $attempt" - break - fi - echo "::warning::attempt $attempt crashed before writing the report (C-extension shutdown/import segfault); retrying" - done - set -e - python - <<'PY' - import sys, xml.etree.ElementTree as ET - try: - root = ET.parse("pytest-results.xml").getroot() - except Exception as e: - print(f"::error::no JUnit report after retries ({e}) — pytest never " - "finished; failing") - sys.exit(1) - suites = list(root.iter("testsuite")) - bad = sum(int(s.get("failures", 0)) + int(s.get("errors", 0)) for s in suites) - total = sum(int(s.get("tests", 0)) for s in suites) - print(f"JUnit: {total} tests, {bad} failures+errors") - sys.exit(1 if bad else 0) - PY diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..7c116ba2 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,78 @@ +name: release + +# ============================================================================ +# CANONICAL release pipeline — IDENTICAL across every scitex repo (synced by +# `scitex-dev ecosystem sync-workflows`). On a `v*` tag (cut on main after the +# deliberate develop->main promotion PR): build the dist, publish to PyPI +# **only if the repo is a pip package**, and cut a GitHub Release. +# +# Promotion develop->main is a DELIBERATE PR — there is intentionally NO +# sync-main job here. PyPI opt-out: drop a `.github/NO_PYPI` marker file. +# ============================================================================ + +on: + push: + tags: ["v*"] + workflow_dispatch: + +permissions: + contents: write # create the GitHub Release + id-token: write # PyPI trusted publishing (OIDC) + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Build sdist + wheel + run: | + python -m pip install --upgrade pip build + python -m build + - uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ + + publish-pypi: + needs: build + runs-on: ubuntu-latest + environment: pypi + steps: + - uses: actions/checkout@v4 + - name: Is this a pip package? + id: gate + run: | + if [ -f .github/NO_PYPI ]; then + echo "publish=false" >> "$GITHUB_OUTPUT" + echo "::notice::.github/NO_PYPI present — not a pip package, skipping PyPI publish" + else + echo "publish=true" >> "$GITHUB_OUTPUT" + fi + - if: steps.gate.outputs.publish == 'true' + uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + - name: Publish to PyPI (OIDC trusted publisher) + if: steps.gate.outputs.publish == 'true' + uses: pypa/gh-action-pypi-publish@release/v1 + + github-release: + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + - name: Create GitHub Release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release create "${{ github.ref_name }}" dist/* \ + --title "${{ github.ref_name }}" --generate-notes \ + || gh release upload "${{ github.ref_name }}" dist/* --clobber diff --git a/.github/workflows/rtd-sphinx-build-on-ubuntu-latest.yml b/.github/workflows/rtd-sphinx-build-on-ubuntu-latest.yml deleted file mode 100644 index 1dc137f5..00000000 --- a/.github/workflows/rtd-sphinx-build-on-ubuntu-latest.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: docs - -on: - push: - branches: [main, develop] - pull_request: - -jobs: - sphinx: - # Skip when HEAD is the bot's own previous docs auto-commit — breaks - # the infinite loop without using `[skip ci]`, which would also - # suppress the tag-push and release workflows when the auto-commit - # is the most recent commit on develop at release time. - if: "!contains(github.event.head_commit.message, 'docs(sphinx_html): refresh from CI build')" - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - token: ${{ secrets.GITHUB_TOKEN }} - - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install package + docs deps - run: pip install -e ".[docs]" - - - name: Build Sphinx HTML - # `-W` (warnings → errors) was previously enabled on PRs only, but - # the only warnings we surface come from autodoc'd docstrings in - # peer packages (e.g. scitex_config / scitex_session / - # figrecipe.pyplot's matplotlib re-exports). Those are upstream - # docstring issues we can't fix from this repo, and develop's - # build already runs without `-W`. Align PR builds with develop - # so we don't gate merges on cosmetic upstream noise. - run: sphinx-build -b html docs/sphinx docs/sphinx/_build/html - - - name: Refresh src/scitex/_sphinx_html/ (develop push only) - # main is PR-only — its _sphinx_html/ bundle arrives via the - # develop -> main PR. Direct bot-pushes to main are inhibited - # so the branch history stays linear-via-PR. - if: github.event_name == 'push' && github.ref == 'refs/heads/develop' - run: | - rm -rf src/scitex/_sphinx_html - cp -rf docs/sphinx/_build/html src/scitex/_sphinx_html - touch src/scitex/_sphinx_html/.nojekyll - - - name: Commit refreshed HTML if it changed (develop push only) - # Best-effort: develop is a protected branch, so the bot push is - # declined with GH006 ("protected branch hook declined"). The bundle - # only needs to be current for production serving, which is refreshed - # at release time via the develop -> main PR. The Sphinx BUILD step - # above remains strict/required; only this commit-back is non-fatal so - # the docs workflow stops going red on every develop push. - continue-on-error: true - if: github.event_name == 'push' && github.ref == 'refs/heads/develop' - run: | - if [ -n "$(git status --porcelain src/scitex/_sphinx_html)" ]; then - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add src/scitex/_sphinx_html - git commit -m "docs(sphinx_html): refresh from CI build" - git push - fi diff --git a/.github/workflows/scitex-quality-audit-on-ubuntu-latest.yml b/.github/workflows/scitex-quality-audit-on-ubuntu-latest.yml deleted file mode 100644 index 73a75226..00000000 --- a/.github/workflows/scitex-quality-audit-on-ubuntu-latest.yml +++ /dev/null @@ -1,140 +0,0 @@ -name: quality - -# Nightly ecosystem-wide static audit — catches regressions like the missing -# scitex-config dep that broke 6 PyPI packages on 2026-04-28. Outputs a JSON -# artifact and creates a GitHub issue if any CRITICAL/HIGH findings surface. - -on: - schedule: - - cron: "0 18 * * *" # 18:00 UTC (~03:00 JST), after Test - workflow_dispatch: - push: - branches: [develop] - paths: - - "scripts/quality/audit_ecosystem.py" - - "src/scitex_dev/_ecosystem/_core.py" - - "src/scitex_dev/ecosystem.py" - - ".github/workflows/quality-audit.yml" - -jobs: - audit: - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - name: Checkout scitex-dev - uses: actions/checkout@v4 - with: - path: scitex-dev - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - - name: Shallow-clone every ecosystem package - run: | - set -e - mkdir -p proj - python3 - <<'PY' - import json, re, subprocess, sys - from pathlib import Path - # 0.11.0 layout: ecosystem.py moved to _ecosystem/_core.py. - eco_new = Path("scitex-dev/src/scitex_dev/_ecosystem/_core.py") - eco_old = Path("scitex-dev/src/scitex_dev/ecosystem.py") - src = (eco_new if eco_new.is_file() else eco_old).read_text() - for m in re.finditer( - r'^\s*"([\w-]+)"\s*:\s*\{(.*?)\},?\s*$', src, - re.MULTILINE | re.DOTALL): - name, body = m.group(1), m.group(2) - g = re.search(r'"github_repo"\s*:\s*"([^"]+)"', body) - cat = re.search(r'"category"\s*:\s*"([^"]+)"', body) - category = cat.group(1) if cat else "library" - if category not in ("umbrella", "library", "external-lib"): - continue - if not g: - continue - dest = Path("proj") / name - if dest.exists(): - continue - print(f"clone {g.group(1)} -> {dest}") - r = subprocess.run( - ["git", "clone", "--depth", "1", "--no-tags", - f"https://github.com/{g.group(1)}.git", str(dest)], - capture_output=True, text=True) - if r.returncode: - print(f" WARN: {r.stderr.strip()[:200]}", file=sys.stderr) - PY - - - name: Run ecosystem audit - id: audit - run: | - python3 scitex-dev/scripts/quality/audit_ecosystem.py \ - --projects-root proj \ - --scitex-dev-root scitex-dev \ - --out audit.json - echo "json_path=$(pwd)/audit.json" >> "$GITHUB_OUTPUT" - # Surface summary in step output. - python3 - <<'PY' - import json - d = json.load(open("audit.json")) - s = d["summary"] - print(f"::notice ::Packages audited: {s['packages_audited']}") - print(f"::notice ::Findings by severity: {s['by_severity']}") - if s["by_severity"].get("CRITICAL", 0): - print(f"::error ::{s['by_severity']['CRITICAL']} CRITICAL findings — see audit.json") - if s["by_severity"].get("HIGH", 0): - print(f"::warning ::{s['by_severity']['HIGH']} HIGH findings — see audit.json") - PY - continue-on-error: true # never block the workflow on findings - - - name: Upload audit JSON - uses: actions/upload-artifact@v4 - with: - name: ecosystem-audit - path: audit.json - retention-days: 30 - - - name: Open / update tracking issue on CRITICAL or HIGH - if: ${{ always() }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - set -e - counts=$(python3 -c "import json; d=json.load(open('audit.json')); s=d['summary']['by_severity']; print(s.get('CRITICAL',0), s.get('HIGH',0))") - read crit high <<< "$counts" - if [[ "$crit" -eq 0 && "$high" -eq 0 ]]; then - echo "No CRITICAL/HIGH; skipping issue." - exit 0 - fi - # Build a markdown body listing offenders. - body=$(python3 - <<'PY' - import json - d = json.load(open("audit.json")) - out = ["# Ecosystem Quality Audit findings", ""] - out.append(f"Generated: {d['generated_at']}") - out.append(f"Packages: {d['summary']['packages_audited']}") - out.append(f"By severity: `{d['summary']['by_severity']}`") - out.append("") - for sev in ("CRITICAL", "HIGH"): - hits = [(p, f) for p in d["packages"] for f in p["findings"] if f["severity"] == sev] - if not hits: - continue - out.append(f"## {sev}") - for p, f in hits: - out.append(f"- **{p['package']}** §{f['section']} ({f['lens']}) — {f['message']}") - if f.get("detail"): - out.append(f" - {f['detail']}") - print("\n".join(out)) - PY - ) - # Reuse a single rolling issue per day. - title="quality-audit: $(date -u +%Y-%m-%d) — CRITICAL=$crit HIGH=$high" - existing=$(gh issue list -R "$GITHUB_REPOSITORY" --label quality-audit --state open --json number,title \ - | python3 -c "import sys,json; d=json.load(sys.stdin); print(next((x['number'] for x in d if x['title'].startswith('quality-audit:')), ''))") - if [[ -n "$existing" ]]; then - gh issue comment "$existing" -R "$GITHUB_REPOSITORY" --body "$body" - gh issue edit "$existing" -R "$GITHUB_REPOSITORY" --title "$title" - else - gh issue create -R "$GITHUB_REPOSITORY" --title "$title" --body "$body" \ - --label quality-audit - fi