diff --git a/.github/tier-policy.yml b/.github/tier-policy.yml new file mode 100644 index 0000000..24cb4f7 --- /dev/null +++ b/.github/tier-policy.yml @@ -0,0 +1,98 @@ +# SPDX-FileCopyrightText: 2026 European Space Agency (ESA) +# SPDX-License-Identifier: Apache-2.0 +# +# Tier decision policy used by `.github/workflows/ci.yml` (job: `tier-decision`). +# +# Why this file exists: +# - It is the single place where "what should escalate to tier 1?" is defined. +# - CI reads this policy from the PR BASE branch (not the PR head), so a PR +# cannot modify its own judge. +# +# Tier model (see scripts/ci_tier_decision.py): +# 0 — Baseline only +# 1 — Baseline + Extended (locked_paths, sme_owned_paths, marker fail, Dependabot major) +# 2 — Baseline + Extended + Heavy (tier_2_paths, VERSION→main, or manual run_heavy when tier≥1) +# +# Policy is read from base_sha (PR merge target, or develop on dispatch without PR). +# +# Glob semantics: +# - Patterns follow gitwildmatch syntax (similar to .gitignore). +# - `**` matches across directory boundaries. + +locked_paths: + # ---- Version files (root + per-processor) ---------------------------------- + - "VERSION" + - "pyproject.toml" + - "bps-*/pyproject.toml" + + # ---- CI configuration and repository governance ---------------------------- + - ".github/workflows/**" + - ".github/CODEOWNERS" + - ".github/dependabot.yml" + - ".github/PULL_REQUEST_TEMPLATE.md" + - ".github/tier-policy.yml" + + # ---- Commit-time and lint configuration ----------------------------------- + - ".pre-commit-config.yaml" + - "ruff.toml" + - "noxfile.py" + + # ---- Licensing and REUSE compliance --------------------------------------- + - "LICENSES/**" + - "REUSE.toml" + + # ---- Test harness, markers, and baselines (the judge itself) -------------- + - "pytest.ini" + - "test/baseline/**" + - "test/extended/**" + - "test/heavy/**" + - "**/test/baseline/**" + - "**/test/smoke/**" + + # ---- Calibration / auxiliary inputs to processors ------------------------- + - "bps-l1_pre_processor/bps/l1_pre_processor/aux_ins/**" + - "bps-l1_pre_processor/test/aux_ins/**" + - "bps-l1_processor/bps/l1_processor/aux_tec/**" + - "bps-transcoder/bps/transcoder/auxiliaryfiles/**" + +# Paths that force tier 2 (Heavy) in addition to Extended. +# Leave empty to rely on promotion.version_to_main_is_tier_2 and manual run_heavy only. +tier_2_paths: [] + +# Promotion / release rules (tier 2 = Heavy required). +promotion: + # PR targeting main that changes VERSION → tier 2 (ESA review expected at merge time). + version_to_main_is_tier_2: true + +# Dependabot: semver-major dependency PRs → tier 1 (Extended). +dependabot: + major_bump_is_tier_1: true + +# Paths that require SME escalation. +# If any of these change, tier is elevated and Extended runs. +sme_owned_paths: + - "bps-common/**" + - "bps-task-tables/**" + - "bps-transcoder/**" + - "bps-l1_binaries/**" + - "bps-l1_core_processor/**" + - "bps-l1_framing_processor/**" + - "bps-l1_pre_processor/**" + - "bps-l1_processor/**" + - "bps-l2a_processor/**" + - "bps-l2b_agb_processor/**" + - "bps-l2b_fd_processor/**" + - "bps-l2b_fh_processor/**" + - "bps-stack_binaries/**" + - "bps-stack_cal_processor/**" + - "bps-stack_coreg_processor/**" + - "bps-stack_pre_processor/**" + - "bps-stack_processor/**" + - "**/test/baseline/**" + - "**/test/smoke/**" + - "test/extended/**" + - "test/heavy/**" + +# Marker used for baseline signal tests. +# Must match a marker declared in `pytest.ini`. +baseline_marker: "baseline" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..b205c19 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,871 @@ +# SPDX-FileCopyrightText: 2026 European Space Agency (ESA) - ACRI-ST +# SPDX-License-Identifier: Apache-2.0 +--- +name: CI + +on: + pull_request: + branches: ["main", "develop", "release"] + workflow_dispatch: + inputs: + pr_number: + description: "Pull request number to process (optional: auto-detected from selected branch)" + required: false + type: string + run_heavy: + description: "Run Heavy after Extended" + required: false + default: false + type: boolean + +permissions: + contents: read + pull-requests: read + +jobs: + context: + name: Context + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + outputs: + pr_number: ${{ steps.resolve.outputs.pr_number }} + head_sha: ${{ steps.resolve.outputs.head_sha }} + base_sha: ${{ steps.resolve.outputs.base_sha }} + base_ref: ${{ steps.resolve.outputs.base_ref }} + run_heavy: ${{ steps.resolve.outputs.run_heavy }} + steps: + - id: resolve + uses: actions/github-script@v8 + env: + # Pass inputs via env to avoid script-injection through ${{ }} interpolation. + MANUAL_PR_NUMBER: ${{ inputs.pr_number }} + MANUAL_RUN_HEAVY: ${{ inputs.run_heavy }} + with: + script: | + const { owner, repo } = context.repo; + + const resolvePrNumber = async () => { + // 1) pull_request event: use triggering PR number. + if (context.eventName === "pull_request") { + return context.payload.pull_request.number; + } + + // 2) workflow_dispatch with explicit pr_number. + const manualInput = (process.env.MANUAL_PR_NUMBER || "").trim(); + if (manualInput) { + const parsed = Number.parseInt(manualInput, 10); + if (!Number.isInteger(parsed) || parsed <= 0) { + core.setFailed(`Invalid pr_number input: '${manualInput}'.`); + return null; + } + return parsed; + } + + // 3) workflow_dispatch without pr_number: auto-detect from branch. + const branchName = (context.ref || "").replace(/^refs\/heads\//, ""); + if (!branchName) { + core.setFailed("Could not detect branch name for workflow_dispatch. Please provide pr_number."); + return null; + } + + const { data: prs } = await github.rest.pulls.list({ + owner, + repo, + state: "open", + head: `${owner}:${branchName}`, + per_page: 2 + }); + + if (prs.length === 0) { + core.notice(`No open PR found for branch '${branchName}'. Running in branch mode without PR context.`); + return null; + } + if (prs.length > 1) { + core.setFailed(`Multiple open PRs found for branch '${branchName}'. Provide pr_number manually.`); + return null; + } + + core.notice(`Auto-detected PR #${prs[0].number} from branch '${branchName}'.`); + return prs[0].number; + }; + + const prNumber = await resolvePrNumber(); + // run_heavy only applies to workflow_dispatch; defaults to false on PR events. + const runHeavy = + context.eventName !== "pull_request" && process.env.MANUAL_RUN_HEAVY === "true" + ? "true" + : "false"; + + const { data: repoData } = await github.rest.repos.get({ owner, repo }); + const defaultBranch = repoData.default_branch || "main"; + let baseRef = "develop"; + let baseSha = ""; + try { + const { data: baseBranch } = await github.rest.repos.getBranch({ + owner, + repo, + branch: baseRef + }); + baseSha = baseBranch.commit.sha; + } catch { + baseRef = defaultBranch; + const { data: baseBranch } = await github.rest.repos.getBranch({ + owner, + repo, + branch: baseRef + }); + baseSha = baseBranch.commit.sha; + core.notice(`Branch 'develop' not found. Falling back to '${baseRef}' as compare base.`); + } + + let headSha = context.sha; + if (prNumber !== null) { + const { data: pr } = await github.rest.pulls.get({ + owner, + repo, + pull_number: prNumber + }); + headSha = pr.head.sha; + baseSha = pr.base.sha; + baseRef = pr.base.ref; + core.setOutput("pr_number", String(prNumber)); + } else { + core.setOutput("pr_number", ""); + core.notice(`workflow_dispatch without PR: using base '${baseRef}' (${baseSha}) and head '${headSha}'.`); + } + + core.setOutput("head_sha", headSha); + core.setOutput("base_sha", baseSha); + core.setOutput("base_ref", baseRef); + core.setOutput("run_heavy", runHeavy); + + baseline-marker-signal: + name: Baseline / Marker signal + needs: [context] + if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + outputs: + baseline_ok: ${{ steps.signal.outputs.baseline_ok }} + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ needs.context.outputs.head_sha }} + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Run baseline marker and collect signal + id: signal + run: | + echo "Running baseline marker on ref: ${{ needs.context.outputs.head_sha }}" + python --version + python -m pip install --upgrade pip pytest + pytest --version + if [ ! -d test/baseline ]; then + echo "baseline_ok=false" >> "$GITHUB_OUTPUT" + echo "::notice::Missing test/baseline. Tier will be elevated." + exit 0 + fi + set +e + pytest test/baseline -m baseline -q --maxfail=1 + rc=$? + set -e + if [ "$rc" -eq 0 ]; then + echo "baseline_ok=true" >> "$GITHUB_OUTPUT" + else + echo "baseline_ok=false" >> "$GITHUB_OUTPUT" + echo "::notice::Baseline marker differs/fails; tier will be elevated." + fi + + baseline-dco: + name: Baseline / DCO + needs: [context] + if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Verify Signed-off-by trailer on each non-merge commit + env: + BASE_SHA: ${{ needs.context.outputs.base_sha }} + HEAD_SHA: ${{ needs.context.outputs.head_sha }} + run: | + echo "Running DCO check from $BASE_SHA to $HEAD_SHA" + failed=0 + + normalize_identity() { + printf "%s" "$1" \ + | tr '[:upper:]' '[:lower:]' \ + | sed -E 's/[[:space:]]+/ /g; s/^ //; s/ $//' + } + + for sha in $(git rev-list "$BASE_SHA".."$HEAD_SHA"); do + parent_count=$(git rev-list --parents -n 1 "$sha" | awk '{print NF - 1}') + if [ "$parent_count" -gt 1 ]; then + continue + fi + + author_identity=$(git log -1 --format='%an <%ae>' "$sha") + committer_identity=$(git log -1 --format='%cn <%ce>' "$sha") + author_norm=$(normalize_identity "$author_identity") + committer_norm=$(normalize_identity "$committer_identity") + + signoffs=$(git log -1 --format=%B "$sha" \ + | awk 'BEGIN{IGNORECASE=1} /^Signed-off-by:[[:space:]]*/ {sub(/^Signed-off-by:[[:space:]]*/, "", $0); print $0}') + + if [ -z "$signoffs" ]; then + echo "::error::Commit $sha is missing Signed-off-by trailer." + failed=1 + continue + fi + + match_found=0 + while IFS= read -r signoff; do + [ -z "$signoff" ] && continue + signoff_norm=$(normalize_identity "$signoff") + if [ "$signoff_norm" = "$author_norm" ] || [ "$signoff_norm" = "$committer_norm" ]; then + match_found=1 + break + fi + done < <(printf '%s\n' "$signoffs") + + if [ "$match_found" -ne 1 ]; then + echo "::error::Commit $sha has Signed-off-by trailer(s), but none matches author '$author_identity' or committer '$committer_identity'." + failed=1 + fi + done + [ "$failed" -eq 0 ] || exit 1 + + baseline-reuse: + name: Baseline / REUSE + needs: [context] + if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ needs.context.outputs.head_sha }} + - name: REUSE context + run: | + echo "Running REUSE on ref: ${{ needs.context.outputs.head_sha }}" + - uses: fsfe/reuse-action@v6 + + baseline-pre-commit: + name: Baseline / Pre-commit + needs: [context] + if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + env: + PRE_COMMIT_HOME: ~/.cache/pre-commit + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ needs.context.outputs.head_sha }} + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Cache pre-commit environments + uses: actions/cache@v4 + with: + path: ~/.cache/pre-commit + key: pre-commit-${{ runner.os }}-py312-${{ hashFiles('.pre-commit-config.yaml') }} + restore-keys: | + pre-commit-${{ runner.os }}-py312- + - name: Run pre-commit + run: | + echo "Running pre-commit on ref: ${{ needs.context.outputs.head_sha }}" + python -m pip install --upgrade pip pre-commit + pre-commit --version + pre-commit run --all-files + + baseline-security: + name: Baseline / Security + needs: [context] + if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ needs.context.outputs.head_sha }} + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Run bandit + run: | + echo "Running Bandit on ref: ${{ needs.context.outputs.head_sha }}" + python -m pip install --upgrade pip "bandit[toml]" + bandit --version + bandit -r . -f json -o bandit-report.json -ll -ii --exit-zero + - name: Upload bandit report + uses: actions/upload-artifact@v4 + with: + name: bandit-report + path: bandit-report.json + + baseline-build: + name: Baseline / Build + needs: [context] + if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ needs.context.outputs.head_sha }} + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Build package + run: | + echo "Building on ref: ${{ needs.context.outputs.head_sha }}" + python --version + python -m pip install --upgrade pip setuptools wheel build + python -m build --version + python -m build --no-isolation || echo "::warning::Monorepo root build is non-blocking during governance migration." + - name: Upload build artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: baseline-build-dist + path: dist/* + if-no-files-found: ignore + + baseline-sensitive-files: + name: Baseline / Sensitive files + needs: [context] + if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ needs.context.outputs.head_sha }} + - name: Detect forbidden files + run: | + MAX_SIZE_MB=10 + if find . -type f -name "*.bak" -not -path "./.git/*" | grep -q .; then + echo "::error::Found forbidden *.bak file(s)." + find . -type f -name "*.bak" -not -path "./.git/*" + exit 1 + fi + if find . -type f -not -path "./.git/*" -size "+${MAX_SIZE_MB}M" | grep -q .; then + echo "::error::Found file(s) larger than ${MAX_SIZE_MB} MB." + find . -type f -not -path "./.git/*" -size "+${MAX_SIZE_MB}M" + exit 1 + fi + + baseline-unit-tests: + name: Baseline / Unit tests + needs: [context] + if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ needs.context.outputs.head_sha }} + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Run pytest unit marker + run: | + echo "Running unit tests on ref: ${{ needs.context.outputs.head_sha }}" + python --version + python -m pip install --upgrade pip pytest + pytest --version + if [ -f test/requirements.txt ]; then + python -m pip install -r test/requirements.txt || true + fi + set +e + pytest -m unit -q + rc=$? + set -e + if [ "$rc" -ne 0 ]; then + echo "::warning::Unit tests are non-blocking during governance migration." + fi + + baseline-docs: + name: Baseline / Docs + needs: [context] + if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ needs.context.outputs.head_sha }} + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Build docs when Sphinx config exists + run: | + echo "Building docs on ref: ${{ needs.context.outputs.head_sha }}" + python --version + python -m pip install --upgrade pip sphinx sphinx-rtd-theme myst-parser + if [ -f docs/api/conf.py ]; then + sphinx-build -b html docs/api docs/_build/html -W --keep-going + else + echo "No docs/api/conf.py found; skipping Sphinx build." + fi + - name: Upload docs artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: baseline-docs-html + path: docs/_build/html + if-no-files-found: ignore + + baseline-dependabot: + name: Baseline / Dependabot governance + needs: [context] + if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + pull-requests: read + contents: read + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ needs.context.outputs.head_sha }} + - name: Verify Dependabot config exists + run: | + [ -f .github/dependabot.yml ] || { + echo "::error::.github/dependabot.yml is missing." + exit 1 + } + - name: Report major update signal + uses: actions/github-script@v8 + with: + script: | + const prNumber = Number("${{ needs.context.outputs.pr_number }}"); + if (!Number.isInteger(prNumber) || prNumber <= 0) { + core.info("No PR context available (manual branch mode). Skipping Dependabot major update signal."); + return; + } + const prResp = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber + }); + const pr = prResp.data; + if (pr.user.login !== "dependabot[bot]") { + core.info("Not a Dependabot PR."); + return; + } + const title = pr.title || ""; + const match = title.match(/from\s+(\d+)\.[^\s]*\s+to\s+(\d+)\.[^\s]*/); + if (match && match[1] !== match[2]) { + core.notice("Dependabot major update detected. Tier escalation is handled by tier-decision."); + return; + } + core.info("Dependabot PR is not a major bump based on title signal."); + + baseline-gate: + name: Baseline / Gate + needs: + - baseline-marker-signal + - baseline-dco + - baseline-reuse + - baseline-pre-commit + - baseline-security + - baseline-build + - baseline-sensitive-files + - baseline-unit-tests + - baseline-docs + - baseline-dependabot + if: always() + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + outputs: + baseline_ok: ${{ steps.out.outputs.baseline_ok }} + steps: + - id: out + env: + EVENT_NAME: ${{ github.event_name }} + MARKER_RESULT: ${{ needs.baseline-marker-signal.result }} + DCO_RESULT: ${{ needs.baseline-dco.result }} + REUSE_RESULT: ${{ needs.baseline-reuse.result }} + PRE_COMMIT_RESULT: ${{ needs.baseline-pre-commit.result }} + SECURITY_RESULT: ${{ needs.baseline-security.result }} + BUILD_RESULT: ${{ needs.baseline-build.result }} + SENSITIVE_RESULT: ${{ needs.baseline-sensitive-files.result }} + UNIT_RESULT: ${{ needs.baseline-unit-tests.result }} + DOCS_RESULT: ${{ needs.baseline-docs.result }} + DEPENDABOT_RESULT: ${{ needs.baseline-dependabot.result }} + run: | + check_result() { + name="$1" + value="$2" + if [ "$value" != "success" ]; then + echo "::error::Required job '$name' finished with '$value'." + failed=1 + fi + } + + failed=0 + if [ "$EVENT_NAME" = "pull_request" ] || [ "$EVENT_NAME" = "workflow_dispatch" ]; then + check_result "baseline-marker-signal" "$MARKER_RESULT" + check_result "baseline-dco" "$DCO_RESULT" + check_result "baseline-reuse" "$REUSE_RESULT" + check_result "baseline-pre-commit" "$PRE_COMMIT_RESULT" + check_result "baseline-security" "$SECURITY_RESULT" + check_result "baseline-build" "$BUILD_RESULT" + check_result "baseline-sensitive-files" "$SENSITIVE_RESULT" + check_result "baseline-unit-tests" "$UNIT_RESULT" + check_result "baseline-docs" "$DOCS_RESULT" + check_result "baseline-dependabot" "$DEPENDABOT_RESULT" + fi + + [ "$failed" -eq 0 ] || exit 1 + echo "baseline_ok=true" >> "$GITHUB_OUTPUT" + + # Tier 0/1/2 after baseline. Runs even when baseline failed (PR guidance). + tier-decision: + name: Tier decision + needs: [context, baseline-marker-signal, baseline-gate] + if: >- + always() + && (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch') + && needs.baseline-gate.result != 'skipped' + && needs.baseline-marker-signal.result != 'skipped' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + pull-requests: read + outputs: + tier: ${{ steps.decide.outputs.tier }} + reasons: ${{ steps.decide.outputs.reasons }} + reasons_json: ${{ steps.decide.outputs.reasons_json }} + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Install tier-decision dependencies + run: python -m pip install --upgrade pip pyyaml + - id: decide + env: + REPO: ${{ github.repository }} + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ needs.context.outputs.pr_number }} + HEAD_SHA: ${{ needs.context.outputs.head_sha }} + BASE_SHA: ${{ needs.context.outputs.base_sha }} + BASE_REF: ${{ needs.context.outputs.base_ref }} + BASELINE_OK: ${{ needs.baseline-marker-signal.outputs.baseline_ok }} + RUN_HEAVY: ${{ needs.context.outputs.run_heavy }} + run: python scripts/ci_tier_decision.py + + extended: + name: Extended + needs: [context, baseline-gate, tier-decision] + if: >- + needs.baseline-gate.result == 'success' + && needs.tier-decision.result == 'success' + && needs.tier-decision.outputs.tier != '0' + runs-on: ubuntu-latest + timeout-minutes: 35 + permissions: + contents: read + steps: + # Handover note for ARESYS: + # - Put integration checks in test/extended + # - Mark tests with @pytest.mark.extended + # - Current mode is intentionally non-blocking while test harness/TDS wiring is in progress + - name: Checkout analyzed revision + uses: actions/checkout@v6 + with: + ref: ${{ needs.context.outputs.head_sha }} + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Install pytest runtime + run: python -m pip install --upgrade pip pytest + - name: Run Extended suite (non-blocking during migration) + run: | + TEST_DIR="test/extended" + TEST_MARKER="extended" + + # If no suite exists yet, keep CI green but emit a clear signal. + if [ ! -d "$TEST_DIR" ]; then + echo "::notice::No $TEST_DIR directory yet. Create it with @$TEST_MARKER tests to activate Extended validation." + exit 0 + fi + + # Temporary policy: report failures without blocking merges. + pytest "$TEST_DIR" -m "$TEST_MARKER" -q --maxfail=1 || { + echo "::warning::Extended tests failed. Still non-blocking until harness/TDS wiring is finalized." + exit 0 + } + + heavy: + name: Heavy + needs: [context, baseline-gate, tier-decision, extended] + if: >- + needs.baseline-gate.result == 'success' + && needs.tier-decision.result == 'success' + && needs.tier-decision.outputs.tier == '2' + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: read + steps: + # Handover note for ARESYS: + # - Put deep/high-cost checks in test/heavy + # - Mark tests with @pytest.mark.heavy + # - This stage only runs when tier-decision upclasses to tier 2 + - name: Checkout analyzed revision + uses: actions/checkout@v6 + with: + ref: ${{ needs.context.outputs.head_sha }} + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Install pytest runtime + run: python -m pip install --upgrade pip pytest + - name: Run Heavy suite (non-blocking during migration) + run: | + TEST_DIR="test/heavy" + TEST_MARKER="heavy" + + # If no suite exists yet, keep CI green but emit a clear signal. + if [ ! -d "$TEST_DIR" ]; then + echo "::notice::No $TEST_DIR directory yet. Create it with @$TEST_MARKER tests to activate Heavy validation." + exit 0 + fi + + # Temporary policy: report failures without blocking merges. + pytest "$TEST_DIR" -m "$TEST_MARKER" -q --maxfail=1 || { + echo "::warning::Heavy tests failed. Still non-blocking until harness/TDS wiring is finalized." + exit 0 + } + + pr-guidance-bot: + name: PR guidance comment + needs: + - context + - baseline-gate + - tier-decision + - extended + - heavy + if: always() && (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch') + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + pull-requests: write + issues: write + steps: + - name: Create or update sticky PR guidance comment + uses: actions/github-script@v8 + env: + PR_NUMBER: ${{ needs.context.outputs.pr_number }} + BASELINE_GATE: ${{ needs.baseline-gate.result }} + TIER: ${{ needs.tier-decision.outputs.tier }} + TIER_REASONS: ${{ needs.tier-decision.outputs.reasons }} + TIER_REASONS_JSON: ${{ needs.tier-decision.outputs.reasons_json }} + EXTENDED_RESULT: ${{ needs.extended.result }} + HEAVY_RESULT: ${{ needs.heavy.result }} + RUN_HEAVY: ${{ needs.context.outputs.run_heavy }} + with: + script: | + const marker = ""; + const prNumber = Number(process.env.PR_NUMBER); + if (!Number.isInteger(prNumber) || prNumber <= 0) { + core.info("No valid PR number. Skipping PR guidance comment."); + return; + } + + const baseline = process.env.BASELINE_GATE || "unknown"; + const tier = process.env.TIER || "unknown"; + const reasonsJsonRaw = process.env.TIER_REASONS_JSON || ""; + let reasonsList = []; + if (reasonsJsonRaw) { + try { + const parsed = JSON.parse(reasonsJsonRaw); + if (Array.isArray(parsed)) reasonsList = parsed.filter(Boolean).map(String); + } catch (error) { + core.warning(`Invalid TIER_REASONS_JSON payload: ${error.message}`); + } + } + if (reasonsList.length === 0) { + const fallback = process.env.TIER_REASONS || "No tier reason available."; + reasonsList = [fallback]; + } + const reasons = reasonsList.join(" | "); + const reasonsMarkdown = reasonsList.map((reason) => `- ${reason}`).join("\n"); + const extended = process.env.EXTENDED_RESULT || "skipped"; + const heavy = process.env.HEAVY_RESULT || "skipped"; + const runHeavy = (process.env.RUN_HEAVY || "false") === "true"; + const eventName = context.eventName || "unknown"; + const executionMode = eventName === "workflow_dispatch" + ? "workflow_dispatch" + : "pull_request"; + + const icon = (result) => { + if (result === "success") return "✅"; + if (result === "failure" || result === "cancelled" || result === "timed_out") return "❌"; + if (result === "skipped") return "⏭️"; + return "ℹ️"; + }; + + const tierLine = tier === "0" + ? "Tier 0: baseline only (no Extended / Heavy required)." + : (tier === "1" + ? "Tier 1: Extended required (integration / governance paths)." + : "Tier 2: Extended + Heavy required (promotion, tier_2_paths, or manual run_heavy)."); + + const heavyLine = tier === "0" + ? "Heavy: not required." + : (tier === "1" + ? "Heavy: optional — run workflow_dispatch with `run_heavy=true` to reach tier 2." + : (runHeavy + ? `${icon(heavy)} Heavy: **${heavy}** (tier 2 via \`run_heavy=true\`).` + : `${icon(heavy)} Heavy: **${heavy}** (required for tier 2).`)); + + const actions = []; + if (baseline !== "success") { + actions.push("Fix Baseline checks first. Extended/Heavy decisions are secondary until Baseline is green."); + } + if (tier !== "0" && (extended === "failure" || extended === "cancelled" || extended === "timed_out")) { + actions.push("Review Extended logs and resolve failing checks before merge."); + } + if (tier !== "0" && extended === "skipped" && baseline !== "success") { + actions.push("Extended was skipped because Baseline is not green yet."); + } + if (tier === "2" && (heavy === "failure" || heavy === "cancelled" || heavy === "timed_out")) { + actions.push("Heavy was requested but did not pass. Check Heavy logs before approval."); + } + if (tier === "2" && heavy === "skipped") { + actions.push("Heavy is required for Tier 2. Verify why the Heavy stage was skipped."); + } + if (tier === "1" && !runHeavy) { + actions.push("If this change is high impact, run Heavy manually from Actions using `run_heavy=true`."); + } + if (actions.length === 0) { + actions.push("No immediate action required. CI status looks good for the current policy."); + } + + const body = `${marker} + ## CI status summary + + ${icon(baseline)} Baseline gate: **${baseline}** + Tier decision: **${tier}** + ${tierLine} + ${icon(extended)} Extended result: **${extended}** + ${heavyLine} + ℹExecution mode: **${executionMode}** + + **Tier rationale (summary):** ${reasons} + **Tier rationale (details):** + ${reasonsMarkdown} + + ## Suggested next actions + ${actions.map((a) => `- ${a}`).join("\n")} + `; + + const { owner, repo } = context.repo; + const comments = await github.paginate( + github.rest.issues.listComments, + { owner, repo, issue_number: prNumber, per_page: 100 } + ); + const existing = comments.find((comment) => + comment.user?.type === "Bot" && typeof comment.body === "string" && comment.body.includes(marker) + ); + + if (existing) { + await github.rest.issues.updateComment({ + owner, + repo, + comment_id: existing.id, + body + }); + core.info(`Updated sticky guidance comment: ${existing.id}`); + } else { + const created = await github.rest.issues.createComment({ + owner, + repo, + issue_number: prNumber, + body + }); + core.info(`Created sticky guidance comment: ${created.data.id}`); + } + + ci-gate: + name: CI gate + needs: + - context + - baseline-gate + - tier-decision + - extended + - heavy + if: always() + runs-on: ubuntu-latest + steps: + - env: + TIER: ${{ needs.tier-decision.outputs.tier }} + RUN_HEAVY: ${{ needs.context.outputs.run_heavy }} + BASELINE_GATE: ${{ needs.baseline-gate.result }} + EXTENDED_RESULT: ${{ needs.extended.result }} + HEAVY_RESULT: ${{ needs.heavy.result }} + run: | + [ "$BASELINE_GATE" = "success" ] || { echo "::error::Baseline gate failed."; exit 1; } + + case "$TIER" in + 0) + echo "Tier 0: baseline gate is sufficient." + ;; + 1) + [ "$EXTENDED_RESULT" = "success" ] || { + echo "::error::Tier 1 requires Extended to pass." + exit 1 + } + echo "Tier 1: baseline + extended passed." + ;; + 2) + [ "$EXTENDED_RESULT" = "success" ] || { + echo "::error::Tier 2 requires Extended to pass." + exit 1 + } + [ "$HEAVY_RESULT" = "success" ] || { + echo "::error::Tier 2 requires Heavy to pass." + exit 1 + } + echo "Tier 2: baseline + extended + heavy passed." + ;; + *) + echo "::error::Unknown tier '$TIER' (tier-decision may have been skipped)." + exit 1 + ;; + esac + echo "CI gate passed." diff --git a/.github/workflows/extended-ci.yml b/.github/workflows/extended-ci.yml new file mode 100644 index 0000000..2e01e70 --- /dev/null +++ b/.github/workflows/extended-ci.yml @@ -0,0 +1,137 @@ +# SPDX-FileCopyrightText: 2026 European Space Agency (ESA) +# SPDX-License-Identifier: Apache-2.0 +--- +name: Extended CI + +on: + workflow_dispatch: + inputs: + pr_number: + description: "Pull request number to validate" + required: true + type: string + processors: + description: "Comma-separated processors or 'all'" + required: true + default: "all" + type: string + dataset_path: + description: "Optional dataset path or URL (TBD)" + required: false + default: "" + type: string + +permissions: + contents: read + pull-requests: read + +jobs: + validate-scope: + name: Scope validation + runs-on: ubuntu-latest + environment: extended-ci + outputs: + selected_processors: ${{ steps.scope.outputs.selected_processors }} + touched_processors: ${{ steps.scope.outputs.touched_processors }} + steps: + - name: Validate requested processors against PR diff + id: scope + uses: actions/github-script@v8 + with: + script: | + const prNumber = Number("${{ inputs.pr_number }}"); + if (!Number.isInteger(prNumber) || prNumber <= 0) { + core.setFailed("Input pr_number must be a positive integer."); + return; + } + + const knownProcessors = [ + "bps-common", + "bps-task-tables", + "bps-transcoder", + "bps-l1_binaries", + "bps-l1_core_processor", + "bps-l1_framing_processor", + "bps-l1_pre_processor", + "bps-l1_processor", + "bps-l2a_processor", + "bps-l2b_agb_processor", + "bps-l2b_fd_processor", + "bps-l2b_fh_processor", + "bps-stack_binaries", + "bps-stack_cal_processor", + "bps-stack_coreg_processor", + "bps-stack_pre_processor", + "bps-stack_processor" + ]; + + const files = await github.paginate(github.rest.pulls.listFiles, { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + per_page: 100 + }); + const touched = new Set(); + for (const file of files) { + const topLevel = (file.filename || "").split("/")[0]; + if (knownProcessors.includes(topLevel)) { + touched.add(topLevel); + } + } + + const raw = ("${{ inputs.processors }}" || "").trim(); + const requested = raw.toLowerCase() === "all" + ? Array.from(touched) + : raw.split(",").map(p => p.trim()).filter(Boolean); + + if (requested.length === 0) { + core.notice( + `No processor directories touched by PR #${prNumber}. Extended CI is pass-through for now.` + ); + core.setOutput("selected_processors", ""); + core.setOutput("touched_processors", Array.from(touched).join(",")); + return; + } + + const invalid = requested.filter(p => !knownProcessors.includes(p)); + if (invalid.length > 0) { + core.setFailed(`Unknown processor(s): ${invalid.join(", ")}`); + return; + } + + const notTouched = requested.filter(p => !touched.has(p)); + if (notTouched.length > 0) { + core.setFailed( + `Requested processor(s) not touched by PR #${prNumber}: ${notTouched.join(", ")}` + ); + return; + } + + core.info(`Touched processors: ${Array.from(touched).join(", ")}`); + core.info(`Selected processors: ${requested.join(", ")}`); + core.setOutput("selected_processors", requested.join(",")); + core.setOutput("touched_processors", Array.from(touched).join(",")); + + run-extended: + name: Extended tests + needs: validate-scope + if: needs.validate-scope.outputs.selected_processors != '' + runs-on: ubuntu-latest + environment: extended-ci + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Install pytest + run: python -m pip install --upgrade pip pytest + - name: Run extended marker suite + run: | + if [ ! -d test/extended ]; then + echo "::notice::test/extended is missing; extended pipeline is pass-through for now." + exit 0 + fi + pytest test/extended -m extended -q --maxfail=1 || { + echo "::warning::Extended tests are temporary non-blocking until harness is wired." + exit 0 + } diff --git a/.github/workflows/heavy-ci.yml b/.github/workflows/heavy-ci.yml new file mode 100644 index 0000000..6a86b26 --- /dev/null +++ b/.github/workflows/heavy-ci.yml @@ -0,0 +1,168 @@ +# SPDX-FileCopyrightText: 2026 European Space Agency (ESA) +# SPDX-License-Identifier: Apache-2.0 +--- +name: Heavy CI + +on: + workflow_dispatch: + inputs: + pr_number: + description: "Pull request number to validate" + required: true + type: string + processors: + description: "Comma-separated processors or 'all'" + required: true + default: "all" + type: string + dataset_path: + description: "Optional dataset path or URL (TBD)" + required: false + default: "" + type: string + schedule: + - cron: "0 2 * * 0" + +permissions: + contents: read + pull-requests: read + +jobs: + validate-scope: + name: Scope validation + runs-on: ubuntu-latest + environment: heavy-ci + outputs: + selected_processors: ${{ steps.scope.outputs.selected_processors }} + touched_processors: ${{ steps.scope.outputs.touched_processors }} + steps: + - name: Validate requested processors against PR diff + id: scope + uses: actions/github-script@v8 + with: + script: | + const knownProcessors = [ + "bps-common", + "bps-task-tables", + "bps-transcoder", + "bps-l1_binaries", + "bps-l1_core_processor", + "bps-l1_framing_processor", + "bps-l1_pre_processor", + "bps-l1_processor", + "bps-l2a_processor", + "bps-l2b_agb_processor", + "bps-l2b_fd_processor", + "bps-l2b_fh_processor", + "bps-stack_binaries", + "bps-stack_cal_processor", + "bps-stack_coreg_processor", + "bps-stack_pre_processor", + "bps-stack_processor" + ]; + + if (context.eventName === "schedule") { + core.info("Scheduled run: forcing all processors."); + core.setOutput("selected_processors", knownProcessors.join(",")); + core.setOutput("touched_processors", knownProcessors.join(",")); + return; + } + + const prNumber = Number("${{ inputs.pr_number }}"); + if (!Number.isInteger(prNumber) || prNumber <= 0) { + core.setFailed("Input pr_number must be a positive integer for workflow_dispatch."); + return; + } + + const files = await github.paginate(github.rest.pulls.listFiles, { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + per_page: 100 + }); + const touched = new Set(); + for (const file of files) { + const topLevel = (file.filename || "").split("/")[0]; + if (knownProcessors.includes(topLevel)) { + touched.add(topLevel); + } + } + + const raw = ("${{ inputs.processors }}" || "").trim(); + const requested = raw.toLowerCase() === "all" + ? Array.from(touched) + : raw.split(",").map(p => p.trim()).filter(Boolean); + + if (requested.length === 0) { + core.notice( + `No processor directories touched by PR #${prNumber}. Heavy CI is pass-through for now.` + ); + core.setOutput("selected_processors", ""); + core.setOutput("touched_processors", Array.from(touched).join(",")); + return; + } + + const invalid = requested.filter(p => !knownProcessors.includes(p)); + if (invalid.length > 0) { + core.setFailed(`Unknown processor(s): ${invalid.join(", ")}`); + return; + } + + const notTouched = requested.filter(p => !touched.has(p)); + if (notTouched.length > 0) { + core.setFailed( + `Requested processor(s) not touched by PR #${prNumber}: ${notTouched.join(", ")}` + ); + return; + } + + core.setOutput("selected_processors", requested.join(",")); + core.setOutput("touched_processors", Array.from(touched).join(",")); + + run-extended-prereq: + name: Extended prerequisite + needs: validate-scope + if: needs.validate-scope.outputs.selected_processors != '' + runs-on: ubuntu-latest + environment: heavy-ci + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Install pytest + run: python -m pip install --upgrade pip pytest + - name: Run extended marker suite first + run: | + if [ ! -d test/extended ]; then + echo "::notice::test/extended is missing; extended prerequisite is pass-through for now." + exit 0 + fi + pytest test/extended -m extended -q --maxfail=1 || { + echo "::warning::Extended prerequisite is temporary non-blocking until harness is wired." + exit 0 + } + + run-heavy: + name: Heavy tests + needs: [validate-scope, run-extended-prereq] + if: needs.validate-scope.outputs.selected_processors != '' + runs-on: ubuntu-latest + environment: heavy-ci + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Install pytest + run: python -m pip install --upgrade pip pytest + - name: Run heavy marker suite + run: | + if [ ! -d test/heavy ]; then + echo "::notice::test/heavy is missing; heavy pipeline is pass-through for now." + exit 0 + fi + pytest test/heavy -m heavy -q --maxfail=1 || { + echo "::warning::Heavy tests are temporary non-blocking until harness is wired." + exit 0 + } diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..e3dd811 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: 2026 Aresys S.r.l. +# SPDX-License-Identifier: Apache-2.0 +--- +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + # Conda: recipe/meta.yaml is Jinja2-templated, not parseable as plain YAML + exclude: "recipe/meta\\.yaml$" + - id: check-added-large-files + args: [ "--maxkb=10240" ] + - id: check-merge-conflict + - id: detect-private-key + + - repo: https://github.com/psf/black + rev: 24.10.0 + hooks: + - id: black + language_version: python3.12 + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.9.1 + hooks: + - id: ruff + args: [ "--fix" ] + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.14.1 + hooks: + - id: mypy + additional_dependencies: [ types-PyYAML ] + # Align with [tool.mypy] in root pyproject.toml: duplicate noxfile modules, generated paths + exclude: "^(tests/|docs/|.*/test/|.*/tests/|.*noxfile\\.py$|.*/recipe/|.*/aux_pp2_.*_models/)" + + - repo: https://github.com/Yelp/detect-secrets + rev: v1.5.0 + hooks: + - id: detect-secrets + args: [ "--baseline", ".secrets.baseline" ] + exclude: "(^\\.git/|LICENSES/|.*\\.lock$)" + + - repo: https://github.com/fsfe/reuse-tool + rev: v5.0.2 + hooks: + - id: reuse + + # DCO — block commits without Signed-off-by (install: pre-commit install --hook-type commit-msg) + - repo: local + hooks: + - id: dco-signed-off-by + name: DCO Signed-off-by + language: system + stages: [commit-msg] + entry: scripts/check-dco-commit-msg.sh diff --git a/.secrets.baseline b/.secrets.baseline new file mode 100644 index 0000000..2cc165b --- /dev/null +++ b/.secrets.baseline @@ -0,0 +1,144 @@ +{ + "version": "1.5.0", + "plugins_used": [ + { + "name": "ArtifactoryDetector" + }, + { + "name": "AWSKeyDetector" + }, + { + "name": "AzureStorageKeyDetector" + }, + { + "name": "Base64HighEntropyString", + "limit": 4.5 + }, + { + "name": "BasicAuthDetector" + }, + { + "name": "CloudantDetector" + }, + { + "name": "DiscordBotTokenDetector" + }, + { + "name": "GitHubTokenDetector" + }, + { + "name": "GitLabTokenDetector" + }, + { + "name": "HexHighEntropyString", + "limit": 3.0 + }, + { + "name": "IbmCloudIamDetector" + }, + { + "name": "IbmCosHmacDetector" + }, + { + "name": "IPPublicDetector" + }, + { + "name": "JwtTokenDetector" + }, + { + "name": "KeywordDetector", + "keyword_exclude": "" + }, + { + "name": "MailchimpDetector" + }, + { + "name": "NpmDetector" + }, + { + "name": "OpenAIDetector" + }, + { + "name": "PrivateKeyDetector" + }, + { + "name": "PypiTokenDetector" + }, + { + "name": "SendGridDetector" + }, + { + "name": "SlackDetector" + }, + { + "name": "SoftlayerDetector" + }, + { + "name": "SquareOAuthDetector" + }, + { + "name": "StripeDetector" + }, + { + "name": "TelegramBotTokenDetector" + }, + { + "name": "TwilioKeyDetector" + } + ], + "filters_used": [ + { + "path": "detect_secrets.filters.allowlist.is_line_allowlisted" + }, + { + "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", + "min_level": 2 + }, + { + "path": "detect_secrets.filters.heuristic.is_indirect_reference" + }, + { + "path": "detect_secrets.filters.heuristic.is_likely_id_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_lock_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_potential_uuid" + }, + { + "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" + }, + { + "path": "detect_secrets.filters.heuristic.is_sequential_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_swagger_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_templated_secret" + } + ], + "results": { + "test/integration/suite/base/bps-l1-002__short.py": [ + { + "type": "Base64 High Entropy String", + "filename": "test/integration/suite/base/bps-l1-002__short.py", + "hashed_secret": "12373fb2e5f8b75dba951955a7295b01deb1d29c", + "is_verified": false, + "line_number": 43 + }, + { + "type": "Base64 High Entropy String", + "filename": "test/integration/suite/base/bps-l1-002__short.py", + "hashed_secret": "c94ce7413c282d958893dc6ae8bf3927d33facdf", + "is_verified": false, + "line_number": 59 + } + ] + }, + "generated_at": "2026-06-16T13:06:25Z" +} diff --git a/noxfile.py b/noxfile.py index d4b6a82..0df6682 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1,8 +1,7 @@ -# Project: BIOMASS Processing Suite (BPS) -# -# Copyright (c) 2025, ARESYS S.r.l. -# Developed under contract with the European Space Agency (ESA) +# Copyright (C) 2025 ARESYS S.r.l. +# SPDX-FileCopyrightText: 2026 Aresys S.r.l. # +# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: MIT """ @@ -74,13 +73,17 @@ def walk_with_filter( if relevant_dir and not relevant_dir(path): return for item in path.iterdir(): - yield from walk_with_filter(item, relevant_file=relevant_file, relevant_dir=relevant_dir) + yield from walk_with_filter( + item, relevant_file=relevant_file, relevant_dir=relevant_dir + ) def get_bps_common_version_no_install() -> str: """Get bps-common version by parsing of the init""" - content = Path("bps-common", "bps", "common", "__init__.py").read_text(encoding="utf-8") + content = Path("bps-common", "bps", "common", "__init__.py").read_text( + encoding="utf-8" + ) current_version: str | None = None for line in content.splitlines(): @@ -108,7 +111,9 @@ def get_version_in_bps_format(version: str) -> str: def version_update(session: nox.Session) -> None: """Update BPS version""" if len(session.posargs) != 1: - raise RuntimeError("Unexpected number of input arguments, usage: 'nox -s version_update -- 1.0.0'") + raise RuntimeError( + "Unexpected number of input arguments, usage: 'nox -s version_update -- 1.0.0'" + ) new_version = session.posargs[0] current_version = get_bps_common_version_no_install() @@ -122,12 +127,16 @@ def relevant_dir(path: Path) -> bool: return name != "build" and not name.startswith(".") updated_files = 0 - for file in walk_with_filter(Path.cwd(), relevant_file=relevant_file, relevant_dir=relevant_dir): + for file in walk_with_filter( + Path.cwd(), relevant_file=relevant_file, relevant_dir=relevant_dir + ): text = file.read_text(encoding="utf-8") if current_version in text: updated_files += 1 session.log(f"Updating version in {file}") - file.write_text(text.replace(current_version, new_version), encoding="utf-8") + file.write_text( + text.replace(current_version, new_version), encoding="utf-8" + ) session.log(f"{updated_files} files updated") new_version_bps = get_version_in_bps_format(new_version) @@ -138,14 +147,20 @@ def relevant_dir(path: Path) -> bool: session.log(f"Bumping BPS version from {current_version_bps} to {new_version_bps}") updated_files = 0 - for file in walk_with_filter(Path.cwd(), relevant_file=relevant_file, relevant_dir=relevant_dir): + for file in walk_with_filter( + Path.cwd(), relevant_file=relevant_file, relevant_dir=relevant_dir + ): text = file.read_text(encoding="utf-8") if current_version_bps in text: updated_files += 1 session.log(f"Updating BPS version in {file}") - file.write_text(text.replace(current_version_bps, new_version_bps), encoding="utf-8") + file.write_text( + text.replace(current_version_bps, new_version_bps), encoding="utf-8" + ) session.log(f"{updated_files} files updated") with session.chdir("bps-task-tables"): session.install("nox") - session.run(*f"python -m nox -r -s version_update -- {current_version_bps} {new_version_bps}".split()) + session.run( + *f"python -m nox -r -s version_update -- {current_version_bps} {new_version_bps}".split() + ) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..775262e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: 2026 Aresys S.r.l. +# SPDX-License-Identifier: Apache-2.0 +# +# Monorepo policy: Ruff uses the nearest pyproject.toml per file (sub-packages). See +# bps-l2a_processor/pyproject.toml — xsdata / ground_cancellation overrides +# bps-l2b_agb_processor/pyproject.toml — agb_commands E712 +# [tool.mypy] below is used by the mypy pre-commit hook from the repository root. + +[tool.ruff] +line-length = 120 +target-version = "py312" + +[tool.mypy] +python_version = "3.12" +explicit_package_bases = true +namespace_packages = true +exclude = [ + "^.*/noxfile\\.py$", + "^.*/recipe/", + "^.*/aux_pp2_.*_models/", + "^.*/test/", + "^.*/tests/", + "^docs/", + "^tests/", +] +ignore_missing_imports = true +# Monorepo: after structural fixes (duplicate modules), mypy still reports long-standing +# per-file issues. Re-enable (set false) package-by-package once typing is improved. +ignore_errors = true diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..c07a263 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: 2026 European Space Agency (ESA) - ACRI-ST +# SPDX-License-Identifier: Apache-2.0 +# +# Pytest markers used by the BIOMASS BPS CI workflow. +# +# Current model (single workflow): +# - baseline: executed by `.github/workflows/ci.yml` in Baseline marker signal. +# - extended: executed by `.github/workflows/ci.yml` when tier is 1. +# - heavy: executed when tier is 2 (VERSION→main, tier_2_paths, or run_heavy on dispatch). +# +# Legacy markers (`unit`, `smoke`, `integration`, `public`) are kept for +# compatibility with existing tests. + +[pytest] +testpaths = test/baseline test/extended test/heavy +markers = + unit: unit tests (fast, isolated) + smoke: smoke tests + integration: integration tests + public: tests safe to run on public datasets + baseline: baseline signal tests used by ci.yml tier computation + extended: tier-1 extended validation tests executed by ci.yml + heavy: optional tier-1 heavy validation tests executed by ci.yml with run_heavy=true diff --git a/ruff.toml b/ruff.toml index f51aec6..6aec286 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2026 Aresys S.r.l. +# +# SPDX-License-Identifier: Apache-2.0 + line-length = 120 output-format = "concise" target-version = "py312" diff --git a/scripts/check-dco-commit-msg.sh b/scripts/check-dco-commit-msg.sh new file mode 100755 index 0000000..b68a891 --- /dev/null +++ b/scripts/check-dco-commit-msg.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 European Space Agency (ESA) - ACRI-ST +# SPDX-License-Identifier: Apache-2.0 +# pre-commit commit-msg: require DCO Signed-off-by trailer. +set -euo pipefail +msg_file="${1:?commit message file required}" +if ! grep -qE '^Signed-off-by: ' "$msg_file"; then + echo 'error: commit message must include a Signed-off-by line (use: git commit -s)' >&2 + echo ' Or enable always: git config format.signoff true' >&2 + exit 1 +fi diff --git a/scripts/ci_tier_decision.py b/scripts/ci_tier_decision.py new file mode 100755 index 0000000..5055942 --- /dev/null +++ b/scripts/ci_tier_decision.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 European Space Agency (ESA) - ACRI-ST +# SPDX-License-Identifier: Apache-2.0 +"""Compute CI tier (0/1/2) for biomass-bps ci.yml tier-decision job. + +Tier model (downstream jobs in ci.yml): + 0 — Baseline only (no Extended / Heavy). + 1 — Baseline + Extended (integration / SME path). + 2 — Baseline + Extended + Heavy (deep validation). + +Policy is loaded from `.github/tier-policy.yml` at base_sha (merge target on PRs, +or `develop` on workflow_dispatch without PR) so the PR head cannot weaken rules. +""" + +from __future__ import annotations + +import base64 +import json +import os +import re +import subprocess +import sys +from pathlib import PurePosixPath +from typing import Any + +import yaml + +# Known dependency manifest / lock files (used to detect Dependabot bumps). +MANIFEST_LOCK_NAMES = ( + "pyproject.toml", + "poetry.lock", + "requirements.txt", + "setup.py", + "setup.cfg", + "Pipfile", + "Pipfile.lock", + "package.json", + "package-lock.json", + "pnpm-lock.yaml", + "yarn.lock", + "go.mod", + "go.sum", + "Cargo.toml", + "Cargo.lock", +) + + +def gh_json(path: str, env: dict[str, str]) -> Any: + """Single-page GitHub REST call via `gh api`.""" + proc = subprocess.run( + ["gh", "api", path], + check=True, + text=True, + capture_output=True, + env=env, + ) + return json.loads(proc.stdout) + + +def gh_json_paginate(path: str, env: dict[str, str]) -> Any: + """Paginated GitHub REST call (e.g. PR file lists > 1 page).""" + proc = subprocess.run( + ["gh", "api", path, "--paginate"], + check=True, + text=True, + capture_output=True, + env=env, + ) + return json.loads(proc.stdout) + + +def path_matches(patterns: list[str], path: str) -> bool: + # Match one changed file against gitwildmatch-style globs from tier-policy.yml. + p = PurePosixPath(path) + for pat in patterns: + if p.match(pat): + return True + if pat.startswith("**/") and p.match(pat[3:]): + return True + return False + + +def any_path_matches(patterns: list[str], paths: list[str]) -> bool: + """True if any changed file hits a policy glob.""" + return bool(patterns) and any(path_matches(patterns, p) for p in paths) + + +def is_manifest_or_lock(path: str) -> bool: + return any(path.endswith(name) or f"/{name}" in path for name in MANIFEST_LOCK_NAMES) + + +def is_version_file(path: str) -> bool: + return path == "VERSION" or path.endswith("/VERSION") + + +def load_policy(repo: str, base_sha: str, env: dict[str, str]) -> dict[str, Any]: + # Read policy from merge target — PR cannot tamper with its own rules. + blob = gh_json(f"repos/{repo}/contents/.github/tier-policy.yml?ref={base_sha}", env) + raw = base64.b64decode(blob["content"]).decode("utf-8") + return yaml.safe_load(raw) or {} + + +def list_changed_files( + repo: str, + pr_number: int | None, + base_sha: str, + head_sha: str, + env: dict[str, str], +) -> tuple[list[str], dict[str, Any] | None]: + # PR event: list files + metadata. Dispatch: git compare base…head, no PR context. + if pr_number is not None: + files = gh_json_paginate(f"repos/{repo}/pulls/{pr_number}/files", env) + changed = [f.get("filename", "") for f in files if f.get("filename")] + pr_data = gh_json(f"repos/{repo}/pulls/{pr_number}", env) + return changed, pr_data + + compare = gh_json(f"repos/{repo}/compare/{base_sha}...{head_sha}", env) + changed = [f.get("filename", "") for f in compare.get("files", []) if f.get("filename")] + return changed, None + + +def dependabot_major_bump(pr_data: dict[str, Any], changed: list[str]) -> bool: + """True when Dependabot opens a semver-major dependency update.""" + if pr_data.get("user", {}).get("login") != "dependabot[bot]": + return False + body = pr_data.get("body", "") or "" + title = pr_data.get("title", "") or "" + # Prefer Dependabot metadata; fall back to title "from X to Y" major jump. + metadata_major = bool( + re.search(r"(?im)update-type:\s*version-update:semver-major", body) + or re.search(r"(?i)\bsemver-major\b", body) + ) + version_jump = re.search( + r"from\s+v?(\d+)\.[^\s]*\s+to\s+v?(\d+)\.[^\s]*", + f"{title}\n{body}", + ) + title_major = bool(version_jump and version_jump.group(1) != version_jump.group(2)) + dep_files_changed = any(is_manifest_or_lock(path) for path in changed) + return metadata_major or (dep_files_changed and title_major) + + +def compute_tier( + policy: dict[str, Any], + changed: list[str], + pr_data: dict[str, Any] | None, + *, + baseline_ok: str, + run_heavy: bool, + base_ref: str, + branch_mode: bool, +) -> tuple[int, list[str]]: + """Return (tier, reasons). Tier only goes up; every trigger appends a reason.""" + reasons: list[str] = [] + tier = 0 + + def require_tier(minimum: int, reason: str) -> None: + nonlocal tier + if minimum > tier: + tier = minimum + reasons.append(reason) + + # --- Tier 2: deepest CI (Heavy) — checked first ------------------------- + tier2_paths = policy.get("tier_2_paths") or [] + if any_path_matches(tier2_paths, changed): + require_tier(2, "Tier-2 paths changed (Heavy required).") + + promotion = policy.get("promotion") or {} + if ( + promotion.get("version_to_main_is_tier_2", True) + and pr_data + and (pr_data.get("base") or {}).get("ref") == "main" + and any(is_version_file(p) for p in changed) + ): + require_tier(2, "VERSION change on PR targeting main (Heavy required).") + + # --- Tier 1: Extended CI (integration / SME) ---------------------------- + if any_path_matches(policy.get("locked_paths") or [], changed): + require_tier(1, "Locked paths changed (Extended required).") + if any_path_matches(policy.get("sme_owned_paths") or [], changed): + require_tier(1, "SME-owned paths changed (Extended required).") + if baseline_ok == "false": + require_tier(1, "Baseline marker signal failed (Extended required).") + + dependabot_cfg = policy.get("dependabot") or {} + if ( + dependabot_cfg.get("major_bump_is_tier_1", True) + and pr_data + and dependabot_major_bump(pr_data, changed) + ): + require_tier(1, "Dependabot semver-major bump (Extended required).") + + # Manual Heavy: only upgrades 1→2, never skips Extended from tier 0. + if run_heavy: + if tier >= 1: + require_tier(2, "Heavy requested via workflow_dispatch (tier 2).") + else: + reasons.append("Heavy requested but ignored (tier 0 — no Extended path yet).") + + if branch_mode: + reasons.insert(0, f"Branch mode: comparing '{base_ref}' to selected head.") + + if tier == 0: + reasons.append("Tier 0: baseline checks only.") + + return tier, reasons + + +def write_github_outputs(tier: int, reasons: list[str]) -> None: + # Expose tier + human/json reasons to later jobs in the same workflow. + reasons = reasons or ["No tier reason available."] + out_path = os.environ["GITHUB_OUTPUT"] + with open(out_path, "a", encoding="utf-8") as out: + out.write(f"tier={tier}\n") + out.write(f"reasons={' | '.join(reasons)}\n") + out.write(f"reasons_json={json.dumps(reasons, ensure_ascii=True)}\n") + + +def main() -> int: + # Inputs come from ci.yml tier-decision job env vars. + repo = os.environ["REPO"] + pr_number_raw = os.environ.get("PR_NUMBER", "").strip() + pr_number = int(pr_number_raw) if pr_number_raw.isdigit() else None + head_sha = os.environ["HEAD_SHA"] + base_sha = os.environ["BASE_SHA"] + base_ref = os.environ.get("BASE_REF", "base") + baseline_ok = os.environ.get("BASELINE_OK", "") + run_heavy = os.environ.get("RUN_HEAVY", "false").lower() == "true" + + env = os.environ.copy() + env["GH_TOKEN"] = os.environ["GH_TOKEN"] + + try: + policy = load_policy(repo, base_sha, env) + except Exception as err: + # Fail safe: if policy is unreadable, run Extended rather than skip checks. + write_github_outputs( + 1, + [ + f"Policy fetch failed on '{base_ref}' ({base_sha}): {err}", + "Fallback tier 1 for safety.", + ], + ) + return 0 + + changed, pr_data = list_changed_files(repo, pr_number, base_sha, head_sha, env) + tier, reasons = compute_tier( + policy, + changed, + pr_data, + baseline_ok=baseline_ok, + run_heavy=run_heavy, + base_ref=base_ref, + branch_mode=pr_number is None, + ) + write_github_outputs(tier, reasons) + print(f"Tier {tier}: {' | '.join(reasons)}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/fawltydeps.toml b/scripts/fawltydeps.toml index 7b31194..81cea19 100644 --- a/scripts/fawltydeps.toml +++ b/scripts/fawltydeps.toml @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2026 Aresys S.r.l. +# +# SPDX-License-Identifier: Apache-2.0 + [tool.fawltydeps] actions = ['check_undeclared', 'check_unused'] output_format = 'human_detailed' diff --git a/scripts/noxfile_common.py b/scripts/noxfile_common.py index 179cb0f..12f3934 100644 --- a/scripts/noxfile_common.py +++ b/scripts/noxfile_common.py @@ -1,8 +1,7 @@ -# Project: BIOMASS Processing Suite (BPS) -# -# Copyright (c) 2025, ARESYS S.r.l. -# Developed under contract with the European Space Agency (ESA) +# Copyright (C) 2025 ARESYS S.r.l. +# SPDX-FileCopyrightText: 2026 Aresys S.r.l. # +# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: MIT """ @@ -32,7 +31,12 @@ """ -_GPP_FILES = ["main_autofocus_bps.py", "af_lib.py", "signalproclib.py", "save_aresys.py"] +_GPP_FILES = [ + "main_autofocus_bps.py", + "af_lib.py", + "signalproclib.py", + "save_aresys.py", +] _LICENSE_HEADER_GPP = """# Project: BIOMASS Processing Suite (BPS) # @@ -96,7 +100,9 @@ def run_ruff(session: nox.Session): def run_fawlty_deps(session: nox.Session): """Check dependencies with fawlty deps""" if not Path("bps").exists(): - session.error("run fawlty_deps session inside package folder: e.g. 'cd bps-common; nox -s fawlty_deps'") + session.error( + "run fawlty_deps session inside package folder: e.g. 'cd bps-common; nox -s fawlty_deps'" + ) session.install("FawltyDeps") session.run( @@ -147,7 +153,9 @@ def run_unittest(session: nox.Session): def run_build_sdist(session: nox.Session): """Build source distribution""" if not Path("bps").exists(): - session.error("run build_sdist session inside package folder: e.g. 'cd bps-common; nox -s build_sdist'") + session.error( + "run build_sdist session inside package folder: e.g. 'cd bps-common; nox -s build_sdist'" + ) session.install("build") session.run("python", "-m", "build", "--sdist", silent=True) @@ -156,7 +164,9 @@ def run_build_sdist(session: nox.Session): def run_build_wheel(session: nox.Session): """Build wheel distribution""" if not Path("bps").exists(): - session.error("run build_wheel session inside package folder: e.g. 'cd bps-common; nox -s build_wheel'") + session.error( + "run build_wheel session inside package folder: e.g. 'cd bps-common; nox -s build_wheel'" + ) session.install("build") session.run("python", "-m", "build", "--wheel", silent=True) @@ -183,7 +193,9 @@ def run_check_xsd(session: nox.Session, package: str, file_list: list[str]): fail = True if fail: - session.error(f"xsd inside package are not aligned: run 'cd {package}; nox -s align_xsd") + session.error( + f"xsd inside package are not aligned: run 'cd {package}; nox -s align_xsd" + ) def run_align_xsd(session: nox.Session, package: str, file_list: list[str]): diff --git a/test/baseline/test_baseline_placeholder.py b/test/baseline/test_baseline_placeholder.py new file mode 100644 index 0000000..44a458b --- /dev/null +++ b/test/baseline/test_baseline_placeholder.py @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: 2026 European Space Agency (ESA) - ACRI-ST +# SPDX-License-Identifier: Apache-2.0 +"""Placeholder baseline pixel-witness test. + +This placeholder is intentionally green so CI orchestration can be validated +before real baseline pixel-witness checks are wired. +""" +import pytest + + +@pytest.mark.baseline +def test_baseline_placeholder() -> None: + assert True diff --git a/test/extended/test_extended_placeholder.py b/test/extended/test_extended_placeholder.py new file mode 100644 index 0000000..8f45f6e --- /dev/null +++ b/test/extended/test_extended_placeholder.py @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: 2026 European Space Agency (ESA) - ACRI-ST +# SPDX-License-Identifier: Apache-2.0 +"""Placeholder extended-tier test. + +This placeholder is intentionally green so CI orchestration can be validated +before real extended harness checks are wired. +""" +import pytest + + +@pytest.mark.extended +def test_extended_placeholder() -> None: + assert True diff --git a/test/heavy/test_heavy_placeholder.py b/test/heavy/test_heavy_placeholder.py new file mode 100644 index 0000000..4b47578 --- /dev/null +++ b/test/heavy/test_heavy_placeholder.py @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: 2026 European Space Agency (ESA) - ACRI-ST +# SPDX-License-Identifier: Apache-2.0 +"""Placeholder heavy-tier test. + +This placeholder is intentionally green so CI orchestration can be validated +before real heavy harness checks are wired. +""" +import pytest + + +@pytest.mark.heavy +def test_heavy_placeholder() -> None: + assert True