Feat: Adding test framework with benchmarks

.gitattributes

@@ -0,0 +1 @@
+*.sh text eol=lf

.github/workflows/diagnostic-regression.yml

@@ -0,0 +1,149 @@
+name: Diagnostic Regression Suite
+
+# The Diagnostic Regression Suite is the deterministic half of the kit's
+# testing. Diagnostic scripts are tools, not agents, so the same database state
+# must return the same answer every time. It is fast, free and runs on every PR.
+#
+# Cross-Model Skill Evaluations are deliberately NOT here — they cost AI
+# credits. See skill-evaluations.yml, which is dispatch/schedule only.
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+env:
+  MONGOSH_VERSION: "2.3.8"
+  MONGOSH_SHA256: "23edb768189663aaa9732a2340a25b5fc05a314940538809a7840be7f2ce221f"
+  DOCUMENTDB_LOCAL_IMAGE: "ghcr.io/microsoft/documentdb/documentdb-local@sha256:0fcf634531c1917ad0855ff9f4354aca0a5c5d8e435f08250568deb37eeb0ad5"
+
+jobs:
+  # -------------------------------------------------------------------------
+  # Everything that needs no database. Runs in ~30s with no Docker at all, so
+  # a broken eval config or a wrong cost metric is caught before the expensive
+  # job even starts.
+  # -------------------------------------------------------------------------
+  static:
+    name: Infrastructure-free contracts
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install test deps
+        run: pip install -r testing/requirements.txt
+
+      - name: Token accounting, eval configs, router
+        working-directory: testing
+        # These scenarios override the container fixture with a no-op, so they
+        # run with no Docker, no credentials and no network.
+        run: python -m pytest -ra scenarios/token-accounting scenarios/evals-config scenarios/portable-cli
+
+  windows:
+    name: Windows portable CLI contracts
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install test deps
+        run: pip install -r testing/requirements.txt
+
+      - name: Validate Python and PowerShell launchers
+        working-directory: testing
+        run: python -m pytest -ra scenarios/portable-cli
+
+      - name: Execute a PowerShell launcher
+        shell: pwsh
+        run: .\scripts\perf-advisor.ps1 --help
+
+  # -------------------------------------------------------------------------
+  # The full scenario suite against a live container.
+  # -------------------------------------------------------------------------
+  scenarios:
+    name: Diagnostic scenario contracts
+    runs-on: ubuntu-latest
+    needs: static
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Start DocumentDB Local
+        run: |
+          # Throwaway credential for this ephemeral CI container: generated per
+          # run, never hardcoded. Masked so it cannot leak into the log.
+          DOCDB_PASSWORD="Ci$(openssl rand -hex 12)"
+          echo "::add-mask::$DOCDB_PASSWORD"
+          echo "DOCDB_PASSWORD=$DOCDB_PASSWORD" >> "$GITHUB_ENV"
+          docker run -d --name documentdb-local \
+            -p 10260:10260 -p 9712:9712 \
+            -e USERNAME=docdbadmin -e PASSWORD="$DOCDB_PASSWORD" \
+            "$DOCUMENTDB_LOCAL_IMAGE"
+
+      - name: Install mongosh into the container
+        # NOT optional. The documentdb-local image ships psql but NO mongosh
+        # (verified against the pristine image), while every diagnostic script
+        # drives the database with `docker exec <container> mongosh ...`.
+        # Without this step every scenario fails with a confusing "executable
+        # file not found", which looks like a broken test rather than a missing
+        # dependency.
+        run: |
+          MV="${MONGOSH_VERSION}"
+          curl -fsSL "https://downloads.mongodb.com/compass/mongosh-${MV}-linux-x64.tgz" \
+            -o /tmp/mongosh.tgz
+          echo "${MONGOSH_SHA256}  /tmp/mongosh.tgz" | sha256sum -c -
+          tar xzf /tmp/mongosh.tgz -C /tmp
+          docker cp "/tmp/mongosh-${MV}-linux-x64/bin/mongosh" \
+            documentdb-local:/usr/local/bin/mongosh
+          docker cp "/tmp/mongosh-${MV}-linux-x64/bin/mongosh_crypt_v1.so" \
+            documentdb-local:/usr/local/lib/ || true
+          INSTALLED_VERSION="$(docker exec documentdb-local mongosh --version | tr -d '\r')"
+          if [ "$INSTALLED_VERSION" != "$MV" ]; then
+            echo "::error::Expected mongosh $MV, installed $INSTALLED_VERSION"
+            exit 1
+          fi
+          echo "Installed compatible mongosh $INSTALLED_VERSION"
+          rm -rf /tmp/mongosh.tgz "/tmp/mongosh-${MV}-linux-x64"
+
+      - name: Wait for the gateway
+        run: |
+          for i in $(seq 1 90); do
+            if docker exec documentdb-local \
+                 mongosh "localhost:10260/admin" -u docdbadmin -p "$DOCDB_PASSWORD" \
+                 --authenticationMechanism SCRAM-SHA-256 --tls \
+                 --tlsAllowInvalidCertificates --quiet \
+                 --eval "db.runCommand({ping:1}).ok" 2>/dev/null | grep -q 1; then
+              echo "Ready after ${i}s"; exit 0
+            fi
+            sleep 2
+          done
+          echo "Gateway did not become ready in time" >&2
+          docker logs documentdb-local || true
+          exit 1
+
+      - name: Install test deps
+        run: pip install -r testing/requirements.txt
+
+      - name: Run the scenario suite
+        working-directory: testing
+        env:
+          DOCDB_PASSWORD: ${{ env.DOCDB_PASSWORD }}
+        run: python -m pytest -ra
+
+      - name: Container logs on failure
+        if: failure()
+        run: docker logs documentdb-local || true

.github/workflows/skill-efficacy-benchmark.yml

@@ -0,0 +1,170 @@
+name: MSBench Skill-Efficacy Benchmark
+
+# The MSBench Skill-Efficacy Benchmark is the publication layer: a hermetic,
+# externally-citable pass@k for the DocumentDB agent kit, run on Microsoft's
+# MSBench platform.
+#
+# Split by what each job needs:
+#
+#   validate  — no Docker, no MSBench, no network. Runs on every PR.
+#   build     — Docker only. Builds the images and proves the ORACLE scores 1
+#               and an EMPTY submission scores 0. Manual/scheduled.
+#   publish   — needs internal MSBench access. Manual only.
+#
+# The paid/gated jobs never trigger on push or pull_request.
+
+on:
+  pull_request:
+    branches: [main]
+    paths:
+      - "benchmarks/**"
+      - "testing/scenarios/benchmark-*/**"
+  push:
+    branches: [main]
+    paths:
+      - "benchmarks/**"
+  workflow_dispatch:
+    inputs:
+      run_oracle:
+        description: "Build the images and run the oracle end-to-end"
+        required: false
+        default: true
+        type: boolean
+      pass_at_k:
+        description: "Attempts per instance when submitting to MSBench"
+        required: false
+        default: "5"
+        type: string
+
+permissions:
+  contents: read
+
+jobs:
+  # -------------------------------------------------------------------------
+  validate:
+    name: Benchmark structure (free)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install test deps
+        run: pip install -r testing/requirements.txt
+
+      - name: Validate registration, task layout and cost metrics
+        working-directory: testing
+        # No container, no credentials, no network — these override the
+        # container fixture with a no-op.
+        run: python -m pytest -ra scenarios/benchmark-config scenarios/benchmark-metrics
+
+      - name: Lint shell entrypoints
+        run: |
+          sudo apt-get update -qq && sudo apt-get install -y -qq shellcheck
+          # Lint EVERY shell script under benchmarks/, not a hand-maintained
+          # list. An explicit list silently stops covering new scripts, which is
+          # how a real bug (an agent prompt that was never passed, SC2034) sat
+          # unlinted in the route-efficiency harness.
+          #
+          # SC1091: sourced files live inside the container image, not the repo.
+          mapfile -t scripts < <(find benchmarks -name '*.sh' -type f | sort)
+          printf 'linting %d script(s)\n' "${#scripts[@]}"
+          printf '  %s\n' "${scripts[@]}"
+          shellcheck -e SC1091 "${scripts[@]}"
+
+  # -------------------------------------------------------------------------
+  oracle:
+    name: Build images + oracle must pass, empty must fail
+    runs-on: ubuntu-latest
+    needs: validate
+    if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_oracle }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Stage inputs and build benchmark images
+        working-directory: benchmarks/documentdb-sdk-skills
+        # build.sh stages the ignored .skills/ and .wheels/ build inputs before
+        # invoking Docker. Calling either Dockerfile directly fails in a clean
+        # checkout because those generated directories do not exist.
+        run: bash build.sh
+
+      - name: Oracle must score 1
+        # The positive control. If this fails, the grader is impossible to
+        # satisfy and every measured score is meaningless.
+        working-directory: benchmarks/documentdb-sdk-skills
+        # The image intentionally does not bake /solution or /tests. The
+        # control runner mounts them read-only and asserts the reward.
+        run: bash verify-controls.sh --only oracle
+
+      - name: Empty submission must score 0
+        # The negative control. A grader that never fails is worthless, and this
+        # is the cheapest possible proof that it can.
+        working-directory: benchmarks/documentdb-sdk-skills
+        run: bash verify-controls.sh --only empty
+
+  # -------------------------------------------------------------------------
+  publish:
+    name: Submit to MSBench (internal access required)
+    runs-on: ubuntu-latest
+    needs: oracle
+    if: ${{ github.event_name == 'workflow_dispatch' }}
+    environment: msbench
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Azure login
+        uses: azure/login@7184910d9eb2b1c5e48f7073824a90609bb9b6d6 # v2.3.1
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
+      - name: Install msbench-cli
+        env:
+          FEED: https://pkgs.dev.azure.com/devdiv/_packaging/MicrosoftSweBench/pypi/simple/
+        run: |
+          pip install keyring artifacts-keyring
+          TOKEN=$(az account get-access-token \
+            --resource 499b84ac-1321-427f-aa17-267ca6975798 \
+            --query accessToken -o tsv)
+          echo "::add-mask::$TOKEN"
+          pip install msbench-cli --index-url="https://user:${TOKEN}@${FEED#https://}"
+          msbench-cli version
+
+      - name: Run both arms
+        # BOTH arms, always. A treatment score without its control is not a
+        # result — it cannot be interpreted, let alone published.
+        run: |
+          K="${{ inputs.pass_at_k }}"
+          for arm in documentdb-sdk-skills documentdb-sdk-skills-noskills; do
+            echo "::group::$arm"
+            msbench-cli run \
+              --benchmark "$arm" \
+              --dataset "benchmarks/documentdb-sdk-skills/msbench-registration/$arm/dataset.jsonl" \
+              --pass_at_k "$K" \
+              --runner benchmarks/documentdb-sdk-skills/shared/ces/runner.sh
+            echo "::endgroup::"
+          done
+
+      - name: Summarise
+        if: always()
+        run: |
+          {
+            echo "## MSBench Skill-Efficacy Benchmark"
+            echo
+            echo "Report the **delta** between \`documentdb-sdk-skills\` (kit"
+            echo "installed, never mentioned) and \`documentdb-sdk-skills-noskills\`"
+            echo "(control). An absolute pass@k on either arm alone is not a result."
+            echo
+            echo "Per-instance token cost and observed model provenance are in"
+            echo "\`custom_metrics.json\`; export them with"
+            echo '`msbench-cli report --run_id <id> --output report.json`.'
+            echo "The report generator refuses comparisons with missing or different"
+            echo "model identifiers between treatment and control."
+          } >> "$GITHUB_STEP_SUMMARY"