diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 60725c1d..a4c01319 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -17,9 +17,9 @@ jobs: python-version: ["3.9"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Test @@ -31,7 +31,7 @@ jobs: coverage run -m unittest discover coverage json - name: Upload coverage artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: coverage_artifact path: coverage.json @@ -40,9 +40,9 @@ jobs: needs: test runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Download coverage artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: coverage_artifact - name: Check coverage @@ -54,9 +54,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up python 3.9 - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.9 - name: Run submission check diff --git a/benchmarks/matbench_v0.1_SciKG_Verify/info.json b/benchmarks/matbench_v0.1_SciKG_Verify/info.json new file mode 100644 index 00000000..c44117c3 --- /dev/null +++ b/benchmarks/matbench_v0.1_SciKG_Verify/info.json @@ -0,0 +1,15 @@ +{ + "authors": "Gaurav Sahu", + "algorithm": "SciKG Verify", + "algorithm_long": "A temporally filtered scientific-knowledge-graph verifier applied to frozen MODNet v0.1.12 MatBench dielectric predictions. The method retrieves compatible pre-cutoff external measurements and relations, applies strict structure/property/source compatibility checks, and uses nested out-of-fold selection to decide whether and how strongly to correct each prediction.", + "bibtex_refs": "", + "notes": "Evidence-augmented evaluation: the method uses external literature and database evidence available before the benchmark cutoff at inference time. Mean official-fold MAE is 0.24932956785914703. The outer fold is excluded from all selector fitting and parameter selection. This external-evidence contract should be displayed explicitly in any leaderboard entry.", + "requirements": { + "python": [ + "matbench==0.6", + "numpy", + "pandas", + "scikit-learn" + ] + } +} diff --git a/benchmarks/matbench_v0.1_SciKG_Verify/results.json.gz b/benchmarks/matbench_v0.1_SciKG_Verify/results.json.gz new file mode 100644 index 00000000..0b7ca2ce Binary files /dev/null and b/benchmarks/matbench_v0.1_SciKG_Verify/results.json.gz differ diff --git a/benchmarks/matbench_v0.1_SciKG_Verify/submission_client.py b/benchmarks/matbench_v0.1_SciKG_Verify/submission_client.py new file mode 100644 index 00000000..e650da39 --- /dev/null +++ b/benchmarks/matbench_v0.1_SciKG_Verify/submission_client.py @@ -0,0 +1,41 @@ +"""Thin public client for the private Scientia verifier service. + +The service URL and a revocable benchmark key are supplied at runtime. No +knowledge graph, extracted evidence, model weights, or verifier logic is +distributed with this client. +""" + +from __future__ import annotations + +import json +import os +from urllib.request import Request, urlopen + + +def verify_materials(endpoint: str, fold: int, candidates: list[dict]) -> list[dict]: + base_url = os.environ["SCIENTIA_VERIFIER_API_URL"].rstrip("/") + api_key = os.environ["SCIENTIA_VERIFIER_API_KEY"] + payload = json.dumps({ + "endpoint": endpoint, + "fold": fold, + "candidates": candidates, + }).encode("utf-8") + request = Request( + f"{base_url}/v1/verify/materials", + data=payload, + headers={ + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + }, + method="POST", + ) + with urlopen(request, timeout=300) as response: + body = json.load(response) + predictions = body.get("predictions") + if not isinstance(predictions, list) or len(predictions) != len(candidates): + raise RuntimeError("verifier API returned a malformed prediction batch") + expected = [str(row["id"]) for row in candidates] + received = [str(row["id"]) for row in predictions] + if received != expected: + raise RuntimeError("verifier API changed candidate order or identity") + return predictions diff --git a/requirements-dev.txt b/requirements-dev.txt index 4c2ec0e9..97a35f33 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,12 @@ pytest coverage==6.4.3 wheel==0.37.1 -monty==2022.4.26 +# The current test suite still references np.float_, removed in NumPy 2. +numpy<2 +# Matminer and pymatgen installed by matbench require a current Monty. The +# previous 2022 pin downgraded the compatible version selected by pip and made +# both the test and benchmark-submission jobs fail during import. +monty>=2024.7.29 isort==5.10.1 black==22.3.0 -flake8==4.0.1 \ No newline at end of file +flake8==4.0.1