New Benchmark: Add SciKG Verify MatBench dielectric result

.github/workflows/python-package.yml

@@ -17,9 +17,9 @@ jobs:
         python-version: ["3.9"]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Test
@@ -31,7 +31,7 @@ jobs:
           coverage run -m unittest discover
           coverage json
       - name: Upload coverage artifact
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: coverage_artifact
           path: coverage.json
@@ -40,9 +40,9 @@ jobs:
     needs: test
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Download coverage artifact
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: coverage_artifact
       - name: Check coverage
@@ -54,9 +54,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up python 3.9
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: 3.9
     - name: Run submission check

benchmarks/matbench_v0.1_SciKG_Verify/info.json

@@ -0,0 +1,15 @@
+{
+  "authors": "Gaurav Sahu",
+  "algorithm": "SciKG Verify",
+  "algorithm_long": "A temporally filtered scientific-knowledge-graph verifier applied to frozen MODNet v0.1.12 MatBench dielectric predictions. The method retrieves compatible pre-cutoff external measurements and relations, applies strict structure/property/source compatibility checks, and uses nested out-of-fold selection to decide whether and how strongly to correct each prediction.",
+  "bibtex_refs": "",
+  "notes": "Evidence-augmented evaluation: the method uses external literature and database evidence available before the benchmark cutoff at inference time. Mean official-fold MAE is 0.24932956785914703. The outer fold is excluded from all selector fitting and parameter selection. This external-evidence contract should be displayed explicitly in any leaderboard entry.",
+  "requirements": {
+    "python": [
+      "matbench==0.6",
+      "numpy",
+      "pandas",
+      "scikit-learn"
+    ]
+  }
+}

benchmarks/matbench_v0.1_SciKG_Verify/submission_client.py

@@ -0,0 +1,41 @@
+"""Thin public client for the private Scientia verifier service.
+
+The service URL and a revocable benchmark key are supplied at runtime. No
+knowledge graph, extracted evidence, model weights, or verifier logic is
+distributed with this client.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from urllib.request import Request, urlopen
+
+
+def verify_materials(endpoint: str, fold: int, candidates: list[dict]) -> list[dict]:
+    base_url = os.environ["SCIENTIA_VERIFIER_API_URL"].rstrip("/")
+    api_key = os.environ["SCIENTIA_VERIFIER_API_KEY"]
+    payload = json.dumps({
+        "endpoint": endpoint,
+        "fold": fold,
+        "candidates": candidates,
+    }).encode("utf-8")
+    request = Request(
+        f"{base_url}/v1/verify/materials",
+        data=payload,
+        headers={
+            "Authorization": f"Bearer {api_key}",
+            "Content-Type": "application/json",
+        },
+        method="POST",
+    )
+    with urlopen(request, timeout=300) as response:
+        body = json.load(response)
+    predictions = body.get("predictions")
+    if not isinstance(predictions, list) or len(predictions) != len(candidates):
+        raise RuntimeError("verifier API returned a malformed prediction batch")
+    expected = [str(row["id"]) for row in candidates]
+    received = [str(row["id"]) for row in predictions]
+    if received != expected:
+        raise RuntimeError("verifier API changed candidate order or identity")
+    return predictions

requirements-dev.txt

@@ -1,7 +1,12 @@
 pytest
 coverage==6.4.3
 wheel==0.37.1
-monty==2022.4.26
+# The current test suite still references np.float_, removed in NumPy 2.
+numpy<2
+# Matminer and pymatgen installed by matbench require a current Monty.  The
+# previous 2022 pin downgraded the compatible version selected by pip and made
+# both the test and benchmark-submission jobs fail during import.
+monty>=2024.7.29
 isort==5.10.1
 black==22.3.0
-flake8==4.0.1
+flake8==4.0.1