new_benchmark

benchmarks/matbench_v0.1_hydragnn/compile.py

@@ -0,0 +1,49 @@
+import os
+import glob
+import json
+from matbench.bench import MatbenchBenchmark
+from hydragnn_gfm_finetuning.utils.ensemble_utils import build_arg_parser
+
+def run_compile(args):
+    mb = MatbenchBenchmark(autoload=False, subset=args.task_names)
+    for task_obj in mb.tasks:
+        task_obj.load()
+
+    for task_obj in mb.tasks:
+        task_name = task_obj.dataset_name
+        first_underscore_index = task_name.find('_')
+        name = task_name[first_underscore_index:]
+        pattern = os.path.join(args.output_dir+name, f"{task_name}_fold*_predictions.json")
+        pred_files = sorted(glob.glob(pattern))
+        if not pred_files:
+            raise FileNotFoundError(
+                f"No prediction files found for {task_name} matching:\n  {pattern}\n"
+                "Run `evaluate --matbench` for each fold first."
+            )
+
+        fold_predictions = {}
+        for path in pred_files:
+            with open(path) as f:
+                data = json.load(f)
+            fold_predictions[data["fold_idx"]] = data["predictions"]
+            print(f"  [{task_name}] Loaded fold {data['fold_idx']} "
+                  f"({len(data['predictions'])} predictions): {path}")
+
+        missing = set(task_obj.folds) - set(fold_predictions.keys())
+        if missing:
+            raise ValueError(
+                f"[{task_name}] Missing predictions for folds {sorted(missing)}. "
+                "Run `evaluate --matbench` for those folds before compiling."
+            )
+
+        for fold_idx in sorted(task_obj.folds):
+            task_obj.record(fold_idx, fold_predictions[fold_idx])
+            print(f"  [{task_name}] Recorded fold {fold_idx}")
+
+    mb.add_metadata({"algorithm": "HydraGNN_GFM_FineTuning"})
+    mb.to_file(args.output_file)
+
+if __name__ == "__main__":
+    parser = build_arg_parser()
+    args = parser.parse_args()
+    run_compile(args)

benchmarks/matbench_v0.1_hydragnn/compile.sh

@@ -0,0 +1 @@
+python -u compile.py --task_names $@ --output_dir results --output_file results.json.gz

benchmarks/matbench_v0.1_hydragnn/evaluate.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+
+""" Run HydraGNN's main train/test/validate loop on the given dataset / model combination,
+    refactored so the main flow is a callable function that accepts an `args` object.
+"""
+
+from hydragnn_gfm_finetuning.utils.ensemble_utils import build_arg_parser, evaluate_finetuned_checkpoint
+
+if __name__ == "__main__":
+    parser = build_arg_parser()
+    args = parser.parse_args()
+    args.pretrained_model_ensemble_path = './pretrained_models'
+
+    last_underscore_index = args.datasetname.rfind('_')
+    task = args.datasetname[:last_underscore_index]
+    if task in ["matbench_mp_is_metal"]:
+        args.finetuning_config = './finetuning_config_bce.json'
+    elif task in ["matbench_jdft2d"]:
+        args.finetuning_config = './finetuning_config.json'
+
+    # ---- feature schema (explicit override) ----
+    graph_feature_names = ["energy"]
+    graph_feature_dims = [1]
+    node_feature_names = ["atomic_number", "cartesian_coordinates"]
+    node_feature_dims = [1, 3]
+    dictionary_variables = {}
+    dictionary_variables['graph_feature_names'] = graph_feature_names
+    dictionary_variables['graph_feature_dims'] = graph_feature_dims
+    dictionary_variables['node_feature_names'] = node_feature_names
+    dictionary_variables['node_feature_dims'] = node_feature_dims
+
+    evaluate_finetuned_checkpoint(dictionary_variables, args)

benchmarks/matbench_v0.1_hydragnn/evaluate_our_finetune.sh

@@ -0,0 +1,3 @@
+for i in {0..4}; do
+  python -u evaluate.py --datasetname matbench_"$1"_"$i" --modelname matbench_"$1"_"$i" --checkpoint_root "$PWD"/finetuned_models/matbench_"$1"_"$i" --output_dir results_"$1" --matbench
+done

benchmarks/matbench_v0.1_hydragnn/evaluate_your_finetune.sh

@@ -0,0 +1,3 @@
+for i in {0..4}; do
+  python -u evaluate.py --datasetname matbench_"$1"_"$i" --modelname matbench_"$1"_"$i" --checkpoint_root "$PWD"/logs/matbench_"$1"_"$i" --output_dir results_"$1" --matbench
+done

benchmarks/matbench_v0.1_hydragnn/finetune.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+
+""" Run HydraGNN's main train/test/validate loop on the given dataset / model combination,
+    refactored so the main flow is a callable function that accepts an `args` object.
+"""
+
+from hydragnn_gfm_finetuning.utils.ensemble_utils import build_arg_parser, run_finetune
+
+if __name__ == "__main__":
+    parser = build_arg_parser()
+    args = parser.parse_args()
+
+    # The paths below assume that you are running this script from the root directory.
+    args.pretrained_model_ensemble_path = './pretrained_models'
+    last_underscore_index = args.datasetname.rfind('_')
+    task = args.datasetname[:last_underscore_index]
+    if task in ["matbench_mp_is_metal"]:
+        args.finetuning_config = './finetuning_config_bce.json'
+    elif task in ["matbench_jdft2d"]:
+        args.finetuning_config = './finetuning_config.json'
+
+    # ---- feature schema (explicit override) ----
+    graph_feature_names = ["energy"]
+    graph_feature_dims = [1]
+    node_feature_names = ["atomic_number", "cartesian_coordinates"]
+    node_feature_dims = [1, 3]
+    dictionary_variables = {}
+    dictionary_variables['graph_feature_names'] = graph_feature_names
+    dictionary_variables['graph_feature_dims'] = graph_feature_dims
+    dictionary_variables['node_feature_names'] = node_feature_names
+    dictionary_variables['node_feature_dims'] = node_feature_dims
+
+    run_finetune(dictionary_variables, args)

benchmarks/matbench_v0.1_hydragnn/finetune.sh

@@ -0,0 +1,3 @@
+for i in {0..4}; do
+  python -u finetune.py --datasetname matbench_"$1"_"$i" --modelname matbench_"$1"_"$i" --num_epochs $2 --checkpoint_dir --matbench
+done

benchmarks/matbench_v0.1_hydragnn/finetuning_config.json

@@ -0,0 +1,63 @@
+{
+    "Verbosity": {
+        "level": 2
+    },
+    "NeuralNetwork": {
+        "Profile": {"enable": 1},
+        "Architecture": {
+            "freeze_conv_layers": false,
+            "output_heads": {
+                "graph": [
+                    {
+                        "type": "branch-0",
+                        "architecture": {
+                            "dim_pretrained": 50,
+                            "num_sharedlayers": 2,
+                            "dim_sharedlayers": 5,
+                            "num_headlayers": 2,
+                            "dim_headlayers": [
+                                50,
+                                25
+                            ]
+                        }
+                    }
+                ]
+            },
+            "task_weights": [1.0],
+            "output_dim": [
+                1
+            ],
+            "output_type": [
+                "graph"
+            ]
+        },
+        "Variables_of_interest": {
+            "input_node_features": [0, 1, 2, 3],
+            "output_names": ["pred"],
+            "output_index": [0],
+            "output_dim": [1],
+            "type": ["graph"],
+            "denormalize_output": false
+        },
+        "Training": {
+            "Checkpoint" : true,
+            "num_epoch": 10,
+            "perc_train": 0.7,
+            "loss_function_types": ["mae"],
+            "batch_size": 32,
+            "precision": "fp64",
+            "energy_target_mode": "total",
+            "continue": 1,
+            "startfrom": "existing_model",
+            "Optimizer": {
+                "type": "AdamW",
+                "learning_rate": 1e-4
+            }
+        }
+    },
+    "Visualization": {
+        "plot_init_solution": true,
+        "plot_hist_solution": false,
+        "create_plots": true
+    }
+}

benchmarks/matbench_v0.1_hydragnn/finetuning_config_bce.json

@@ -0,0 +1,63 @@
+{
+    "Verbosity": {
+        "level": 2
+    },
+    "NeuralNetwork": {
+        "Profile": {"enable": 1},
+        "Architecture": {
+            "freeze_conv_layers": false,
+            "output_heads": {
+                "graph": [
+                    {
+                        "type": "branch-0",
+                        "architecture": {
+                            "dim_pretrained": 50,
+                            "num_sharedlayers": 2,
+                            "dim_sharedlayers": 5,
+                            "num_headlayers": 2,
+                            "dim_headlayers": [
+                                50,
+                                25
+                            ]
+                        }
+                    }
+                ]
+            },
+            "task_weights": [1.0],
+            "output_dim": [
+                1
+            ],
+            "output_type": [
+                "graph"
+            ]
+        },
+        "Variables_of_interest": {
+            "input_node_features": [0, 1, 2, 3],
+            "output_names": ["pred"],
+            "output_index": [0],
+            "output_dim": [1],
+            "type": ["graph"],
+            "denormalize_output": false
+        },
+        "Training": {
+            "Checkpoint" : true,
+            "num_epoch": 10,
+            "perc_train": 0.7,
+            "loss_function_types": ["binary"],
+            "batch_size": 32,
+            "precision": "fp64",
+            "energy_target_mode": "not_energy",
+            "continue": 0,
+            "startfrom": "existing_model",
+            "Optimizer": {
+                "type": "AdamW",
+                "learning_rate": 1e-4
+            }
+        }
+    },
+    "Visualization": {
+        "plot_init_solution": true,
+        "plot_hist_solution": false,
+        "create_plots": true
+    }
+}

benchmarks/matbench_v0.1_hydragnn/info.json

@@ -0,0 +1,17 @@
+{
+  "authors": "Isaac Lyngaas and Massimiliano Lupo Pasini and Benjamin Stump and Linda Ungerboeck",
+  "algorithm": "Hydragnn",
+  "algorithm_long": "",
+  "bibtex_refs": [
+    "@article{pasini2026exascale,\n      title={Exascale Multi-Task Graph Foundation Models for Imbalanced, Multi-Fidelity Atomistic Data},\n      author={Pasini, Massimiliano Lupo and Choi, Jong Youl and Mehta, Kshitij and Messerly, Richard and Weaver, Rylie and Ungerboeck, Linda and Lyngaas, Isaac and Stump, Benajmin and Aji, Ashwin M and Schulz, Karl W and others},\n      journal={arXiv preprint arXiv:2604.15380},\n      year={2026}\n}",
+    "@techreport{lupo2026hydragnn_predictive_gfm_2026,\n      title={HydraGNN\_Predictive\_GFM\_2026-Ensemble of predictive graph foundation models for atomistic materials modeling},\n      author={Lupo Pasini, Massimiliano and Choi, Jong Youl and Mehta, Kshitij and Messerly, Richard and Weaver, Rylie and Aji, Ashwin M and Schulz, Karl W and Polo, Jorda},\n      year={2026},\n      institution={Oak Ridge National Laboratory (ORNL), Oak Ridge, TN (United States)}"
+}
+  ],
+  "notes": "The original version is not capable to train on the official matbench. The data needs to be preprocessed in order to finetune with pre-trained Hydragnn models. Finetuning is performed using code from the HydraGNN and HydraGNN_GFM_FineTuning4Materials repositories which are both pip installable and used within the local scripts preprocess_data.py, finetune.py, evaluate.py, and compile.py . We provide two bash scripts one that evaluates finetuned models that we have trained ourselves and one that goes through the entire workflow of preprocessing, finetuning, evaluating and compiling. Only two matbench tasks are finetuned matbench_jdft2d and matbench_mp_is_metal.",
+  "requirements": {
+    "python": [
+      "git+https://github.com/ORNL/HydraGNN.git",
+      "git+https://github.com/ORNL/HydraGNN_GFM_FineTuning4Materials.git"
+    ]
+  }
+}

benchmarks/matbench_v0.1_hydragnn/installation

@@ -0,0 +1,12 @@
+conda create -n matbench_hydragnn python=3.11 -y
+conda activate matbench_hydragnn
+git clone https://github.com/ORNL/HydraGNN.git
+cd HydraGNN
+git checkout 704d03afd11513b3467831feabf404c04992e4ae
+sed -i '140,142s/^/#/' hydragnn/utils/datasets/pickledataset.py
+python -m pip install .
+cd ..
+git clone https://github.com/irlyngaas/HydraGNN_GFM_FineTuning4Materials.git
+cd HydraGNN_GFM_FineTuning4Materials
+git checkout matbench
+bash install.sh