diff --git a/lambench/metrics/results/final_rankings.json b/lambench/metrics/results/final_rankings.json index 67b8ac64..594c01bb 100644 --- a/lambench/metrics/results/final_rankings.json +++ b/lambench/metrics/results/final_rankings.json @@ -2,71 +2,81 @@ { "Model": "DPA-3.2-5M", "Generalizability-FF Error \u2193": 0.145, - "Generalizability-PC Error \u2193": 0.267, + "Generalizability-PC Error \u2193": 0.265, "Applicability-Instability \u2193": 0.0, - "Applicability-Efficiency \u2191": 0.2 + "Applicability-Efficiency \u2191": 0.2, + "Applicability-Roughness \u2193": 0.221 }, { "Model": "MACE-MH-1", "Generalizability-FF Error \u2193": 0.174, - "Generalizability-PC Error \u2193": 0.268, + "Generalizability-PC Error \u2193": 0.264, "Applicability-Instability \u2193": 0.021, - "Applicability-Efficiency \u2191": 0.162 + "Applicability-Efficiency \u2191": 0.162, + "Applicability-Roughness \u2193": 0.134 }, { "Model": "DPA-3.1-3M", "Generalizability-FF Error \u2193": 0.195, - "Generalizability-PC Error \u2193": 0.296, + "Generalizability-PC Error \u2193": 0.291, "Applicability-Instability \u2193": 0.0, - "Applicability-Efficiency \u2191": 0.25 + "Applicability-Efficiency \u2191": 0.25, + "Applicability-Roughness \u2193": 0.45 }, { "Model": "Orb-v3-omat", "Generalizability-FF Error \u2193": 0.198, - "Generalizability-PC Error \u2193": 0.444, + "Generalizability-PC Error \u2193": 0.435, "Applicability-Instability \u2193": 0.0, - "Applicability-Efficiency \u2191": 0.39 + "Applicability-Efficiency \u2191": 0.39, + "Applicability-Roughness \u2193": 0.162 }, { "Model": "DPA-2.4-7M", "Generalizability-FF Error \u2193": 0.24, - "Generalizability-PC Error \u2193": 0.38, + "Generalizability-PC Error \u2193": 0.37, "Applicability-Instability \u2193": 0.039, - "Applicability-Efficiency \u2191": 0.617 + "Applicability-Efficiency \u2191": 0.617, + "Applicability-Roughness \u2193": 0.231 }, { "Model": "SevenNet-MF-ompa", "Generalizability-FF Error \u2193": 0.244, - "Generalizability-PC Error \u2193": 0.475, + "Generalizability-PC Error \u2193": 0.464, "Applicability-Instability \u2193": 0.0, - "Applicability-Efficiency \u2191": 0.085 + "Applicability-Efficiency \u2191": 0.085, + "Applicability-Roughness \u2193": 0.207 }, { "Model": "Orb-v2", "Generalizability-FF Error \u2193": 0.261, - "Generalizability-PC Error \u2193": 0.556, + "Generalizability-PC Error \u2193": 0.546, "Applicability-Instability \u2193": 2.649, - "Applicability-Efficiency \u2191": 1.341 + "Applicability-Efficiency \u2191": 1.341, + "Applicability-Roughness \u2193": 0.562 }, { "Model": "GRACE-2L-OAM", "Generalizability-FF Error \u2193": 0.263, - "Generalizability-PC Error \u2193": 0.432, + "Generalizability-PC Error \u2193": 0.427, "Applicability-Instability \u2193": 0.309, - "Applicability-Efficiency \u2191": 0.639 + "Applicability-Efficiency \u2191": 0.639, + "Applicability-Roughness \u2193": NaN }, { "Model": "MatterSim-v1-5M", "Generalizability-FF Error \u2193": 0.286, - "Generalizability-PC Error \u2193": 0.494, + "Generalizability-PC Error \u2193": 0.488, "Applicability-Instability \u2193": 0.0, - "Applicability-Efficiency \u2191": 0.393 + "Applicability-Efficiency \u2191": 0.393, + "Applicability-Roughness \u2193": 0.195 }, { "Model": "MACE-MPA-0", "Generalizability-FF Error \u2193": 0.298, - "Generalizability-PC Error \u2193": 0.471, + "Generalizability-PC Error \u2193": 0.463, "Applicability-Instability \u2193": 0.0, - "Applicability-Efficiency \u2191": 0.293 + "Applicability-Efficiency \u2191": 0.293, + "Applicability-Roughness \u2193": 0.184 } ] diff --git a/lambench/metrics/results/pc_barplot.json b/lambench/metrics/results/pc_barplot.json index f925586d..5953816d 100644 --- a/lambench/metrics/results/pc_barplot.json +++ b/lambench/metrics/results/pc_barplot.json @@ -12,16 +12,16 @@ "Orb-v2": 0.34 }, "Molecules": { - "DPA-3.1-3M": 0.29, - "Orb-v3-omat": 0.42, - "SevenNet-MF-ompa": 0.53, - "MatterSim-v1-5M": 0.49, - "GRACE-2L-OAM": 0.48, - "MACE-MH-1": 0.22, + "DPA-3.1-3M": 0.27, + "Orb-v3-omat": 0.39, + "SevenNet-MF-ompa": 0.49, + "MatterSim-v1-5M": 0.47, + "GRACE-2L-OAM": 0.46, + "MACE-MH-1": 0.21, "DPA-3.2-5M": 0.17, - "DPA-2.4-7M": 0.4, - "MACE-MPA-0": 0.57, - "Orb-v2": 0.47 + "DPA-2.4-7M": 0.37, + "MACE-MPA-0": 0.55, + "Orb-v2": 0.45 }, "Catalysis": { "DPA-3.1-3M": 0.53, diff --git a/lambench/metrics/results/results.json b/lambench/metrics/results/results.json index c86ab444..d02e170b 100644 --- a/lambench/metrics/results/results.json +++ b/lambench/metrics/results/results.json @@ -255,6 +255,10 @@ "surface": { "MAE": 10.701869293788128, "RMSE": 15.207025348080826 + }, + "rxn_path39": { + "MAE": 6.48444146353687, + "RMSE": 8.129901791481975 } }, "adaptability_results": { @@ -303,6 +307,9 @@ "average_time": 399.66949, "standard_deviation": 62.837368, "success_rate": 100.0 + }, + "homonuclear_diatomics": { + "avg_roughness": 0.4498518411605154 } }, "model": { @@ -583,6 +590,10 @@ "surface": { "MAE": 7.730285229089973, "RMSE": 10.715703262149056 + }, + "rxn_path39": { + "MAE": 8.023407534379665, + "RMSE": 10.127686384191708 } }, "applicability_results": { @@ -597,6 +608,9 @@ "momenta_diff": 3e-06, "simulation_time": 199.693032, "success_rate": 1.0 + }, + "homonuclear_diatomics": { + "avg_roughness": 0.16246333842726995 } }, "model": { @@ -877,6 +891,10 @@ "surface": { "MAE": 6.116051042875389, "RMSE": 9.060789597643824 + }, + "rxn_path39": { + "MAE": 9.998645766111366, + "RMSE": 13.99415003416049 } }, "applicability_results": { @@ -891,6 +909,9 @@ "momenta_diff": 4e-06, "simulation_time": 1191.318543, "success_rate": 1.0 + }, + "homonuclear_diatomics": { + "avg_roughness": 0.2066565410761743 } }, "model": { @@ -1171,6 +1192,10 @@ "surface": { "MAE": 17.680635636387308, "RMSE": 21.342328325509524 + }, + "rxn_path39": { + "MAE": 11.757265695870144, + "RMSE": 15.676521644551752 } }, "applicability_results": { @@ -1185,6 +1210,9 @@ "average_time": 254.233376, "standard_deviation": 47.648748, "success_rate": 100.0 + }, + "homonuclear_diatomics": { + "avg_roughness": 0.195204582137426 } }, "model": { @@ -1465,6 +1493,10 @@ "surface": { "MAE": 8.07711063774917, "RMSE": 10.890204413178536 + }, + "rxn_path39": { + "MAE": 12.359474979660057, + "RMSE": 17.681777157764113 } }, "applicability_results": { @@ -1479,6 +1511,9 @@ "average_time": 156.478982, "standard_deviation": 8.79467, "success_rate": 100.0 + }, + "homonuclear_diatomics": { + "avg_roughness": null } }, "model": { @@ -1759,6 +1794,10 @@ "surface": { "MAE": 6.590571606490859, "RMSE": 9.41158714353 + }, + "rxn_path39": { + "MAE": 4.591300602185612, + "RMSE": 8.396600759168892 } }, "applicability_results": { @@ -1773,6 +1812,9 @@ "average_time": 616.60677, "standard_deviation": 36.566305, "success_rate": 100.0 + }, + "homonuclear_diatomics": { + "avg_roughness": 0.13423846846251986 } }, "model": { @@ -2053,6 +2095,10 @@ "surface": { "MAE": 11.031323547761554, "RMSE": 15.713459882864909 + }, + "rxn_path39": { + "MAE": 4.826671006662312, + "RMSE": 11.994406239897968 } }, "applicability_results": { @@ -2067,6 +2113,9 @@ "average_time": 499.483171, "standard_deviation": 190.45185, "success_rate": 100.0 + }, + "homonuclear_diatomics": { + "avg_roughness": 0.22080299505416406 } }, "model": { @@ -2349,6 +2398,10 @@ "surface": { "MAE": 33.13795075940262, "RMSE": 48.74320150682058 + }, + "rxn_path39": { + "MAE": 6.4083312566975685, + "RMSE": 8.287706589505312 } }, "applicability_results": { @@ -2363,6 +2416,9 @@ "average_time": 162.043808, "standard_deviation": 0.54926, "success_rate": 100.0 + }, + "homonuclear_diatomics": { + "avg_roughness": 0.23079402870310545 } }, "model": { @@ -2645,6 +2701,10 @@ "surface": { "MAE": 9.635507291050429, "RMSE": 13.046462981912905 + }, + "rxn_path39": { + "MAE": 13.77581087776534, + "RMSE": 19.85227719907728 } }, "applicability_results": { @@ -2659,6 +2719,9 @@ "average_time": 341.398952, "standard_deviation": 32.878672, "success_rate": 100.0 + }, + "homonuclear_diatomics": { + "avg_roughness": 0.1841240341472523 } }, "model": { @@ -2941,6 +3004,10 @@ "surface": { "MAE": 9.389163142132626, "RMSE": 15.22927158149521 + }, + "rxn_path39": { + "MAE": 9.130139226898388, + "RMSE": 11.864551133443074 } }, "applicability_results": { @@ -2955,6 +3022,9 @@ "average_time": 74.560947, "standard_deviation": 5.027263, "success_rate": 100.0 + }, + "homonuclear_diatomics": { + "avg_roughness": 0.5622248696271649 } }, "model": {