From c4b73585c9b479e156daf8f92ddc6842f963682e Mon Sep 17 00:00:00 2001 From: HatPdotS Date: Mon, 6 Jul 2026 22:58:33 +0200 Subject: [PATCH 1/4] Fixed beta estimation instability on gpu --- docs/changelog.rst | 5 ++ pyproject.toml | 2 +- tests/integration/test_gpu_scale_stability.py | 54 ++++++++++++ .../test_estimate_beta_determinism.py | 84 +++++++++++++++++++ torchref/__init__.py | 2 +- torchref/base/targets/xray_ml_sigmaa.py | 54 +++++++++--- torchref/cli/refine.py | 2 +- torchref/model/model_ft.py | 2 +- 8 files changed, 191 insertions(+), 14 deletions(-) create mode 100644 tests/integration/test_gpu_scale_stability.py create mode 100644 tests/unit/refinement/test_estimate_beta_determinism.py diff --git a/docs/changelog.rst b/docs/changelog.rst index b604a80..78601f5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,6 +1,11 @@ Changelog ========= +Version 0.6.1 +------------- +- Fixed bug in beta estimation that caused instability in GPU refinement + + Version 0.6.0 ------------- - Switched to per-atom cutoff radii for the electron-density sampling and added a global sigma cutoff diff --git a/pyproject.toml b/pyproject.toml index 6cbeaa4..1ecb1db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "torchref" -version = "0.6.0" +version = "0.6.1" description = "Pytorch based crystallographic refinement" readme = "README.md" requires-python = ">=3.10" diff --git a/tests/integration/test_gpu_scale_stability.py b/tests/integration/test_gpu_scale_stability.py new file mode 100644 index 0000000..0cd5045 --- /dev/null +++ b/tests/integration/test_gpu_scale_stability.py @@ -0,0 +1,54 @@ +"""GPU refinement must not collapse the per-bin scale (regression). + +Guards the ``estimate_beta`` non-determinism bug: on CUDA, a non-stable +``argsort`` + atomic ``scatter_add`` + the cancellation ``wi = A*B - C**2`` +produced a different (often floored) beta per process, which froze a bad beta +into ``refine_lbfgs`` and collapsed a per-bin ``log_scale`` to ~-99 → R-free +blow-up. A short GPU refinement should now (a) stay finite / not collapse the +scale and (b) track the CPU refinement, which the deterministic + float32-stable +``estimate_beta`` restores. +""" + +import pytest +import torch + + +@pytest.mark.integration +@pytest.mark.gpu +@pytest.mark.slow +def test_gpu_5cycle_refinement_does_not_collapse_and_matches_cpu(pdb_dir, mtz_dir): + if not torch.cuda.is_available(): + pytest.skip("CUDA required") + pdb = pdb_dir / "1DAW.pdb" + mtz = mtz_dir / "1DAW.mtz" + if not pdb.exists() or not mtz.exists(): + pytest.skip("1DAW fixture not present") + + from torchref import LBFGSRefinement + + def run(device): + ref = LBFGSRefinement( + data_file=str(mtz), + pdb=str(pdb), + target_mode="ml", + verbose=0, + device=torch.device(device), + ) + ref.refine(macro_cycles=5) + rwork, rfree = ref.get_rfactor() + log_scale_min = float(ref.scaler.log_scale.detach().min()) + return float(rwork), float(rfree), log_scale_min + + rw_gpu, rf_gpu, ls_gpu = run("cuda") + rw_cpu, rf_cpu, ls_cpu = run("cpu") + + # (a) GPU refinement stays sane: R-free finite and not blown up, and no + # per-bin log_scale collapsed (the bug drove one to ~-99; healthy ~-3). + assert rf_gpu == rf_gpu, "GPU R-free is NaN" + assert rf_gpu < 0.6, f"GPU R-free blew up ({rf_gpu:.3f}) — scale likely collapsed" + assert ls_gpu > -15.0, f"GPU per-bin log_scale collapsed (min={ls_gpu:.2f})" + + # (b) GPU tracks CPU — the device-parity the estimate_beta fix restores. + assert abs(rf_gpu - rf_cpu) < 0.03, ( + f"GPU R-free {rf_gpu:.4f} diverged from CPU {rf_cpu:.4f}" + ) diff --git a/tests/unit/refinement/test_estimate_beta_determinism.py b/tests/unit/refinement/test_estimate_beta_determinism.py new file mode 100644 index 0000000..4b84bdc --- /dev/null +++ b/tests/unit/refinement/test_estimate_beta_determinism.py @@ -0,0 +1,84 @@ +"""Determinism / device-parity / float32-stability of ``estimate_beta``. + +Regression for the GPU refinement blow-up: ``estimate_beta`` used a CUDA +atomic ``scatter_add`` for its per-shell moments and a non-stable ``argsort`` +for binning, and computed ``wi = A*B - C**2`` (a catastrophic cancellation as +the shell correlation rho -> 1). In float32 that made beta non-deterministic +across GPU processes (bins snapping to the 1.0 floor at random), which froze a +bad beta into ``refine_lbfgs`` and collapsed the per-bin scale. + +The fix: stable ``argsort``, an atomic-free contiguous ``segment_reduce``, and +sum-of-squares/centered reformulations of ``wi`` and ``OMEGA``. These tests pin +that beta is (a) bit-identical across repeated GPU calls, (b) matches CPU, and +(c) matches a float64 reference (i.e. the float32 path is accurate) — using an +input engineered to be the hard case: tied ``d_star_sq`` and high-correlation +shells. +""" + +import pytest +import torch + +from torchref.base.targets.xray_ml_sigmaa import estimate_beta + + +def _hard_inputs(dtype=torch.float32): + """Free-set inputs that trigger the failure mode: many tied d_star_sq + (so binning is argsort-tie-sensitive) and near-perfectly-correlated shells + (so wi = A*B - C^2 catastrophically cancels).""" + g = torch.Generator().manual_seed(1234) + n = 2000 + # heavily tied resolutions: only 40 distinct d_star_sq values over n refl + dss_levels = torch.linspace(0.02, 0.25, 40) + idx = torch.randint(0, 40, (n,), generator=g) + d_star_sq = dss_levels[idx] + # amplitudes: larger at low resolution + base = 200.0 * torch.exp(-8.0 * d_star_sq) + 1.0 + F_obs = base * (0.5 + torch.rand(n, generator=g)) + # F_calc almost perfectly correlated with F_obs -> rho -> 1 (cancellation) + F_calc = F_obs * 0.95 + 0.01 * base * torch.randn(n, generator=g) + F_calc = F_calc.abs() + centric = torch.rand(n, generator=g) < 0.1 + epsilon = torch.ones(n) + free_mask = torch.ones(n, dtype=torch.bool) + to = lambda t: t.to(dtype) if t.is_floating_point() else t + return ( + to(F_obs), to(F_calc), centric, to(epsilon), to(d_star_sq), free_mask, + ) + + +@pytest.mark.unit +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +class TestEstimateBetaGPUDeterminism: + def test_gpu_bit_identical_across_calls(self): + """Same input -> identical beta on every GPU call (was: 15x spread).""" + args = [t.cuda() for t in _hard_inputs()] + betas = [estimate_beta(*args)[1].detach().cpu() for _ in range(8)] + for b in betas[1:]: + assert torch.equal(b, betas[0]), "estimate_beta non-deterministic on GPU" + + def test_gpu_matches_cpu(self): + """GPU beta matches CPU beta to float32 rounding (was: totally different).""" + cpu_args = _hard_inputs() + gpu_args = [t.cuda() for t in cpu_args] + b_cpu = estimate_beta(*cpu_args)[1] + b_gpu = estimate_beta(*gpu_args)[1].cpu() + torch.testing.assert_close(b_gpu, b_cpu, rtol=1e-4, atol=1e-4) + # no shell should be spuriously floored while CPU has a finite value + assert (b_gpu.min() > 1.0 + 1e-6) == (b_cpu.min() > 1.0 + 1e-6) + + +@pytest.mark.unit +class TestEstimateBetaFloat32Accuracy: + def test_float32_matches_float64(self): + """The float32 path (stable wi/OMEGA) tracks a float64 reference — i.e. + the reformulation is accurate, not just deterministic.""" + args32 = _hard_inputs(torch.float32) + args64 = _hard_inputs(torch.float64) + b32 = estimate_beta(*args32)[1].double() + b64 = estimate_beta(*args64)[1] + torch.testing.assert_close(b32, b64, rtol=2e-3, atol=1e-2) + + def test_beta_non_negative_and_finite(self): + _, bbin, _ = estimate_beta(*_hard_inputs()) + assert torch.isfinite(bbin).all() + assert (bbin > 0).all() diff --git a/torchref/__init__.py b/torchref/__init__.py index a239f47..092bfef 100644 --- a/torchref/__init__.py +++ b/torchref/__init__.py @@ -51,7 +51,7 @@ General utilities and debugging tools. """ -__version__ = "0.6.0" +__version__ = "0.6.1" import os diff --git a/torchref/base/targets/xray_ml_sigmaa.py b/torchref/base/targets/xray_ml_sigmaa.py index 41b2fb1..cb92f09 100644 --- a/torchref/base/targets/xray_ml_sigmaa.py +++ b/torchref/base/targets/xray_ml_sigmaa.py @@ -225,7 +225,10 @@ def estimate_beta( return beta, None, None # --- sort free reflections by resolution, equal-count chunks ---------- - order = torch.argsort(dss_all[free_idx]) + # stable=True so tied d_star_sq break deterministically and identically on + # CPU/GPU (a non-stable CUDA argsort shuffles ties per process, which + # reshuffles shell membership and makes beta non-deterministic). + order = torch.argsort(dss_all[free_idx], stable=True) fo = fo_all[free_idx][order] fc = fc_all[free_idx][order] cen = cen_all[free_idx][order] @@ -241,8 +244,15 @@ def estimate_beta( n_bins = max(n_by_count, min(min_bins, n_cap)) seg = (torch.arange(n_free, device=device) * n_bins) // n_free # (n_free,) + # Segments are contiguous (data is sorted by resolution and ``seg`` is a + # non-decreasing ramp), so the per-shell reductions are contiguous segment + # sums. Use ``segment_reduce`` (atomic-free, one program per segment) rather + # than ``scatter_add`` (CUDA atomicAdd, non-deterministic accumulation + # order): bit-stable run-to-run and identical CPU/GPU to float32 rounding. + seg_lengths = torch.bincount(seg, minlength=n_bins) + def segsum(x): - return torch.zeros(n_bins, device=device, dtype=dtype).scatter_add(0, seg, x) + return torch.segment_reduce(x, "sum", lengths=seg_lengths, unsafe=True) w = torch.where(cen, torch.ones_like(fo), 2.0 * torch.ones_like(fo)) SUMw = segsum(w).clamp(min=1e-30) @@ -253,19 +263,43 @@ def segsum(x): A = segsum(w * fm2e) / SUMw B = segsum(w * fo2e) / SUMw C = segsum(w * bj) / SUMw - D = segsum(w * bj * bj) / SUMw - p = segsum(w * fm2e * fm2e) / SUMw - q = segsum(w * fo2e * fo2e) / SUMw + AB = (A * B).clamp(min=1e-30) - r = (p - A * A) * (q - B * B) + # --- wi/AB = 1 - rho^2, computed float32-stably --- + # The naive ``wi = A*B - C**2`` is a catastrophic cancellation when the + # shell correlation rho -> 1 (it subtracts two nearly-equal large numbers; + # in float32 it is 6-59% wrong right where the saturated gate lives). Write + # it instead as the squared orthogonal residual of the weighted amplitude + # vectors u = sqrt(w/eps)*Fc, v = sqrt(w/eps)*Fo: with unit vectors + # u_hat, v_hat and rho = , + # 1 - rho^2 = sum_i (u_hat_i - rho * v_hat_i)^2, + # a sum of non-negative terms (no cancellation, accurate to ~1e-5 in f32). + sc_w = torch.sqrt((w / eps).clamp(min=0.0)) + u = sc_w * fc + v = sc_w * fo + nu = torch.sqrt((A * SUMw).clamp(min=1e-30)) # ||u|| per shell + nv = torch.sqrt((B * SUMw).clamp(min=1e-30)) # ||v|| per shell + rho = (C / torch.sqrt(AB)).clamp(min=-1.0, max=1.0) # per shell + resid = u / nu[seg] - rho[seg] * (v / nv[seg]) # per reflection + wiAB = segsum(resid * resid).clamp(min=0.0) # = 1 - rho^2, per shell + wi = wiAB * AB + + # --- OMEGA = Pearson correlation of X=Fc^2/eps and Y=Fo^2/eps, computed + # float32-stably via CENTERED moments (the naive + # (D - A*B)/sqrt((p - A^2)(q - B^2)) cancels the same way). Cov/Var are + # accumulated about the per-shell means A=, B=. --- + dX = fm2e - A[seg] + dY = fo2e - B[seg] + VarX = segsum(w * dX * dX) / SUMw + VarY = segsum(w * dY * dY) / SUMw + Cov = segsum(w * dX * dY) / SUMw + rr = VarX * VarY OMEGA = torch.where( - r > 0, (D - A * B) / torch.sqrt(r.clamp(min=1e-30)), torch.zeros_like(A) + rr > 0, Cov / torch.sqrt(rr.clamp(min=1e-30)), torch.zeros_like(A) ) - wi = A * B - C * C - AB = (A * B).clamp(min=1e-30) trivial = OMEGA <= 0.0 - saturated = (wi / AB) <= 3.0e-7 + saturated = wiAB <= 3.0e-7 need = (~trivial) & (~saturated) bin_centric = cen # per-free-reflection; FOM uses per-reflection centric diff --git a/torchref/cli/refine.py b/torchref/cli/refine.py index 9780136..4b89c7b 100644 --- a/torchref/cli/refine.py +++ b/torchref/cli/refine.py @@ -93,7 +93,7 @@ def main(): "--xray-mode", type=str, default="ml", - choices=["gaussian", "ls", "ls_wunit_k1", "ml", "bhattacharyya"], + choices=["gaussian", "ls", "ls_wunit_k1", "rice", "ml", "bhattacharyya"], help="X-ray target function. 'ml' (default) is the " "maximum-likelihood Read MLF target with a cross-validated Luzzati " "sigma_A term (Phenix-style alpha/beta). 'bhattacharyya' uses the " diff --git a/torchref/model/model_ft.py b/torchref/model/model_ft.py index e9b3191..214fc0c 100644 --- a/torchref/model/model_ft.py +++ b/torchref/model/model_ft.py @@ -96,7 +96,7 @@ def __init__( The real-space splat radius is no longer a per-structure scalar: each atom is truncated at its own ``N_sigma * sigma_eff`` radius, with - ``N_sigma = torchref.sigma_cutoff_ed`` (default 3.5). Set that config + ``N_sigma = torchref.sigma_cutoff_ed`` (default 3). Set that config value to trade density accuracy against cost. gridsize : tuple of int, optional Explicit grid size tuple (nx, ny, nz). If None, computed automatically. From e9b1a9e00dab2b2c19a4412c28198dc09ec2eeb7 Mon Sep 17 00:00:00 2001 From: HatPdotS Date: Tue, 7 Jul 2026 11:00:26 +0200 Subject: [PATCH 2/4] Fixed readme --- README.md | 21 +- docs/doctest_output.out | 5545 ----------------- .../exF2/collect_exF2_data.py | 4 +- paper/extended_figures/exF3/plot_exF3.py | 4 +- .../analysis/aggregate_weight_grid.py | 20 +- .../analysis/build_torchref_score_worklist.py | 2 +- .../analysis/compare_ramachandran.py | 201 - .../analysis/submit_weight_grid.py | 161 +- paper/figure2_alphafold_start/full_mvd.log | 44 - .../SF_calc_comparison/.gitignore | 8 - .../SF_calc_comparison/README.md | 107 - .../SF_calc_comparison/bench_iso_vs_aniso.py | 206 - .../SF_calc_comparison/benchmark_worker.py | 493 -- .../SF_calc_comparison/plot_results.py | 231 - .../SF_calc_comparison/run_benchmarks.py | 258 - .../SF_calc_comparison/submit_all.sh | 37 - .../submit_bench_iso_aniso_cpu.sbatch | 41 - .../submit_bench_iso_aniso_gpu.sbatch | 33 - .../SF_calc_comparison/submit_cpu.sbatch | 35 - .../SF_calc_comparison/submit_gpu.sbatch | 34 - .../fcalc_benchmark/benchmark_cpu.py | 2 +- .../benchmark_thread_scaling.py | 453 +- .../fcalc_benchmark/benchmark_worker.py | 56 +- .../fcalc_benchmark/submit_fig3a.sbatch | 27 + .../fcalc_benchmark/submit_fig3a_nodes.sh | 59 + .../benchmark_thread_scaling.py | 307 +- .../benchmark_worker.py | 33 +- .../refinement_cycle_benchmark/rerun_ml.log | 68 - .../submit_fig3b.sbatch | 30 + .../submit_fig3b_nodes.sh | 53 + .../refinement_output/out.log | 214 - .../validation/output/extrapol_FULL.log | 81 - .../validation/output/extrapol_IBL.log | 81 - .../validation/output/torchref_FULL.log | 81 - .../validation/output/torchref_IBL.log | 81 - paper/slurm-886596.out | 1 - paper/slurm-889402.out | 1 - 37 files changed, 875 insertions(+), 8238 deletions(-) delete mode 100644 docs/doctest_output.out delete mode 100644 paper/figure2_alphafold_start/analysis/compare_ramachandran.py delete mode 100644 paper/figure2_alphafold_start/full_mvd.log delete mode 100644 paper/figure3_performance/SF_calc_comparison/.gitignore delete mode 100644 paper/figure3_performance/SF_calc_comparison/README.md delete mode 100644 paper/figure3_performance/SF_calc_comparison/bench_iso_vs_aniso.py delete mode 100644 paper/figure3_performance/SF_calc_comparison/benchmark_worker.py delete mode 100644 paper/figure3_performance/SF_calc_comparison/plot_results.py delete mode 100644 paper/figure3_performance/SF_calc_comparison/run_benchmarks.py delete mode 100644 paper/figure3_performance/SF_calc_comparison/submit_all.sh delete mode 100644 paper/figure3_performance/SF_calc_comparison/submit_bench_iso_aniso_cpu.sbatch delete mode 100644 paper/figure3_performance/SF_calc_comparison/submit_bench_iso_aniso_gpu.sbatch delete mode 100644 paper/figure3_performance/SF_calc_comparison/submit_cpu.sbatch delete mode 100644 paper/figure3_performance/SF_calc_comparison/submit_gpu.sbatch create mode 100644 paper/figure3_performance/fcalc_benchmark/submit_fig3a.sbatch create mode 100644 paper/figure3_performance/fcalc_benchmark/submit_fig3a_nodes.sh delete mode 100644 paper/figure3_performance/refinement_cycle_benchmark/rerun_ml.log create mode 100644 paper/figure3_performance/refinement_cycle_benchmark/submit_fig3b.sbatch create mode 100644 paper/figure3_performance/refinement_cycle_benchmark/submit_fig3b_nodes.sh delete mode 100644 paper/figure4_difference_refinement/refinement_output/out.log delete mode 100644 paper/figure4_difference_refinement/validation/output/extrapol_FULL.log delete mode 100644 paper/figure4_difference_refinement/validation/output/extrapol_IBL.log delete mode 100644 paper/figure4_difference_refinement/validation/output/torchref_FULL.log delete mode 100644 paper/figure4_difference_refinement/validation/output/torchref_IBL.log delete mode 100644 paper/slurm-886596.out delete mode 100644 paper/slurm-889402.out diff --git a/README.md b/README.md index 9a237d6..4a60157 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,21 @@ TorchRef is a crystallographic refinement package built entirely on PyTorch. By leveraging PyTorch's automatic differentiation and GPU acceleration, TorchRef enables seamless integration with machine learning workflows and provides a flexible, extensible framework for crystallographic structure refinement. +> **Scope +TorchRef is a mainly a library/framework to build and experiment with. It is not intended to replace mainline refinement programs for standard problems. + +# Benchmark + +![TorchRef AlphaFold-start refinement benchmark](paper/figure2_alphafold_start/figures/figure_af_benchmark.png) + +*Refinement of Phaser-placed AlphaFold models against experimental data, benchmarked on a conserved set of ~720 PDB structures (1.4–3.0 Å) with all engines starting from the same placed models and scored by a single common validator.* + +- **(A) R-factors (PHENIX-validated).** Starting from the AlphaFold prediction (green), TorchRef (red) drives R-work/R-free down to essentially the same cluster as REFMAC (purple) and PHENIX (blue). Median R-free is 0.3167 (TorchRef) vs 0.3166 (PHENIX) and 0.3161 (REFMAC5) +- **(B) Geometry (RMSZ vs REFMAC restraints).** Bond, angle, chiral and main-chain B-factor RMS Z-scores. TorchRef produces valid, physically reasonable geometry; its restraints run slightly looser than PHENIX/REFMAC (bond RMSZ ≈ 1.3) +- **(C) Wall-clock runtime.** Median runtime per structure (4 CPU cores). TorchRef (1.65 min) sits between REFMAC (0.53 min) and PHENIX (4.63 min) — ~2.8× faster than PHENIX, ~3× slower than REFMAC. +- **(D) Convergence speed (normalized).** Fraction of the total R-free improvement achieved per macrocycle. The different programs show differing convergence behavior. + + # Key Features - **Native PyTorch Integration**: Built on PyTorch's `nn.Module` architecture, TorchRef integrates naturally with the PyTorch ecosystem, including machine learning models, optimizers, and GPU acceleration. @@ -55,11 +70,11 @@ pip install -e ".[dev]" ## Dependencies - Python ≥ 3.10 -- PyTorch ≥ 2.40 +- PyTorch ≥ 2.4 - NumPy ≥ 2.0 - Gemmi ≥ 0.5 -- reciprocalspaceship ≥ 0.9 -- SciPy ≥ 1.7 +- reciprocalspaceship ≥ 0.9.18 +- SciPy ≥ 1.10 ## Testing diff --git a/docs/doctest_output.out b/docs/doctest_output.out deleted file mode 100644 index 82a11f1..0000000 --- a/docs/doctest_output.out +++ /dev/null @@ -1,5545 +0,0 @@ -Running Sphinx v8.2.3 -loading translations [en]... done -loading pickled environment... done -[autosummary] generating autosummary for: api/io.rst, api/math_functions.rst, api/model.rst, api/refinement.rst, api/restraints.rst, api/scaling.rst, api/symmetrie.rst, api/utils.rst, changelog.rst, contributing.rst, index.rst, installation.rst, quickstart.rst, user_guide/refinement.rst, user_guide/restraints.rst, user_guide/scaling.rst, user_guide/targets.rst -building [mo]: targets for 0 po files that are out of date -writing output... -building [doctest]: targets for 17 source files that are out of date -updating environment: 0 added, 3 changed, 0 removed -reading sources... [ 33%] api/io -reading sources... [ 67%] api/refinement -reading sources... [100%] api/symmetrie - -looking for now-outdated files... none found -pickling environment... done -checking consistency... done -preparing documents... done -copying assets... -copying assets: done -running tests... - -Document: api/io ----------------- -********************************************************************** -File "../torchref/io/__init__.py", line ?, in default -Failed example: - data.load_mtz('structure.mtz') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - data.load_mtz('structure.mtz') - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/datasets/reflection_data.py", line 233, in load_mtz - reader = mtz.MTZReader(verbose=self.verbose).read(path) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/mtz.py", line 158, in read - self.mtz_data = rs.read_mtz(filepath) - ^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/reciprocalspaceship/io/mtz.py", line 186, in read_mtz - gemmi_mtz = gemmi.read_mtz_file(mtzfile) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - RuntimeError: Failed to open structure.mtz: No such file or directory: structure.mtz -********************************************************************** -File "../torchref/io/__init__.py", line ?, in default -Failed example: - collection.add_dataset('native', native_data) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - collection.add_dataset('native', native_data) - ^^^^^^^^^^^ - NameError: name 'native_data' is not defined -********************************************************************** -File "../torchref/io/__init__.py", line ?, in default -Failed example: - collection.add_dataset('derivative', derivative_data) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - collection.add_dataset('derivative', derivative_data) - ^^^^^^^^^^^^^^^ - NameError: name 'derivative_data' is not defined -********************************************************************** -File "../torchref/io/__init__.py", line ?, in default -Failed example: - reader = mtz.read('data.mtz') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - reader = mtz.read('data.mtz') - ^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/mtz.py", line 347, in read - return MTZReader(verbose=verbose).read(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/mtz.py", line 158, in read - self.mtz_data = rs.read_mtz(filepath) - ^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/reciprocalspaceship/io/mtz.py", line 186, in read_mtz - gemmi_mtz = gemmi.read_mtz_file(mtzfile) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - RuntimeError: Failed to open data.mtz: No such file or directory: data.mtz -********************************************************************** -File "../torchref/io/__init__.py", line ?, in default -Failed example: - data_dict, cell, spacegroup = reader() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - data_dict, cell, spacegroup = reader() - ^^^^^^ - NameError: name 'reader' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - data = CrystalDataset(device='cuda') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - data = CrystalDataset(device='cuda') - ^^^^^^^^^^^^^^ - NameError: name 'CrystalDataset' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - data.cpu() # Move all tensors to CPU -Expected nothing -Got: - ReflectionData(n=2, sg=None) -********************************************************************** -File "None", line ?, in default -Failed example: - data.save('data.pt') # Serialize to file -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - data.save('data.pt') # Serialize to file - ^^^^^^^^^ - AttributeError: 'ReflectionData' object has no attribute 'save' -********************************************************************** -File "None", line ?, in default -Failed example: - data = ReflectionData.load_state('reflection_data.pt', device='cuda') -Expected nothing -Got: - Loaded ReflectionData from reflection_data.pt -********************************************************************** -File "None", line ?, in default -Failed example: - data.save_state('reflection_data.pt') -Expected nothing -Got: - Saved ReflectionData to reflection_data.pt -********************************************************************** -File "None", line ?, in default -Failed example: - data.load_mtz('data.mtz') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - data.load_mtz('data.mtz') - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/datasets/reflection_data.py", line 233, in load_mtz - reader = mtz.MTZReader(verbose=self.verbose).read(path) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/mtz.py", line 158, in read - self.mtz_data = rs.read_mtz(filepath) - ^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/reciprocalspaceship/io/mtz.py", line 186, in read_mtz - gemmi_mtz = gemmi.read_mtz_file(mtzfile) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - RuntimeError: Failed to open data.mtz: No such file or directory: data.mtz -********************************************************************** -File "None", line ?, in default -Failed example: - print(f"Loaded {len(data.hkl)} reflections") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - print(f"Loaded {len(data.hkl)} reflections") - ^^^^^^^^^^^^^ - TypeError: object of type 'NoneType' has no len() -********************************************************************** -File "None", line ?, in default -Failed example: - print(f"Resolution range: {data.resolution.min():.2f} - {data.resolution.max():.2f} Å") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - print(f"Resolution range: {data.resolution.min():.2f} - {data.resolution.max():.2f} Å") - ^^^^^^^^^^^^^^^^^^^ - AttributeError: 'NoneType' object has no attribute 'min' -********************************************************************** -File "None", line ?, in default -Failed example: - hkl, F, sigma, rfree = data() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - hkl, F, sigma, rfree = data() - ^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/datasets/reflection_data.py", line 1318, in __call__ - F = MaskedTensor(F.detach().clone(), to_mask) - ^^^^^^^^ - AttributeError: 'NoneType' object has no attribute 'detach' -********************************************************************** -File "None", line ?, in default -Failed example: - print(F.shape) # Full shape -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - print(F.shape) # Full shape - ^ - NameError: name 'F' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - print(F.sum()) # Only sums valid (unmasked) values -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - print(F.sum()) # Only sums valid (unmasked) values - ^ - NameError: name 'F' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - valid_mask = F.get_mask() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - valid_mask = F.get_mask() - ^ - NameError: name 'F' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - F_values = F.get_data()[valid_mask] -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - F_values = F.get_data()[valid_mask] - ^ - NameError: name 'F' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - filtered = data.cut_res(highres=1.5, lowres=50.0) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - filtered = data.cut_res(highres=1.5, lowres=50.0) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/datasets/reflection_data.py", line 1071, in cut_res - return self.filter_by_resolution(d_min=highres, d_max=lowres) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/datasets/reflection_data.py", line 1014, in filter_by_resolution - self._calculate_resolution() - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/datasets/reflection_data.py", line 630, in _calculate_resolution - raise ValueError( - ValueError: Miller indices (hkl) are required to calculate resolution -********************************************************************** -File "None", line ?, in default -Failed example: - high_res = data.cut_res(highres=1.0, lowres=2.0) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - high_res = data.cut_res(highres=1.0, lowres=2.0) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/datasets/reflection_data.py", line 1071, in cut_res - return self.filter_by_resolution(d_min=highres, d_max=lowres) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/datasets/reflection_data.py", line 1014, in filter_by_resolution - self._calculate_resolution() - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/datasets/reflection_data.py", line 630, in _calculate_resolution - raise ValueError( - ValueError: Miller indices (hkl) are required to calculate resolution -********************************************************************** -File "None", line ?, in default -Failed example: - hkl, F, sigma, rfree = data.forward_indexed() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - hkl, F, sigma, rfree = data.forward_indexed() - ^^^^^^^^^^^^^^^^^^^^ - AttributeError: 'ReflectionData' object has no attribute 'forward_indexed' -********************************************************************** -File "None", line ?, in default -Failed example: - F_np = F.cpu().numpy() # Safe for writing to files -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - F_np = F.cpu().numpy() # Safe for writing to files - ^ - NameError: name 'F' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - data = ReflectionData().load_mtz('data.mtz') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - data = ReflectionData().load_mtz('data.mtz') - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/datasets/reflection_data.py", line 233, in load_mtz - reader = mtz.MTZReader(verbose=self.verbose).read(path) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/mtz.py", line 158, in read - self.mtz_data = rs.read_mtz(filepath) - ^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/reciprocalspaceship/io/mtz.py", line 186, in read_mtz - gemmi_mtz = gemmi.read_mtz_file(mtzfile) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - RuntimeError: Failed to open data.mtz: No such file or directory: data.mtz -********************************************************************** -File "None", line ?, in default -Failed example: - print(f"Original: {len(data)} reflections, {data.spacegroup}") -Expected nothing -Got: - Original: 0 reflections, None -********************************************************************** -File "None", line ?, in default -Failed example: - data_p1 = data.expand_to_p1() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - data_p1 = data.expand_to_p1() - ^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/datasets/reflection_data.py", line 2270, in expand_to_p1 - raise ValueError("ReflectionData has no Miller indices loaded") - ValueError: ReflectionData has no Miller indices loaded -********************************************************************** -File "None", line ?, in default -Failed example: - print(f"Expanded: {len(data_p1)} reflections, {data_p1.spacegroup}") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - print(f"Expanded: {len(data_p1)} reflections, {data_p1.spacegroup}") - ^^^^^^^ - NameError: name 'data_p1' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - data_p1_anom = data.expand_to_p1(include_friedel=False) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - data_p1_anom = data.expand_to_p1(include_friedel=False) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/datasets/reflection_data.py", line 2270, in expand_to_p1 - raise ValueError("ReflectionData has no Miller indices loaded") - ValueError: ReflectionData has no Miller indices loaded -********************************************************************** -File "None", line ?, in default -Failed example: - data = ReflectionData().load_mtz('data.mtz') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - data = ReflectionData().load_mtz('data.mtz') - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/datasets/reflection_data.py", line 233, in load_mtz - reader = mtz.MTZReader(verbose=self.verbose).read(path) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/mtz.py", line 158, in read - self.mtz_data = rs.read_mtz(filepath) - ^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/reciprocalspaceship/io/mtz.py", line 186, in read_mtz - gemmi_mtz = gemmi.read_mtz_file(mtzfile) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - RuntimeError: Failed to open data.mtz: No such file or directory: data.mtz -********************************************************************** -File "None", line ?, in default -Failed example: - data_filled = data.fill(d_min=2.0) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - data_filled = data.fill(d_min=2.0) - ^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/datasets/reflection_data.py", line 2191, in fill - raise ValueError("ReflectionData has no Miller indices loaded") - ValueError: ReflectionData has no Miller indices loaded -********************************************************************** -File "None", line ?, in default -Failed example: - print(f"Original: {len(data)}, Filled: {len(data_filled)}") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - print(f"Original: {len(data)}, Filled: {len(data_filled)}") - ^^^^^^^^^^^ - NameError: name 'data_filled' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - F, sigma_F = data.get_structure_factors_with_sigma() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - F, sigma_F = data.get_structure_factors_with_sigma() - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/datasets/reflection_data.py", line 971, in get_structure_factors_with_sigma - raise ValueError("No amplitude data loaded") - ValueError: No amplitude data loaded -********************************************************************** -File "None", line ?, in default -Failed example: - if sigma_F is not None: - weighted_residual = (F_obs - F_calc) / sigma_F -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - if sigma_F is not None: - ^^^^^^^ - NameError: name 'sigma_F' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - print(f"{mask.sum()} of {len(mask)} reflections are valid") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - print(f"{mask.sum()} of {len(mask)} reflections are valid") - ^^^^^^^^ - AttributeError: 'NoneType' object has no attribute 'sum' -********************************************************************** -File "None", line ?, in default -Failed example: - blocks = ReflectionData.list_cif_data_blocks('1VLM-sf.cif') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - blocks = ReflectionData.list_cif_data_blocks('1VLM-sf.cif') - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/datasets/reflection_data.py", line 283, in list_cif_data_blocks - return cif.list_data_blocks(path) - ^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/cif.py", line 163, in list_data_blocks - reader = CIFReader(filepath) - ^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/cif_readers.py", line 80, in __init__ - self.load(filepath) - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/cif_readers.py", line 92, in load - with open(filepath, "r", encoding="utf-8", errors="ignore") as f: - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - FileNotFoundError: [Errno 2] No such file or directory: '1VLM-sf.cif' -********************************************************************** -File "None", line ?, in default -Failed example: - print(blocks) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - print(blocks) - ^^^^^^ - NameError: name 'blocks' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - data = ReflectionData().load_cif('1VLM-sf.cif', data_block=blocks[1]) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - data = ReflectionData().load_cif('1VLM-sf.cif', data_block=blocks[1]) - ^^^^^^ - NameError: name 'blocks' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - data_p1 = data.expand_to_p1() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - data_p1 = data.expand_to_p1() - ^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/datasets/reflection_data.py", line 2270, in expand_to_p1 - raise ValueError("ReflectionData has no Miller indices loaded") - ValueError: ReflectionData has no Miller indices loaded -********************************************************************** -File "None", line ?, in default -Failed example: - data_merged = data_p1.reduce_to_spacegroup('P21') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - data_merged = data_p1.reduce_to_spacegroup('P21') - ^^^^^^^ - NameError: name 'data_p1' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - data_summed = data_p1.reduce_to_spacegroup('P21', aggregation='sum') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - data_summed = data_p1.reduce_to_spacegroup('P21', aggregation='sum') - ^^^^^^^ - NameError: name 'data_p1' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - data.regenerate_rfree_flags(free_fraction=0.02, n_bins=20, seed=42) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - data.regenerate_rfree_flags(free_fraction=0.02, n_bins=20, seed=42) - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/datasets/reflection_data.py", line 611, in regenerate_rfree_flags - self._generate_rfree_flags( - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/datasets/reflection_data.py", line 323, in _generate_rfree_flags - raise ValueError("Resolution information required to generate R-free flags") - ValueError: Resolution information required to generate R-free flags -********************************************************************** -File "None", line ?, in default -Failed example: - data.regenerate_rfree_flags(free_fraction=0.05, n_bins=10, force=True) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - data.regenerate_rfree_flags(free_fraction=0.05, n_bins=10, force=True) - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/datasets/reflection_data.py", line 611, in regenerate_rfree_flags - self._generate_rfree_flags( - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/datasets/reflection_data.py", line 323, in _generate_rfree_flags - raise ValueError("Resolution information required to generate R-free flags") - ValueError: Resolution information required to generate R-free flags -********************************************************************** -File "None", line ?, in default -Failed example: - reference_hkl = data1.hkl.clone() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - reference_hkl = data1.hkl.clone() - ^^^^^ - NameError: name 'data1' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - data1.validate_hkl(reference_hkl) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - data1.validate_hkl(reference_hkl) - ^^^^^ - NameError: name 'data1' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - data2.validate_hkl(reference_hkl) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - data2.validate_hkl(reference_hkl) - ^^^^^ - NameError: name 'data2' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - assert data1.hkl.shape == data2.hkl.shape -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - assert data1.hkl.shape == data2.hkl.shape - ^^^^^ - NameError: name 'data1' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - data = ReflectionData().load_mtz('observed.mtz') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - data = ReflectionData().load_mtz('observed.mtz') - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/datasets/reflection_data.py", line 233, in load_mtz - reader = mtz.MTZReader(verbose=self.verbose).read(path) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/mtz.py", line 158, in read - self.mtz_data = rs.read_mtz(filepath) - ^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/reciprocalspaceship/io/mtz.py", line 186, in read_mtz - gemmi_mtz = gemmi.read_mtz_file(mtzfile) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - RuntimeError: Failed to open observed.mtz: No such file or directory: observed.mtz -********************************************************************** -File "None", line ?, in default -Failed example: - model = Model().load_pdb('model.pdb') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model = Model().load_pdb('model.pdb') - ^^^^^ - NameError: name 'Model' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - model_ft = ModelFT(model, data.cell, data.spacegroup) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model_ft = ModelFT(model, data.cell, data.spacegroup) - ^^^^^^^ - NameError: name 'ModelFT' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - fcalc = model_ft.forward(data.hkl) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - fcalc = model_ft.forward(data.hkl) - ^^^^^^^^ - NameError: name 'model_ft' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - data.write_mtz('output.mtz', fcalc=fcalc) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - data.write_mtz('output.mtz', fcalc=fcalc) - ^^^^^ - NameError: name 'fcalc' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - native = ReflectionData().load_mtz('native.mtz') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - native = ReflectionData().load_mtz('native.mtz') - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/datasets/reflection_data.py", line 233, in load_mtz - reader = mtz.MTZReader(verbose=self.verbose).read(path) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/mtz.py", line 158, in read - self.mtz_data = rs.read_mtz(filepath) - ^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/reciprocalspaceship/io/mtz.py", line 186, in read_mtz - gemmi_mtz = gemmi.read_mtz_file(mtzfile) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - RuntimeError: Failed to open native.mtz: No such file or directory: native.mtz -********************************************************************** -File "None", line ?, in default -Failed example: - derivative = ReflectionData().load_mtz('derivative.mtz') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - derivative = ReflectionData().load_mtz('derivative.mtz') - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/datasets/reflection_data.py", line 233, in load_mtz - reader = mtz.MTZReader(verbose=self.verbose).read(path) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/mtz.py", line 158, in read - self.mtz_data = rs.read_mtz(filepath) - ^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/reciprocalspaceship/io/mtz.py", line 186, in read_mtz - gemmi_mtz = gemmi.read_mtz_file(mtzfile) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - RuntimeError: Failed to open derivative.mtz: No such file or directory: derivative.mtz -********************************************************************** -File "None", line ?, in default -Failed example: - collection.add_dataset('native', native, set_as_reference=True) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - collection.add_dataset('native', native, set_as_reference=True) - ^^^^^^ - NameError: name 'native' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - collection.add_dataset('derivative', derivative) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - collection.add_dataset('derivative', derivative) - ^^^^^^^^^^ - NameError: name 'derivative' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - native_F = collection['native'].F -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - native_F = collection['native'].F - ~~~~~~~~~~^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/datasets/collection.py", line 186, in __getitem__ - return self._datasets[name] - ~~~~~~~~~~~~~~^^^^^^ - KeyError: 'native' -********************************************************************** -File "None", line ?, in default -Failed example: - collection.add_dataset('native', native_data, set_as_reference=True) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - collection.add_dataset('native', native_data, set_as_reference=True) - ^^^^^^^^^^^ - NameError: name 'native_data' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - collection.add_dataset('derivative', derivative_data) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - collection.add_dataset('derivative', derivative_data) - ^^^^^^^^^^^^^^^ - NameError: name 'derivative_data' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - reader = mtz.read('data.mtz', verbose=1) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - reader = mtz.read('data.mtz', verbose=1) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/mtz.py", line 347, in read - return MTZReader(verbose=verbose).read(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/mtz.py", line 158, in read - self.mtz_data = rs.read_mtz(filepath) - ^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/reciprocalspaceship/io/mtz.py", line 186, in read_mtz - gemmi_mtz = gemmi.read_mtz_file(mtzfile) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - RuntimeError: Failed to open data.mtz: No such file or directory: data.mtz -********************************************************************** -File "None", line ?, in default -Failed example: - data_dict, cell, spacegroup = reader() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - data_dict, cell, spacegroup = reader() - ^^^^^^ - NameError: name 'reader' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - print(f"Found {len(data_dict['HKL'])} reflections in {spacegroup.short_name()}") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - print(f"Found {len(data_dict['HKL'])} reflections in {spacegroup.short_name()}") - ^^^^^^^^^ - NameError: name 'data_dict' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - reader = pdb.read('structure.pdb', verbose=1) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - reader = pdb.read('structure.pdb', verbose=1) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 381, in read - return PDBReader(verbose=verbose).read(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 339, in read - self.dataframe = load_as_dataframe(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 149, in load_as_dataframe - skipheader = find_header_length(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 67, in find_header_length - with open(filepath, "r") as f: - ^^^^^^^^^^^^^^^^^^^ - FileNotFoundError: [Errno 2] No such file or directory: 'structure.pdb' -********************************************************************** -File "None", line ?, in default -Failed example: - df, cell, spacegroup = reader() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - df, cell, spacegroup = reader() - ^^^^^^ - NameError: name 'reader' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - print(f"Loaded {len(df)} atoms") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - print(f"Loaded {len(df)} atoms") - ^^ - NameError: name 'df' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - router = DataRouter("structure.cif") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - router = DataRouter("structure.cif") - ^^^^^^^^^^ - NameError: name 'DataRouter' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - reader = router.get_reader() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - reader = router.get_reader() - ^^^^^^ - NameError: name 'router' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - print(router.data_type) # 'structure' -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - print(router.data_type) # 'structure' - ^^^^^^ - NameError: name 'router' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - reader, data_type = DataRouter.route("7JI4-sf.cif") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - reader, data_type = DataRouter.route("7JI4-sf.cif") - ^^^^^^^^^^ - NameError: name 'DataRouter' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - if data_type == 'reflections': - data_dict, cell, spacegroup = reader() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - if data_type == 'reflections': - ^^^^^^^^^ - NameError: name 'data_type' is not defined -********************************************************************** -1 items had failures: - 68 of 81 in default -81 tests in 1 items. -13 passed and 68 failed. -***Test Failed*** 68 failures. - -Document: api/math_functions ----------------------------- -********************************************************************** -File "None", line ?, in default -Failed example: - fw = FrenchWilson(spacegroup='P21', cell=cell) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - fw = FrenchWilson(spacegroup='P21', cell=cell) - ^^^^ - NameError: name 'cell' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - F_french_wilson = fw(I_obs, sigma_I) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - F_french_wilson = fw(I_obs, sigma_I) - ^^ - NameError: name 'fw' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - frac_coords = math_torch.cartesian_to_fractional_torch(cart_coords, cell) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - frac_coords = math_torch.cartesian_to_fractional_torch(cart_coords, cell) - ^^^^^^^^^^^ - NameError: name 'cart_coords' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - fw_module = FrenchWilson(hkl, unit_cell, 'P212121') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - fw_module = FrenchWilson(hkl, unit_cell, 'P212121') - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/math_functions/french_wilson.py", line 1347, in __init__ - d_spacings = math_torch.get_d_spacing(hkl, unit_cell) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/math_functions/math_torch.py", line 965, in get_d_spacing - s = get_scattering_vectors(hkl, unit_cell, recB) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/math_functions/math_torch.py", line 942, in get_scattering_vectors - recB = reciprocal_basis_matrix(unit_cell) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/math_functions/math_torch.py", line 885, in reciprocal_basis_matrix - angles_rad = torch.deg2rad(unit_cell[3:]) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - TypeError: deg2rad(): argument 'input' (position 1) must be Tensor, not list -********************************************************************** -File "None", line ?, in default -Failed example: - F, sigma_F = fw_module(I, sigma_I) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - F, sigma_F = fw_module(I, sigma_I) - ^^^^^^^^^ - NameError: name 'fw_module' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - hkl = generate_possible_hkl(cell, d_min=2.0) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - hkl = generate_possible_hkl(cell, d_min=2.0) - ^^^^^^^^^^^^^^^^^^^^^ - NameError: name 'generate_possible_hkl' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - print(f"Generated {len(hkl)} reflections") -Expected nothing -Got: - Generated 4 reflections -********************************************************************** -File "None", line ?, in default -Failed example: - voids = find_solvent_voids(mask) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - voids = find_solvent_voids(mask) - ^^^^^^^^^^^^^^^^^^ - NameError: name 'find_solvent_voids' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - print(voids) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - print(voids) - ^^^^^ - NameError: name 'voids' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - from french_wilson_pytorch import FrenchWilsonModule -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - from french_wilson_pytorch import FrenchWilsonModule - ModuleNotFoundError: No module named 'french_wilson_pytorch' -********************************************************************** -File "None", line ?, in default -Failed example: - fw_module = FrenchWilsonModule(hkl, unit_cell, space_group='P212121') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - fw_module = FrenchWilsonModule(hkl, unit_cell, space_group='P212121') - ^^^^^^^^^^^^^^^^^^ - NameError: name 'FrenchWilsonModule' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - F, sigma_F = fw_module(I, sigma_I) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - F, sigma_F = fw_module(I, sigma_I) - ^^^^^^^^^ - NameError: name 'fw_module' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - print(f"F = {F}") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - print(f"F = {F}") - ^ - NameError: name 'F' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - from french_wilson_pytorch import french_wilson_auto -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - from french_wilson_pytorch import french_wilson_auto - ModuleNotFoundError: No module named 'french_wilson_pytorch' -********************************************************************** -File "None", line ?, in default -Failed example: - F, sigma_F, valid = french_wilson_auto( - I, sigma_I, hkl, d_spacings, space_group='P212121' - ) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - F, sigma_F, valid = french_wilson_auto( - ^^^^^^^^^^^^^^^^^^ - NameError: name 'french_wilson_auto' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - F, sigma_F, valid = french_wilson(I, sigma_I, mean_I) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - F, sigma_F, valid = french_wilson(I, sigma_I, mean_I) - ^^^^^^^^^^^^^ - NameError: name 'french_wilson' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - print(f"F = {F}") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - print(f"F = {F}") - ^ - NameError: name 'F' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - F, sigma_F, valid = french_wilson_auto(I, sigma_I, hkl, d_spacings, "P212121") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - F, sigma_F, valid = french_wilson_auto(I, sigma_I, hkl, d_spacings, "P212121") - ^^^^^^^^^^^^^^^^^^ - NameError: name 'french_wilson_auto' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - fw_module = FrenchWilson(hkl, unit_cell, 'P212121') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - fw_module = FrenchWilson(hkl, unit_cell, 'P212121') - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/math_functions/french_wilson.py", line 1347, in __init__ - d_spacings = math_torch.get_d_spacing(hkl, unit_cell) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/math_functions/math_torch.py", line 965, in get_d_spacing - s = get_scattering_vectors(hkl, unit_cell, recB) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/math_functions/math_torch.py", line 942, in get_scattering_vectors - recB = reciprocal_basis_matrix(unit_cell) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/math_functions/math_torch.py", line 885, in reciprocal_basis_matrix - angles_rad = torch.deg2rad(unit_cell[3:]) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - TypeError: deg2rad(): argument 'input' (position 1) must be Tensor, not list -********************************************************************** -File "None", line ?, in default -Failed example: - F, sigma_F = fw_module(I, sigma_I) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - F, sigma_F = fw_module(I, sigma_I) - ^^^^^^^^^ - NameError: name 'fw_module' is not defined -********************************************************************** -1 items had failures: - 20 of 47 in default -47 tests in 1 items. -27 passed and 20 failed. -***Test Failed*** 20 failures. - -Document: api/model -------------------- -********************************************************************** -File "None", line ?, in default -Failed example: - model.load_pdb('structure.pdb') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.load_pdb('structure.pdb') - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 231, in load_pdb - reader = pdb.PDBReader(verbose=self.verbose).read(file) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 339, in read - self.dataframe = load_as_dataframe(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 149, in load_as_dataframe - skipheader = find_header_length(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 67, in find_header_length - with open(filepath, "r") as f: - ^^^^^^^^^^^^^^^^^^^ - FileNotFoundError: [Errno 2] No such file or directory: 'structure.pdb' -********************************************************************** -File "None", line ?, in default -Failed example: - model_ft = ModelFT(data, device='cuda') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model_ft = ModelFT(data, device='cuda') - ^^^^ - NameError: name 'data' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - F_calc = model_ft.get_F_calc() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - F_calc = model_ft.get_F_calc() - ^^^^^^^^ - NameError: name 'model_ft' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - model.load_state_dict(torch.load('model.pt')) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.load_state_dict(torch.load('model.pt')) - ^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 1484, in load - with _open_file_like(f, "rb") as opened_file: - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 759, in _open_file_like - return _open_file(name_or_buffer, mode) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 740, in __init__ - super().__init__(open(name, mode)) - ^^^^^^^^^^^^^^^^ - FileNotFoundError: [Errno 2] No such file or directory: 'model.pt' -********************************************************************** -File "None", line ?, in default -Failed example: - model.load_pdb('structure.pdb') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.load_pdb('structure.pdb') - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model_ft.py", line 140, in load_pdb - super().load_pdb(filename) - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 231, in load_pdb - reader = pdb.PDBReader(verbose=self.verbose).read(file) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 339, in read - self.dataframe = load_as_dataframe(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 149, in load_as_dataframe - skipheader = find_header_length(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 67, in find_header_length - with open(filepath, "r") as f: - ^^^^^^^^^^^^^^^^^^^ - FileNotFoundError: [Errno 2] No such file or directory: 'structure.pdb' -********************************************************************** -File "None", line ?, in default -Failed example: - model = ModelFT().load_pdb('structure.pdb') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model = ModelFT().load_pdb('structure.pdb') - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model_ft.py", line 140, in load_pdb - super().load_pdb(filename) - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 231, in load_pdb - reader = pdb.PDBReader(verbose=self.verbose).read(file) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 339, in read - self.dataframe = load_as_dataframe(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 149, in load_as_dataframe - skipheader = find_header_length(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 67, in find_header_length - with open(filepath, "r") as f: - ^^^^^^^^^^^^^^^^^^^ - FileNotFoundError: [Errno 2] No such file or directory: 'structure.pdb' -********************************************************************** -File "None", line ?, in default -Failed example: - model_copy = model.copy() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model_copy = model.copy() - ^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model_ft.py", line 670, in copy - raise RuntimeError("Cannot copy an uninitialized ModelFT. Load data first.") - RuntimeError: Cannot copy an uninitialized ModelFT. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - model = Model().load_pdb('structure.pdb') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model = Model().load_pdb('structure.pdb') - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 231, in load_pdb - reader = pdb.PDBReader(verbose=self.verbose).read(file) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 339, in read - self.dataframe = load_as_dataframe(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 149, in load_as_dataframe - skipheader = find_header_length(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 67, in find_header_length - with open(filepath, "r") as f: - ^^^^^^^^^^^^^^^^^^^ - FileNotFoundError: [Errno 2] No such file or directory: 'structure.pdb' -********************************************************************** -File "None", line ?, in default -Failed example: - chain_a = model.select("chain A") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - chain_a = model.select("chain A") - ^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model_ft.py", line 146, in select - selection = super().select(selection) - ^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1473, in select - raise RuntimeError( - RuntimeError: Cannot select from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - backbone = model.select("name CA or name C or name N or name O") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - backbone = model.select("name CA or name C or name N or name O") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model_ft.py", line 146, in select - selection = super().select(selection) - ^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1473, in select - raise RuntimeError( - RuntimeError: Cannot select from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - region = model.select("chain B and resseq 10:50") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - region = model.select("chain B and resseq 10:50") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model_ft.py", line 146, in select - selection = super().select(selection) - ^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1473, in select - raise RuntimeError( - RuntimeError: Cannot select from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - no_water = model.select("not resname HOH") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - no_water = model.select("not resname HOH") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model_ft.py", line 146, in select - selection = super().select(selection) - ^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1473, in select - raise RuntimeError( - RuntimeError: Cannot select from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - complex_sel = model.select("chain A and (resname ALA or resname GLY)") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - complex_sel = model.select("chain A and (resname ALA or resname GLY)") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model_ft.py", line 146, in select - selection = super().select(selection) - ^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1473, in select - raise RuntimeError( - RuntimeError: Cannot select from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - model.load_state_dict(torch.load('model.pt')) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.load_state_dict(torch.load('model.pt')) - ^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 1484, in load - with _open_file_like(f, "rb") as opened_file: - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 759, in _open_file_like - return _open_file(name_or_buffer, mode) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 740, in __init__ - super().__init__(open(name, mode)) - ^^^^^^^^^^^^^^^^ - FileNotFoundError: [Errno 2] No such file or directory: 'model.pt' -********************************************************************** -File "None", line ?, in default -Failed example: - model.load_pdb('structure.pdb') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.load_pdb('structure.pdb') - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 231, in load_pdb - reader = pdb.PDBReader(verbose=self.verbose).read(file) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 339, in read - self.dataframe = load_as_dataframe(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 149, in load_as_dataframe - skipheader = find_header_length(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 67, in find_header_length - with open(filepath, "r") as f: - ^^^^^^^^^^^^^^^^^^^ - FileNotFoundError: [Errno 2] No such file or directory: 'structure.pdb' -********************************************************************** -File "None", line ?, in default -Failed example: - loss = xray_loss + w_adp * model.adp_kl_divergence_loss(0.2) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - loss = xray_loss + w_adp * model.adp_kl_divergence_loss(0.2) - ^^^^^^^^^ - NameError: name 'xray_loss' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - structure_factor_loss = compute_structure_factor_loss() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - structure_factor_loss = compute_structure_factor_loss() - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - NameError: name 'compute_structure_factor_loss' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - nll_reg = model.adp_nll_loss(target_log_std=0.2) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - nll_reg = model.adp_nll_loss(target_log_std=0.2) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 986, in adp_nll_loss - log_b = super(PositiveMixedTensor, self.b).forward() - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - AttributeError: 'super' object has no attribute 'forward' -********************************************************************** -File "None", line ?, in default -Failed example: - total_loss = structure_factor_loss + 0.01 * nll_reg -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - total_loss = structure_factor_loss + 0.01 * nll_reg - ^^^^^^^^^^^^^^^^^^^^^ - NameError: name 'structure_factor_loss' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - total_loss.backward() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - total_loss.backward() - ^^^^^^^^^^ - NameError: name 'total_loss' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - atom_nll = model.adp_nll_loss_per_atom(target_log_std=0.2) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - atom_nll = model.adp_nll_loss_per_atom(target_log_std=0.2) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1041, in adp_nll_loss_per_atom - log_b = super(PositiveMixedTensor, self.b).forward() - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - AttributeError: 'super' object has no attribute 'forward' -********************************************************************** -File "None", line ?, in default -Failed example: - threshold = atom_nll.mean() + 2 * atom_nll.std() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - threshold = atom_nll.mean() + 2 * atom_nll.std() - ^^^^^^^^ - NameError: name 'atom_nll' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - outliers = atom_nll > threshold -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - outliers = atom_nll > threshold - ^^^^^^^^ - NameError: name 'atom_nll' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - model.update_mask_from_selection("chain A", "xyz", freeze=True) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.update_mask_from_selection("chain A", "xyz", freeze=True) - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 646, in update_mask_from_selection - current_mask = getattr(self, mask_name) - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1964, in __getattr__ - raise AttributeError( - AttributeError: 'Model' object has no attribute 'xyz_mask' -********************************************************************** -File "None", line ?, in default -Failed example: - model.apply_mask_to_parameter("xyz") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.apply_mask_to_parameter("xyz") - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 699, in apply_mask_to_parameter - self.xyz.update_refinable_mask(self.xyz_mask) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - AttributeError: 'NoneType' object has no attribute 'update_refinable_mask' -********************************************************************** -File "None", line ?, in default -Failed example: - model = Model().load_pdb('structure.pdb') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model = Model().load_pdb('structure.pdb') - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 231, in load_pdb - reader = pdb.PDBReader(verbose=self.verbose).read(file) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 339, in read - self.dataframe = load_as_dataframe(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 149, in load_as_dataframe - skipheader = find_header_length(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 67, in find_header_length - with open(filepath, "r") as f: - ^^^^^^^^^^^^^^^^^^^ - FileNotFoundError: [Errno 2] No such file or directory: 'structure.pdb' -********************************************************************** -File "None", line ?, in default -Failed example: - model_copy = model.copy() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model_copy = model.copy() - ^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 490, in copy - raise RuntimeError("Cannot copy an uninitialized Model. Load data first.") - RuntimeError: Cannot copy an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - model.freeze_selection("chain A", targets='all') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.freeze_selection("chain A", targets='all') - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 752, in freeze_selection - self.update_mask_from_selection( - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 646, in update_mask_from_selection - current_mask = getattr(self, mask_name) - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1964, in __getattr__ - raise AttributeError( - AttributeError: 'Model' object has no attribute 'xyz_mask' -********************************************************************** -File "None", line ?, in default -Failed example: - model.freeze_selection("resseq 10:20", targets='xyz') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.freeze_selection("resseq 10:20", targets='xyz') - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 752, in freeze_selection - self.update_mask_from_selection( - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 646, in update_mask_from_selection - current_mask = getattr(self, mask_name) - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1964, in __getattr__ - raise AttributeError( - AttributeError: 'Model' object has no attribute 'xyz_mask' -********************************************************************** -File "None", line ?, in default -Failed example: - model = Model().load_pdb('structure.pdb') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model = Model().load_pdb('structure.pdb') - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 231, in load_pdb - reader = pdb.PDBReader(verbose=self.verbose).read(file) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 339, in read - self.dataframe = load_as_dataframe(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 149, in load_as_dataframe - skipheader = find_header_length(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 67, in find_header_length - with open(filepath, "r") as f: - ^^^^^^^^^^^^^^^^^^^ - FileNotFoundError: [Errno 2] No such file or directory: 'structure.pdb' -********************************************************************** -File "None", line ?, in default -Failed example: - mask = model.get_selection_mask("chain A") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - mask = model.get_selection_mask("chain A") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1407, in get_selection_mask - raise RuntimeError( - RuntimeError: Cannot get selection mask from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - new_coords = model.xyz()[mask] + translation -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - new_coords = model.xyz()[mask] + translation - ^^^^^^^^^^^ - TypeError: 'NoneType' object is not callable -********************************************************************** -File "None", line ?, in default -Failed example: - model.xyz.set(new_coords, mask) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.xyz.set(new_coords, mask) - ^^^^^^^^^^^^^ - AttributeError: 'NoneType' object has no attribute 'set' -********************************************************************** -File "None", line ?, in default -Failed example: - backbone_mask = model.get_selection_mask("name CA or name C or name N or name O") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - backbone_mask = model.get_selection_mask("name CA or name C or name N or name O") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1407, in get_selection_mask - raise RuntimeError( - RuntimeError: Cannot get selection mask from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - mask = model.get_selection_mask("chain A and (resname ALA or resname GLY)") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - mask = model.get_selection_mask("chain A and (resname ALA or resname GLY)") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1407, in get_selection_mask - raise RuntimeError( - RuntimeError: Cannot get selection mask from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - model = Model().load_pdb('structure.pdb') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model = Model().load_pdb('structure.pdb') - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 231, in load_pdb - reader = pdb.PDBReader(verbose=self.verbose).read(file) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 339, in read - self.dataframe = load_as_dataframe(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 149, in load_as_dataframe - skipheader = find_header_length(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 67, in find_header_length - with open(filepath, "r") as f: - ^^^^^^^^^^^^^^^^^^^ - FileNotFoundError: [Errno 2] No such file or directory: 'structure.pdb' -********************************************************************** -File "None", line ?, in default -Failed example: - chain_a = model.select("chain A") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - chain_a = model.select("chain A") - ^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1473, in select - raise RuntimeError( - RuntimeError: Cannot select from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - backbone = model.select("name CA or name C or name N or name O") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - backbone = model.select("name CA or name C or name N or name O") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1473, in select - raise RuntimeError( - RuntimeError: Cannot select from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - region = model.select("chain B and resseq 10:50") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - region = model.select("chain B and resseq 10:50") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1473, in select - raise RuntimeError( - RuntimeError: Cannot select from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - no_water = model.select("not resname HOH") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - no_water = model.select("not resname HOH") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1473, in select - raise RuntimeError( - RuntimeError: Cannot select from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - complex_sel = model.select("chain A and (resname ALA or resname GLY)") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - complex_sel = model.select("chain A and (resname ALA or resname GLY)") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1473, in select - raise RuntimeError( - RuntimeError: Cannot select from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - model.unfreeze_selection("chain A", targets='all') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.unfreeze_selection("chain A", targets='all') - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 792, in unfreeze_selection - self.update_mask_from_selection( - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 646, in update_mask_from_selection - current_mask = getattr(self, mask_name) - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1964, in __getattr__ - raise AttributeError( - AttributeError: 'Model' object has no attribute 'xyz_mask' -********************************************************************** -File "None", line ?, in default -Failed example: - model.unfreeze_selection("name CA or name C or name N", targets='xyz') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.unfreeze_selection("name CA or name C or name N", targets='xyz') - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 792, in unfreeze_selection - self.update_mask_from_selection( - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 646, in update_mask_from_selection - current_mask = getattr(self, mask_name) - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1964, in __getattr__ - raise AttributeError( - AttributeError: 'Model' object has no attribute 'xyz_mask' -********************************************************************** -File "None", line ?, in default -Failed example: - model.update_mask_from_selection("chain A", "xyz", mode='set', freeze=True) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.update_mask_from_selection("chain A", "xyz", mode='set', freeze=True) - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 646, in update_mask_from_selection - current_mask = getattr(self, mask_name) - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1964, in __getattr__ - raise AttributeError( - AttributeError: 'Model' object has no attribute 'xyz_mask' -********************************************************************** -File "None", line ?, in default -Failed example: - model.apply_mask_to_parameter("xyz") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.apply_mask_to_parameter("xyz") - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 699, in apply_mask_to_parameter - self.xyz.update_refinable_mask(self.xyz_mask) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - AttributeError: 'NoneType' object has no attribute 'update_refinable_mask' -********************************************************************** -File "None", line ?, in default -Failed example: - model.update_mask_from_selection("name CA or name C or name N", "xyz", freeze=False) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.update_mask_from_selection("name CA or name C or name N", "xyz", freeze=False) - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 646, in update_mask_from_selection - current_mask = getattr(self, mask_name) - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1964, in __getattr__ - raise AttributeError( - AttributeError: 'Model' object has no attribute 'xyz_mask' -********************************************************************** -File "None", line ?, in default -Failed example: - model.apply_mask_to_parameter("xyz") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.apply_mask_to_parameter("xyz") - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 699, in apply_mask_to_parameter - self.xyz.update_refinable_mask(self.xyz_mask) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - AttributeError: 'NoneType' object has no attribute 'update_refinable_mask' -********************************************************************** -File "None", line ?, in default -Failed example: - mixed.load_state_dict(torch.load('mixed.pt')) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - mixed.load_state_dict(torch.load('mixed.pt')) - ^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 1484, in load - with _open_file_like(f, "rb") as opened_file: - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 759, in _open_file_like - return _open_file(name_or_buffer, mode) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 740, in __init__ - super().__init__(open(name, mode)) - ^^^^^^^^^^^^^^^^ - FileNotFoundError: [Errno 2] No such file or directory: 'mixed.pt' -********************************************************************** -File "None", line ?, in default -Failed example: - model.b[5] # Get B-factor for atom 5 -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.b[5] # Get B-factor for atom 5 - ~~~~~~~^^^ - TypeError: 'NoneType' object is not subscriptable -********************************************************************** -File "None", line ?, in default -Failed example: - model.b[5:10] # Get B-factors for atoms 5-9 -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.b[5:10] # Get B-factors for atoms 5-9 - ~~~~~~~^^^^^^ - TypeError: 'NoneType' object is not subscriptable -********************************************************************** -File "None", line ?, in default -Failed example: - model.b[mask] # Get B-factors where mask is True -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.b[mask] # Get B-factors where mask is True - ~~~~~~~^^^^^^ - TypeError: 'NoneType' object is not subscriptable -********************************************************************** -File "None", line ?, in default -Failed example: - model.xyz[:, 0] # Get all x-coordinates -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.xyz[:, 0] # Get all x-coordinates - ~~~~~~~~~^^^^^^ - TypeError: 'NoneType' object is not subscriptable -********************************************************************** -File "None", line ?, in default -Failed example: - model.b[:] = 30.0 # Set all B-factors to 30 -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.b[:] = 30.0 # Set all B-factors to 30 - ~~~~~~~^^^ - TypeError: 'NoneType' object does not support item assignment -********************************************************************** -File "None", line ?, in default -Failed example: - model.b[5:10] = 25.0 # Set B-factors 5-9 to 25 -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.b[5:10] = 25.0 # Set B-factors 5-9 to 25 - ~~~~~~~^^^^^^ - TypeError: 'NoneType' object does not support item assignment -********************************************************************** -File "None", line ?, in default -Failed example: - model.b[mask] = new_values # Set B-factors where mask is True -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.b[mask] = new_values # Set B-factors where mask is True - ^^^^^^^^^^ - NameError: name 'new_values' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - model.xyz[mask] = new_coords # Set coordinates for masked atoms -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.xyz[mask] = new_coords # Set coordinates for masked atoms - ^^^^^^^^^^ - NameError: name 'new_coords' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - model.xyz[:, 0] += 1.0 # Shift all x-coordinates (read-modify-write) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.xyz[:, 0] += 1.0 # Shift all x-coordinates (read-modify-write) - ~~~~~~~~~^^^^^^ - TypeError: 'NoneType' object is not subscriptable -********************************************************************** -File "None", line ?, in default -Failed example: - mask = model.get_selection_mask("chain A") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - mask = model.get_selection_mask("chain A") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1407, in get_selection_mask - raise RuntimeError( - RuntimeError: Cannot get selection mask from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - new_coords = original_coords[mask] + shift -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - new_coords = original_coords[mask] + shift - ^^^^^^^^^^^^^^^ - NameError: name 'original_coords' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - model.xyz.set(new_coords, mask) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.xyz.set(new_coords, mask) - ^^^^^^^^^^^^^ - AttributeError: 'NoneType' object has no attribute 'set' -********************************************************************** -File "None", line ?, in default -Failed example: - mask = model.get_selection_mask("resseq 10:20") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - mask = model.get_selection_mask("resseq 10:20") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1407, in get_selection_mask - raise RuntimeError( - RuntimeError: Cannot get selection mask from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - model.b.set(new_b, mask) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.b.set(new_b, mask) - ^^^^^^^^^^^ - AttributeError: 'NoneType' object has no attribute 'set' -********************************************************************** -File "None", line ?, in default -Failed example: - b = PositiveMixedTensor() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - b = PositiveMixedTensor() - ^^^^^^^^^^^^^^^^^^^ - NameError: name 'PositiveMixedTensor' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - b.load_state_dict(torch.load('b_factors.pt')) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - b.load_state_dict(torch.load('b_factors.pt')) - ^^^^^^^^^^^^^^^^^ - AttributeError: 'NoneType' object has no attribute 'load_state_dict' -********************************************************************** -File "None", line ?, in default -Failed example: - b = PositiveMixedTensor(initial_b) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - b = PositiveMixedTensor(initial_b) - ^^^^^^^^^^^^^^^^^^^ - NameError: name 'PositiveMixedTensor' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - output = b() # Returns exp(log_b) = positive values -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - output = b() # Returns exp(log_b) = positive values - ^^^ - TypeError: 'NoneType' object is not callable -********************************************************************** -File "None", line ?, in default -Failed example: - assert (b() > 0).all() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - assert (b() > 0).all() - ^^^ - TypeError: 'NoneType' object is not callable -********************************************************************** -File "None", line ?, in default -Failed example: - mask = model.get_selection_mask("name CA") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - mask = model.get_selection_mask("name CA") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1407, in get_selection_mask - raise RuntimeError( - RuntimeError: Cannot get selection mask from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - model.b.set(new_b, mask) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.b.set(new_b, mask) - ^^^^^^^^^^^ - AttributeError: 'NoneType' object has no attribute 'set' -********************************************************************** -File "None", line ?, in default -Failed example: - occ = OccupancyTensor( - initial_values=torch.tensor([1.0, 1.0, 0.7, 0.7, 0.3, 0.3]), - sharing_groups=sharing_groups, - altloc_groups=[([2, 3], [4, 5])], - ) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - occ = OccupancyTensor( - ^^^^^^^^^^^^^^^ - NameError: name 'OccupancyTensor' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - result = occ() # Atoms 2-3 and 4-5 will sum to 1.0 -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - result = occ() # Atoms 2-3 and 4-5 will sum to 1.0 - ^^^ - NameError: name 'occ' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - freeze_mask = torch.zeros(n_atoms, dtype=torch.bool) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - freeze_mask = torch.zeros(n_atoms, dtype=torch.bool) - ^^^^^^^ - NameError: name 'n_atoms' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - freeze_mask[0:11] = True -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - freeze_mask[0:11] = True - ^^^^^^^^^^^ - NameError: name 'freeze_mask' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - occ.freeze(freeze_mask) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - occ.freeze(freeze_mask) - ^^^ - NameError: name 'occ' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - occ.freeze() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - occ.freeze() - ^^^ - NameError: name 'occ' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - unfreeze_mask = torch.zeros(n_atoms, dtype=torch.bool) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - unfreeze_mask = torch.zeros(n_atoms, dtype=torch.bool) - ^^^^^^^ - NameError: name 'n_atoms' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - unfreeze_mask[100:201] = True -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - unfreeze_mask[100:201] = True - ^^^^^^^^^^^^^ - NameError: name 'unfreeze_mask' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - occ.unfreeze(unfreeze_mask) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - occ.unfreeze(unfreeze_mask) - ^^^ - NameError: name 'occ' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - occ.unfreeze() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - occ.unfreeze() - ^^^ - NameError: name 'occ' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - atom_mask = torch.zeros(n_atoms, dtype=torch.bool) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - atom_mask = torch.zeros(n_atoms, dtype=torch.bool) - ^^^^^^^ - NameError: name 'n_atoms' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - atom_mask[:100] = True -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - atom_mask[:100] = True - ^^^^^^^^^ - NameError: name 'atom_mask' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - occ.update_refinable_mask(atom_mask, in_compressed_space=False) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - occ.update_refinable_mask(atom_mask, in_compressed_space=False) - ^^^ - NameError: name 'occ' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - group_mask = torch.zeros(n_groups, dtype=torch.bool) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - group_mask = torch.zeros(n_groups, dtype=torch.bool) - ^^^^^^^^ - NameError: name 'n_groups' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - group_mask[::2] = True -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - group_mask[::2] = True - ^^^^^^^^^^ - NameError: name 'group_mask' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - occ.update_refinable_mask(group_mask, in_compressed_space=True) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - occ.update_refinable_mask(group_mask, in_compressed_space=True) - ^^^ - NameError: name 'occ' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - model.load_state_dict(torch.load('model.pt')) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.load_state_dict(torch.load('model.pt')) - ^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 1484, in load - with _open_file_like(f, "rb") as opened_file: - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 759, in _open_file_like - return _open_file(name_or_buffer, mode) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 740, in __init__ - super().__init__(open(name, mode)) - ^^^^^^^^^^^^^^^^ - FileNotFoundError: [Errno 2] No such file or directory: 'model.pt' -********************************************************************** -File "None", line ?, in default -Failed example: - model.load_pdb('structure.pdb') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.load_pdb('structure.pdb') - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 231, in load_pdb - reader = pdb.PDBReader(verbose=self.verbose).read(file) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 339, in read - self.dataframe = load_as_dataframe(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 149, in load_as_dataframe - skipheader = find_header_length(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 67, in find_header_length - with open(filepath, "r") as f: - ^^^^^^^^^^^^^^^^^^^ - FileNotFoundError: [Errno 2] No such file or directory: 'structure.pdb' -********************************************************************** -File "None", line ?, in default -Failed example: - model = Model().load_pdb('structure.pdb') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model = Model().load_pdb('structure.pdb') - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 231, in load_pdb - reader = pdb.PDBReader(verbose=self.verbose).read(file) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 339, in read - self.dataframe = load_as_dataframe(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 149, in load_as_dataframe - skipheader = find_header_length(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 67, in find_header_length - with open(filepath, "r") as f: - ^^^^^^^^^^^^^^^^^^^ - FileNotFoundError: [Errno 2] No such file or directory: 'structure.pdb' -********************************************************************** -File "None", line ?, in default -Failed example: - model_copy = model.copy() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model_copy = model.copy() - ^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 490, in copy - raise RuntimeError("Cannot copy an uninitialized Model. Load data first.") - RuntimeError: Cannot copy an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - model.update_mask_from_selection("chain A", "xyz", mode='set', freeze=True) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.update_mask_from_selection("chain A", "xyz", mode='set', freeze=True) - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 646, in update_mask_from_selection - current_mask = getattr(self, mask_name) - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1964, in __getattr__ - raise AttributeError( - AttributeError: 'Model' object has no attribute 'xyz_mask' -********************************************************************** -File "None", line ?, in default -Failed example: - model.apply_mask_to_parameter("xyz") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.apply_mask_to_parameter("xyz") - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 699, in apply_mask_to_parameter - self.xyz.update_refinable_mask(self.xyz_mask) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - AttributeError: 'NoneType' object has no attribute 'update_refinable_mask' -********************************************************************** -File "None", line ?, in default -Failed example: - model.update_mask_from_selection("name CA or name C or name N", "xyz", freeze=False) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.update_mask_from_selection("name CA or name C or name N", "xyz", freeze=False) - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 646, in update_mask_from_selection - current_mask = getattr(self, mask_name) - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1964, in __getattr__ - raise AttributeError( - AttributeError: 'Model' object has no attribute 'xyz_mask' -********************************************************************** -File "None", line ?, in default -Failed example: - model.apply_mask_to_parameter("xyz") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.apply_mask_to_parameter("xyz") - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 699, in apply_mask_to_parameter - self.xyz.update_refinable_mask(self.xyz_mask) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - AttributeError: 'NoneType' object has no attribute 'update_refinable_mask' -********************************************************************** -File "None", line ?, in default -Failed example: - model.update_mask_from_selection("chain A", "xyz", freeze=True) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.update_mask_from_selection("chain A", "xyz", freeze=True) - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 646, in update_mask_from_selection - current_mask = getattr(self, mask_name) - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1964, in __getattr__ - raise AttributeError( - AttributeError: 'Model' object has no attribute 'xyz_mask' -********************************************************************** -File "None", line ?, in default -Failed example: - model.apply_mask_to_parameter("xyz") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.apply_mask_to_parameter("xyz") - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 699, in apply_mask_to_parameter - self.xyz.update_refinable_mask(self.xyz_mask) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - AttributeError: 'NoneType' object has no attribute 'update_refinable_mask' -********************************************************************** -File "None", line ?, in default -Failed example: - model.freeze_selection("chain A", targets='all') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.freeze_selection("chain A", targets='all') - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 752, in freeze_selection - self.update_mask_from_selection( - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 646, in update_mask_from_selection - current_mask = getattr(self, mask_name) - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1964, in __getattr__ - raise AttributeError( - AttributeError: 'Model' object has no attribute 'xyz_mask' -********************************************************************** -File "None", line ?, in default -Failed example: - model.freeze_selection("resseq 10:20", targets='xyz') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.freeze_selection("resseq 10:20", targets='xyz') - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 752, in freeze_selection - self.update_mask_from_selection( - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 646, in update_mask_from_selection - current_mask = getattr(self, mask_name) - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1964, in __getattr__ - raise AttributeError( - AttributeError: 'Model' object has no attribute 'xyz_mask' -********************************************************************** -File "None", line ?, in default -Failed example: - model.unfreeze_selection("chain A", targets='all') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.unfreeze_selection("chain A", targets='all') - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 792, in unfreeze_selection - self.update_mask_from_selection( - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 646, in update_mask_from_selection - current_mask = getattr(self, mask_name) - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1964, in __getattr__ - raise AttributeError( - AttributeError: 'Model' object has no attribute 'xyz_mask' -********************************************************************** -File "None", line ?, in default -Failed example: - model.unfreeze_selection("name CA or name C or name N", targets='xyz') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.unfreeze_selection("name CA or name C or name N", targets='xyz') - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 792, in unfreeze_selection - self.update_mask_from_selection( - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 646, in update_mask_from_selection - current_mask = getattr(self, mask_name) - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1964, in __getattr__ - raise AttributeError( - AttributeError: 'Model' object has no attribute 'xyz_mask' -********************************************************************** -File "None", line ?, in default -Failed example: - structure_factor_loss = compute_structure_factor_loss() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - structure_factor_loss = compute_structure_factor_loss() - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - NameError: name 'compute_structure_factor_loss' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - nll_reg = model.adp_nll_loss(target_log_std=0.2) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - nll_reg = model.adp_nll_loss(target_log_std=0.2) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 986, in adp_nll_loss - log_b = super(PositiveMixedTensor, self.b).forward() - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - AttributeError: 'super' object has no attribute 'forward' -********************************************************************** -File "None", line ?, in default -Failed example: - total_loss = structure_factor_loss + 0.01 * nll_reg -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - total_loss = structure_factor_loss + 0.01 * nll_reg - ^^^^^^^^^^^^^^^^^^^^^ - NameError: name 'structure_factor_loss' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - total_loss.backward() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - total_loss.backward() - ^^^^^^^^^^ - NameError: name 'total_loss' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - atom_nll = model.adp_nll_loss_per_atom(target_log_std=0.2) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - atom_nll = model.adp_nll_loss_per_atom(target_log_std=0.2) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1041, in adp_nll_loss_per_atom - log_b = super(PositiveMixedTensor, self.b).forward() - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - AttributeError: 'super' object has no attribute 'forward' -********************************************************************** -File "None", line ?, in default -Failed example: - threshold = atom_nll.mean() + 2 * atom_nll.std() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - threshold = atom_nll.mean() + 2 * atom_nll.std() - ^^^^^^^^ - NameError: name 'atom_nll' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - outliers = atom_nll > threshold -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - outliers = atom_nll > threshold - ^^^^^^^^ - NameError: name 'atom_nll' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - loss = xray_loss + w_adp * model.adp_kl_divergence_loss(0.2) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - loss = xray_loss + w_adp * model.adp_kl_divergence_loss(0.2) - ^^^^^^^^^ - NameError: name 'xray_loss' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - model = Model().load_pdb('structure.pdb') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model = Model().load_pdb('structure.pdb') - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 231, in load_pdb - reader = pdb.PDBReader(verbose=self.verbose).read(file) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 339, in read - self.dataframe = load_as_dataframe(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 149, in load_as_dataframe - skipheader = find_header_length(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 67, in find_header_length - with open(filepath, "r") as f: - ^^^^^^^^^^^^^^^^^^^ - FileNotFoundError: [Errno 2] No such file or directory: 'structure.pdb' -********************************************************************** -File "None", line ?, in default -Failed example: - mask = model.get_selection_mask("chain A") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - mask = model.get_selection_mask("chain A") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1407, in get_selection_mask - raise RuntimeError( - RuntimeError: Cannot get selection mask from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - new_coords = model.xyz()[mask] + translation -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - new_coords = model.xyz()[mask] + translation - ^^^^^^^^^^^ - TypeError: 'NoneType' object is not callable -********************************************************************** -File "None", line ?, in default -Failed example: - model.xyz.set(new_coords, mask) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.xyz.set(new_coords, mask) - ^^^^^^^^^^^^^ - AttributeError: 'NoneType' object has no attribute 'set' -********************************************************************** -File "None", line ?, in default -Failed example: - backbone_mask = model.get_selection_mask("name CA or name C or name N or name O") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - backbone_mask = model.get_selection_mask("name CA or name C or name N or name O") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1407, in get_selection_mask - raise RuntimeError( - RuntimeError: Cannot get selection mask from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - mask = model.get_selection_mask("chain A and (resname ALA or resname GLY)") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - mask = model.get_selection_mask("chain A and (resname ALA or resname GLY)") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1407, in get_selection_mask - raise RuntimeError( - RuntimeError: Cannot get selection mask from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - model = Model().load_pdb('structure.pdb') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model = Model().load_pdb('structure.pdb') - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 231, in load_pdb - reader = pdb.PDBReader(verbose=self.verbose).read(file) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 339, in read - self.dataframe = load_as_dataframe(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 149, in load_as_dataframe - skipheader = find_header_length(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 67, in find_header_length - with open(filepath, "r") as f: - ^^^^^^^^^^^^^^^^^^^ - FileNotFoundError: [Errno 2] No such file or directory: 'structure.pdb' -********************************************************************** -File "None", line ?, in default -Failed example: - chain_a = model.select("chain A") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - chain_a = model.select("chain A") - ^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1473, in select - raise RuntimeError( - RuntimeError: Cannot select from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - backbone = model.select("name CA or name C or name N or name O") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - backbone = model.select("name CA or name C or name N or name O") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1473, in select - raise RuntimeError( - RuntimeError: Cannot select from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - region = model.select("chain B and resseq 10:50") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - region = model.select("chain B and resseq 10:50") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1473, in select - raise RuntimeError( - RuntimeError: Cannot select from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - no_water = model.select("not resname HOH") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - no_water = model.select("not resname HOH") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1473, in select - raise RuntimeError( - RuntimeError: Cannot select from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - complex_sel = model.select("chain A and (resname ALA or resname GLY)") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - complex_sel = model.select("chain A and (resname ALA or resname GLY)") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1473, in select - raise RuntimeError( - RuntimeError: Cannot select from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - model.load_state_dict(torch.load('model.pt')) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.load_state_dict(torch.load('model.pt')) - ^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 1484, in load - with _open_file_like(f, "rb") as opened_file: - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 759, in _open_file_like - return _open_file(name_or_buffer, mode) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 740, in __init__ - super().__init__(open(name, mode)) - ^^^^^^^^^^^^^^^^ - FileNotFoundError: [Errno 2] No such file or directory: 'model.pt' -********************************************************************** -File "None", line ?, in default -Failed example: - model.load_pdb('structure.pdb') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.load_pdb('structure.pdb') - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model_ft.py", line 140, in load_pdb - super().load_pdb(filename) - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 231, in load_pdb - reader = pdb.PDBReader(verbose=self.verbose).read(file) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 339, in read - self.dataframe = load_as_dataframe(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 149, in load_as_dataframe - skipheader = find_header_length(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 67, in find_header_length - with open(filepath, "r") as f: - ^^^^^^^^^^^^^^^^^^^ - FileNotFoundError: [Errno 2] No such file or directory: 'structure.pdb' -********************************************************************** -File "None", line ?, in default -Failed example: - model = Model().load_pdb('structure.pdb') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model = Model().load_pdb('structure.pdb') - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 231, in load_pdb - reader = pdb.PDBReader(verbose=self.verbose).read(file) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 339, in read - self.dataframe = load_as_dataframe(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 149, in load_as_dataframe - skipheader = find_header_length(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 67, in find_header_length - with open(filepath, "r") as f: - ^^^^^^^^^^^^^^^^^^^ - FileNotFoundError: [Errno 2] No such file or directory: 'structure.pdb' -********************************************************************** -File "None", line ?, in default -Failed example: - chain_a = model.select("chain A") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - chain_a = model.select("chain A") - ^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model_ft.py", line 146, in select - selection = super().select(selection) - ^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1473, in select - raise RuntimeError( - RuntimeError: Cannot select from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - backbone = model.select("name CA or name C or name N or name O") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - backbone = model.select("name CA or name C or name N or name O") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model_ft.py", line 146, in select - selection = super().select(selection) - ^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1473, in select - raise RuntimeError( - RuntimeError: Cannot select from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - region = model.select("chain B and resseq 10:50") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - region = model.select("chain B and resseq 10:50") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model_ft.py", line 146, in select - selection = super().select(selection) - ^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1473, in select - raise RuntimeError( - RuntimeError: Cannot select from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - no_water = model.select("not resname HOH") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - no_water = model.select("not resname HOH") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model_ft.py", line 146, in select - selection = super().select(selection) - ^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1473, in select - raise RuntimeError( - RuntimeError: Cannot select from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - complex_sel = model.select("chain A and (resname ALA or resname GLY)") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - complex_sel = model.select("chain A and (resname ALA or resname GLY)") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model_ft.py", line 146, in select - selection = super().select(selection) - ^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1473, in select - raise RuntimeError( - RuntimeError: Cannot select from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - model = ModelFT().load_pdb('structure.pdb') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model = ModelFT().load_pdb('structure.pdb') - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model_ft.py", line 140, in load_pdb - super().load_pdb(filename) - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 231, in load_pdb - reader = pdb.PDBReader(verbose=self.verbose).read(file) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 339, in read - self.dataframe = load_as_dataframe(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 149, in load_as_dataframe - skipheader = find_header_length(filepath) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/pdb.py", line 67, in find_header_length - with open(filepath, "r") as f: - ^^^^^^^^^^^^^^^^^^^ - FileNotFoundError: [Errno 2] No such file or directory: 'structure.pdb' -********************************************************************** -File "None", line ?, in default -Failed example: - model_copy = model.copy() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model_copy = model.copy() - ^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model_ft.py", line 670, in copy - raise RuntimeError("Cannot copy an uninitialized ModelFT. Load data first.") - RuntimeError: Cannot copy an uninitialized ModelFT. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - mixed.load_state_dict(torch.load('mixed.pt')) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - mixed.load_state_dict(torch.load('mixed.pt')) - ^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 1484, in load - with _open_file_like(f, "rb") as opened_file: - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 759, in _open_file_like - return _open_file(name_or_buffer, mode) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 740, in __init__ - super().__init__(open(name, mode)) - ^^^^^^^^^^^^^^^^ - FileNotFoundError: [Errno 2] No such file or directory: 'mixed.pt' -********************************************************************** -File "None", line ?, in default -Failed example: - model.b[5] # Get B-factor for atom 5 -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.b[5] # Get B-factor for atom 5 - ~~~~~~~^^^ - TypeError: 'NoneType' object is not subscriptable -********************************************************************** -File "None", line ?, in default -Failed example: - model.b[5:10] # Get B-factors for atoms 5-9 -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.b[5:10] # Get B-factors for atoms 5-9 - ~~~~~~~^^^^^^ - TypeError: 'NoneType' object is not subscriptable -********************************************************************** -File "None", line ?, in default -Failed example: - model.b[mask] # Get B-factors where mask is True -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.b[mask] # Get B-factors where mask is True - ~~~~~~~^^^^^^ - TypeError: 'NoneType' object is not subscriptable -********************************************************************** -File "None", line ?, in default -Failed example: - model.xyz[:, 0] # Get all x-coordinates -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.xyz[:, 0] # Get all x-coordinates - ~~~~~~~~~^^^^^^ - TypeError: 'NoneType' object is not subscriptable -********************************************************************** -File "None", line ?, in default -Failed example: - model.b[:] = 30.0 # Set all B-factors to 30 -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.b[:] = 30.0 # Set all B-factors to 30 - ~~~~~~~^^^ - TypeError: 'NoneType' object does not support item assignment -********************************************************************** -File "None", line ?, in default -Failed example: - model.b[5:10] = 25.0 # Set B-factors 5-9 to 25 -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.b[5:10] = 25.0 # Set B-factors 5-9 to 25 - ~~~~~~~^^^^^^ - TypeError: 'NoneType' object does not support item assignment -********************************************************************** -File "None", line ?, in default -Failed example: - model.b[mask] = new_values # Set B-factors where mask is True -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.b[mask] = new_values # Set B-factors where mask is True - ^^^^^^^^^^ - NameError: name 'new_values' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - model.xyz[mask] = new_coords # Set coordinates for masked atoms -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.xyz[mask] = new_coords # Set coordinates for masked atoms - ^^^^^^^^^^ - NameError: name 'new_coords' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - model.xyz[:, 0] += 1.0 # Shift all x-coordinates (read-modify-write) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.xyz[:, 0] += 1.0 # Shift all x-coordinates (read-modify-write) - ~~~~~~~~~^^^^^^ - TypeError: 'NoneType' object is not subscriptable -********************************************************************** -File "None", line ?, in default -Failed example: - mask = model.get_selection_mask("chain A") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - mask = model.get_selection_mask("chain A") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1407, in get_selection_mask - raise RuntimeError( - RuntimeError: Cannot get selection mask from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - new_coords = original_coords[mask] + shift -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - new_coords = original_coords[mask] + shift - ^^^^^^^^^^^^^^^ - NameError: name 'original_coords' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - model.xyz.set(new_coords, mask) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.xyz.set(new_coords, mask) - ^^^^^^^^^^^^^ - AttributeError: 'NoneType' object has no attribute 'set' -********************************************************************** -File "None", line ?, in default -Failed example: - mask = model.get_selection_mask("resseq 10:20") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - mask = model.get_selection_mask("resseq 10:20") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1407, in get_selection_mask - raise RuntimeError( - RuntimeError: Cannot get selection mask from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - model.b.set(new_b, mask) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.b.set(new_b, mask) - ^^^^^^^^^^^ - AttributeError: 'NoneType' object has no attribute 'set' -********************************************************************** -File "None", line ?, in default -Failed example: - b = PositiveMixedTensor() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - b = PositiveMixedTensor() - ^^^^^^^^^^^^^^^^^^^ - NameError: name 'PositiveMixedTensor' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - b.load_state_dict(torch.load('b_factors.pt')) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - b.load_state_dict(torch.load('b_factors.pt')) - ^^^^^^^^^^^^^^^^^ - AttributeError: 'NoneType' object has no attribute 'load_state_dict' -********************************************************************** -File "None", line ?, in default -Failed example: - b = PositiveMixedTensor(initial_b) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - b = PositiveMixedTensor(initial_b) - ^^^^^^^^^^^^^^^^^^^ - NameError: name 'PositiveMixedTensor' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - output = b() # Returns exp(log_b) = positive values -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - output = b() # Returns exp(log_b) = positive values - ^^^ - TypeError: 'NoneType' object is not callable -********************************************************************** -File "None", line ?, in default -Failed example: - assert (b() > 0).all() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - assert (b() > 0).all() - ^^^ - TypeError: 'NoneType' object is not callable -********************************************************************** -File "None", line ?, in default -Failed example: - mask = model.get_selection_mask("name CA") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - mask = model.get_selection_mask("name CA") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 1407, in get_selection_mask - raise RuntimeError( - RuntimeError: Cannot get selection mask from an uninitialized Model. Load data first. -********************************************************************** -File "None", line ?, in default -Failed example: - model.b.set(new_b, mask) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.b.set(new_b, mask) - ^^^^^^^^^^^ - AttributeError: 'NoneType' object has no attribute 'set' -********************************************************************** -File "None", line ?, in default -Failed example: - occ = OccupancyTensor( - initial_values=torch.tensor([1.0, 1.0, 0.7, 0.7, 0.3, 0.3]), - sharing_groups=sharing_groups, - altloc_groups=[([2, 3], [4, 5])], - ) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - occ = OccupancyTensor( - ^^^^^^^^^^^^^^^ - NameError: name 'OccupancyTensor' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - result = occ() # Atoms 2-3 and 4-5 will sum to 1.0 -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - result = occ() # Atoms 2-3 and 4-5 will sum to 1.0 - ^^^ - NameError: name 'occ' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - freeze_mask = torch.zeros(n_atoms, dtype=torch.bool) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - freeze_mask = torch.zeros(n_atoms, dtype=torch.bool) - ^^^^^^^ - NameError: name 'n_atoms' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - freeze_mask[0:11] = True -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - freeze_mask[0:11] = True - ^^^^^^^^^^^ - NameError: name 'freeze_mask' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - occ.freeze(freeze_mask) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - occ.freeze(freeze_mask) - ^^^ - NameError: name 'occ' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - occ.freeze() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - occ.freeze() - ^^^ - NameError: name 'occ' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - unfreeze_mask = torch.zeros(n_atoms, dtype=torch.bool) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - unfreeze_mask = torch.zeros(n_atoms, dtype=torch.bool) - ^^^^^^^ - NameError: name 'n_atoms' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - unfreeze_mask[100:201] = True -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - unfreeze_mask[100:201] = True - ^^^^^^^^^^^^^ - NameError: name 'unfreeze_mask' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - occ.unfreeze(unfreeze_mask) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - occ.unfreeze(unfreeze_mask) - ^^^ - NameError: name 'occ' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - occ.unfreeze() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - occ.unfreeze() - ^^^ - NameError: name 'occ' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - atom_mask = torch.zeros(n_atoms, dtype=torch.bool) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - atom_mask = torch.zeros(n_atoms, dtype=torch.bool) - ^^^^^^^ - NameError: name 'n_atoms' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - atom_mask[:100] = True -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - atom_mask[:100] = True - ^^^^^^^^^ - NameError: name 'atom_mask' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - occ.update_refinable_mask(atom_mask, in_compressed_space=False) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - occ.update_refinable_mask(atom_mask, in_compressed_space=False) - ^^^ - NameError: name 'occ' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - group_mask = torch.zeros(n_groups, dtype=torch.bool) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - group_mask = torch.zeros(n_groups, dtype=torch.bool) - ^^^^^^^^ - NameError: name 'n_groups' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - group_mask[::2] = True -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - group_mask[::2] = True - ^^^^^^^^^^ - NameError: name 'group_mask' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - occ.update_refinable_mask(group_mask, in_compressed_space=True) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - occ.update_refinable_mask(group_mask, in_compressed_space=True) - ^^^ - NameError: name 'occ' is not defined -********************************************************************** -1 items had failures: - 167 of 207 in default -207 tests in 1 items. -40 passed and 167 failed. -***Test Failed*** 167 failures. - -Document: api/refinement ------------------------- -********************************************************************** -File "None", line ?, in default -Failed example: - refinement = Refinement( - data_file='reflections.mtz', - pdb='structure.pdb', - device='cuda' - ) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - refinement = Refinement( - ^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/refinement/base_refinement.py", line 195, in __init__ - raise e - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/refinement/base_refinement.py", line 145, in __init__ - self.reflection_data.load_mtz(data_file) - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/datasets/reflection_data.py", line 233, in load_mtz - reader = mtz.MTZReader(verbose=self.verbose).read(path) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/mtz.py", line 158, in read - self.mtz_data = rs.read_mtz(filepath) - ^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/reciprocalspaceship/io/mtz.py", line 186, in read_mtz - gemmi_mtz = gemmi.read_mtz_file(mtzfile) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - RuntimeError: Failed to open reflections.mtz: No such file or directory: reflections.mtz -********************************************************************** -File "None", line ?, in default -Failed example: - refinement.run_refinement(macro_cycles=10) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - refinement.run_refinement(macro_cycles=10) - ^^^^^^^^^^ - NameError: name 'refinement' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - refinement.load_state_dict(torch.load('refinement.pt')) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - refinement.load_state_dict(torch.load('refinement.pt')) - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/nn/modules/module.py", line 2629, in load_state_dict - raise RuntimeError( - RuntimeError: Error(s) in loading state_dict for Refinement: - Unexpected key(s) in state_dict: "model.pdb", "model.spacegroup", "model.initialized", "model.dtype_float", "model.device", "model.strip_H", "model.altloc_pairs", "model.max_res", "model.radius_angstrom", "scaler.nbins", "scaler.verbose", "scaler.frozen". -********************************************************************** -File "None", line ?, in default -Failed example: - refinement = Refinement(data_file='data.mtz', pdb='model.pdb') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - refinement = Refinement(data_file='data.mtz', pdb='model.pdb') - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/refinement/base_refinement.py", line 195, in __init__ - raise e - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/refinement/base_refinement.py", line 145, in __init__ - self.reflection_data.load_mtz(data_file) - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/datasets/reflection_data.py", line 233, in load_mtz - reader = mtz.MTZReader(verbose=self.verbose).read(path) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/mtz.py", line 158, in read - self.mtz_data = rs.read_mtz(filepath) - ^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/reciprocalspaceship/io/mtz.py", line 186, in read_mtz - gemmi_mtz = gemmi.read_mtz_file(mtzfile) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - RuntimeError: Failed to open data.mtz: No such file or directory: data.mtz -********************************************************************** -File "None", line ?, in default -Failed example: - refinement = Refinement.create_from_state_dict(state) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - refinement = Refinement.create_from_state_dict(state) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/refinement/base_refinement.py", line 1096, in create_from_state_dict - scaler = Scaler(model, reflection_data, verbose=verbose, device=device) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/scaling/scaler.py", line 119, in __init__ - self.register_buffer("s", get_scattering_vectors(data.hkl, self.cell)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/math_functions/math_torch.py", line 942, in get_scattering_vectors - recB = reciprocal_basis_matrix(unit_cell) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/math_functions/math_torch.py", line 885, in reciprocal_basis_matrix - angles_rad = torch.deg2rad(unit_cell[3:]) - ~~~~~~~~~^^^^ - TypeError: 'NoneType' object is not subscriptable -********************************************************************** -File "None", line ?, in default -Failed example: - rwork, rfree = refinement.get_rfactor() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - rwork, rfree = refinement.get_rfactor() - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/refinement/base_refinement.py", line 951, in get_rfactor - return self.scaler.rfactor() - ^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/scaling/scaler.py", line 358, in rfactor - hkl, fobs, _, rfree = self._data() - ^^^^^^^^^^^^ - TypeError: 'NoneType' object is not callable -********************************************************************** -File "None", line ?, in default -Failed example: - print(f"Restored at R-work={rwork:.4f}, R-free={rfree:.4f}") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - print(f"Restored at R-work={rwork:.4f}, R-free={rfree:.4f}") - ^^^^^ - NameError: name 'rwork' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - from torchref.refinement.loss_weighting import ResolutionDependentWeighting -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - from torchref.refinement.loss_weighting import ResolutionDependentWeighting - ModuleNotFoundError: No module named 'torchref.refinement.loss_weighting' -********************************************************************** -File "None", line ?, in default -Failed example: - weighter = ResolutionDependentWeighting() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - weighter = ResolutionDependentWeighting() - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - NameError: name 'ResolutionDependentWeighting' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - refinement = LBFGSRefinement(mtz_file, pdb_file, weighter=weighter, target_mode='ml') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - refinement = LBFGSRefinement(mtz_file, pdb_file, weighter=weighter, target_mode='ml') - ^^^^^^^^ - NameError: name 'mtz_file' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - refinement.refine(macro_cycles=2) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - refinement.refine(macro_cycles=2) - ^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1964, in __getattr__ - raise AttributeError( - AttributeError: 'Refinement' object has no attribute 'refine' -********************************************************************** -File "None", line ?, in default -Failed example: - policy = PolicyComponentWeighting( - refinement, policy_path='policy.pt', - sample=True, temperature=1.0 - ) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - policy = PolicyComponentWeighting( - ^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/refinement/weighting/policy_weighting.py", line 199, in __init__ - self._load_policy(policy_path) - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/refinement/weighting/policy_weighting.py", line 301, in _load_policy - checkpoint = torch.load(checkpoint_path, map_location="cpu", weights_only=False) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 1484, in load - with _open_file_like(f, "rb") as opened_file: - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 759, in _open_file_like - return _open_file(name_or_buffer, mode) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 740, in __init__ - super().__init__(open(name, mode)) - ^^^^^^^^^^^^^^^^ - FileNotFoundError: [Errno 2] No such file or directory: 'policy.pt' -********************************************************************** -File "None", line ?, in default -Failed example: - trajectory = refinement.run_training_trajectory( - policy, n_steps=10, pdb_id='3GR5' - ) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - trajectory = refinement.run_training_trajectory( - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1964, in __getattr__ - raise AttributeError( - AttributeError: 'Refinement' object has no attribute 'run_training_trajectory' -********************************************************************** -File "None", line ?, in default -Failed example: - with open('trajectory.json', 'w') as f: - json.dump(trajectory_to_dict(trajectory), f) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 2, in - json.dump(trajectory_to_dict(trajectory), f) - ^^^^^^^^^^ - NameError: name 'trajectory' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - refinement.load_state_dict(torch.load('refinement.pt')) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - refinement.load_state_dict(torch.load('refinement.pt')) - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/nn/modules/module.py", line 2629, in load_state_dict - raise RuntimeError( - RuntimeError: Error(s) in loading state_dict for Refinement: - Unexpected key(s) in state_dict: "model.pdb", "model.spacegroup", "model.initialized", "model.dtype_float", "model.device", "model.strip_H", "model.altloc_pairs", "model.max_res", "model.radius_angstrom", "scaler.nbins", "scaler.verbose", "scaler.frozen". -********************************************************************** -File "None", line ?, in default -Failed example: - refinement = Refinement(data_file='data.mtz', pdb='model.pdb') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - refinement = Refinement(data_file='data.mtz', pdb='model.pdb') - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/refinement/base_refinement.py", line 195, in __init__ - raise e - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/refinement/base_refinement.py", line 145, in __init__ - self.reflection_data.load_mtz(data_file) - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/datasets/reflection_data.py", line 233, in load_mtz - reader = mtz.MTZReader(verbose=self.verbose).read(path) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/mtz.py", line 158, in read - self.mtz_data = rs.read_mtz(filepath) - ^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/reciprocalspaceship/io/mtz.py", line 186, in read_mtz - gemmi_mtz = gemmi.read_mtz_file(mtzfile) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - RuntimeError: Failed to open data.mtz: No such file or directory: data.mtz -********************************************************************** -File "None", line ?, in default -Failed example: - refinement = Refinement.create_from_state_dict(state) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - refinement = Refinement.create_from_state_dict(state) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/refinement/base_refinement.py", line 1096, in create_from_state_dict - scaler = Scaler(model, reflection_data, verbose=verbose, device=device) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/scaling/scaler.py", line 119, in __init__ - self.register_buffer("s", get_scattering_vectors(data.hkl, self.cell)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/math_functions/math_torch.py", line 942, in get_scattering_vectors - recB = reciprocal_basis_matrix(unit_cell) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/math_functions/math_torch.py", line 885, in reciprocal_basis_matrix - angles_rad = torch.deg2rad(unit_cell[3:]) - ~~~~~~~~~^^^^ - TypeError: 'NoneType' object is not subscriptable -********************************************************************** -File "None", line ?, in default -Failed example: - rwork, rfree = refinement.get_rfactor() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - rwork, rfree = refinement.get_rfactor() - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/refinement/base_refinement.py", line 951, in get_rfactor - return self.scaler.rfactor() - ^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/scaling/scaler.py", line 358, in rfactor - hkl, fobs, _, rfree = self._data() - ^^^^^^^^^^^^ - TypeError: 'NoneType' object is not callable -********************************************************************** -File "None", line ?, in default -Failed example: - print(f"Restored at R-work={rwork:.4f}, R-free={rfree:.4f}") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - print(f"Restored at R-work={rwork:.4f}, R-free={rfree:.4f}") - ^^^^^ - NameError: name 'rwork' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - target = Target() # Creates empty shell -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - target = Target() # Creates empty shell - ^^^^^^ - NameError: name 'Target' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - target.load_state_dict(torch.load('target.pt')) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - target.load_state_dict(torch.load('target.pt')) - ^^^^^^ - NameError: name 'target' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - target = Target(refinement) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - target = Target(refinement) - ^^^^^^ - NameError: name 'Target' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - geom_target = TotalGeometryTarget(refinement) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - geom_target = TotalGeometryTarget(refinement) - ^^^^^^^^^^^^^^^^^^^ - NameError: name 'TotalGeometryTarget' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - loss = geom_target() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - loss = geom_target() - ^^^^^^^^^^^ - NameError: name 'geom_target' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - bond_loss = geom_target['bond']() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - bond_loss = geom_target['bond']() - ^^^^^^^^^^^ - NameError: name 'geom_target' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - for name, target in geom_target.items(): - print(f"{name}: {target()}") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - for name, target in geom_target.items(): - ^^^^^^^^^^^ - NameError: name 'geom_target' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - adp_target = TotalADPTarget(refinement) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - adp_target = TotalADPTarget(refinement) - ^^^^^^^^^^^^^^ - NameError: name 'TotalADPTarget' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - loss = adp_target() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - loss = adp_target() - ^^^^^^^^^^ - NameError: name 'adp_target' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - simu_loss = adp_target['simu']() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - simu_loss = adp_target['simu']() - ^^^^^^^^^^ - NameError: name 'adp_target' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - for name, target in adp_target.items(): - print(f"{name}: {target()}") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - for name, target in adp_target.items(): - ^^^^^^^^^^ - NameError: name 'adp_target' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - from torchref.refinement.loss_weighting import ResolutionDependentWeighting -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - from torchref.refinement.loss_weighting import ResolutionDependentWeighting - ModuleNotFoundError: No module named 'torchref.refinement.loss_weighting' -********************************************************************** -File "None", line ?, in default -Failed example: - weighter = ResolutionDependentWeighting() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - weighter = ResolutionDependentWeighting() - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - NameError: name 'ResolutionDependentWeighting' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - refinement = LBFGSRefinement(mtz_file, pdb_file, weighter=weighter, target_mode='ml') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - refinement = LBFGSRefinement(mtz_file, pdb_file, weighter=weighter, target_mode='ml') - ^^^^^^^^ - NameError: name 'mtz_file' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - refinement.refine(macro_cycles=2) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - refinement.refine(macro_cycles=2) - ^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1964, in __getattr__ - raise AttributeError( - AttributeError: 'Refinement' object has no attribute 'refine' -********************************************************************** -File "None", line ?, in default -Failed example: - policy = PolicyComponentWeighting( - refinement, policy_path='policy.pt', - sample=True, temperature=1.0 - ) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - policy = PolicyComponentWeighting( - ^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/refinement/weighting/policy_weighting.py", line 199, in __init__ - self._load_policy(policy_path) - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/refinement/weighting/policy_weighting.py", line 301, in _load_policy - checkpoint = torch.load(checkpoint_path, map_location="cpu", weights_only=False) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 1484, in load - with _open_file_like(f, "rb") as opened_file: - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 759, in _open_file_like - return _open_file(name_or_buffer, mode) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 740, in __init__ - super().__init__(open(name, mode)) - ^^^^^^^^^^^^^^^^ - FileNotFoundError: [Errno 2] No such file or directory: 'policy.pt' -********************************************************************** -File "None", line ?, in default -Failed example: - trajectory = refinement.run_training_trajectory( - policy, n_steps=10, pdb_id='3GR5' - ) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - trajectory = refinement.run_training_trajectory( - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1964, in __getattr__ - raise AttributeError( - AttributeError: 'Refinement' object has no attribute 'run_training_trajectory' -********************************************************************** -File "None", line ?, in default -Failed example: - with open('trajectory.json', 'w') as f: - json.dump(trajectory_to_dict(trajectory), f) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 2, in - json.dump(trajectory_to_dict(trajectory), f) - ^^^^^^^^^^ - NameError: name 'trajectory' is not defined -********************************************************************** -1 items had failures: - 37 of 53 in default -53 tests in 1 items. -16 passed and 37 failed. -***Test Failed*** 37 failures. - -Document: api/restraints ------------------------- -********************************************************************** -File "None", line ?, in default -Failed example: - iterator = ResidueIterator(model.pdb) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - iterator = ResidueIterator(model.pdb) - ^^^^^^^^^^^^^^^ - NameError: name 'ResidueIterator' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - for chain_id, resseq, residue_df in iterator: - print(f"Processing {chain_id}:{resseq}") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - for chain_id, resseq, residue_df in iterator: - ^^^^^^^^ - NameError: name 'iterator' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - builder = BondRestraintBuilder() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - builder = BondRestraintBuilder() - ^^^^^^^^^^^^^^^^^^^^ - NameError: name 'BondRestraintBuilder' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - for residue in residues: - builder.process_residue(residue, cif_bonds) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - for residue in residues: - ^^^^^^^^ - NameError: name 'residues' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - result = builder.finalize(device) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - result = builder.finalize(device) - ^^^^^^^ - NameError: name 'builder' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - print(result['indices'].shape) # (n_bonds, 2) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - print(result['indices'].shape) # (n_bonds, 2) - ^^^^^^ - NameError: name 'result' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - builder = AngleRestraintBuilder() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - builder = AngleRestraintBuilder() - ^^^^^^^^^^^^^^^^^^^^^ - NameError: name 'AngleRestraintBuilder' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - for residue in residues: - builder.process_residue(residue, cif_angles) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - for residue in residues: - ^^^^^^^^ - NameError: name 'residues' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - result = builder.finalize(device) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - result = builder.finalize(device) - ^^^^^^^ - NameError: name 'builder' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - print(result['indices'].shape) # (n_angles, 3) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - print(result['indices'].shape) # (n_angles, 3) - ^^^^^^ - NameError: name 'result' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - builder = TorsionRestraintBuilder() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - builder = TorsionRestraintBuilder() - ^^^^^^^^^^^^^^^^^^^^^^^ - NameError: name 'TorsionRestraintBuilder' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - for residue in residues: - builder.process_residue(residue, cif_torsions) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - for residue in residues: - ^^^^^^^^ - NameError: name 'residues' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - result = builder.finalize(device) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - result = builder.finalize(device) - ^^^^^^^ - NameError: name 'builder' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - print(result['indices'].shape) # (n_torsions, 4) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - print(result['indices'].shape) # (n_torsions, 4) - ^^^^^^ - NameError: name 'result' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - print(result['periods'].shape) # (n_torsions,) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - print(result['periods'].shape) # (n_torsions,) - ^^^^^^ - NameError: name 'result' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - builder = PlaneRestraintBuilder() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - builder = PlaneRestraintBuilder() - ^^^^^^^^^^^^^^^^^^^^^ - NameError: name 'PlaneRestraintBuilder' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - for residue in residues: - builder.process_residue(residue, cif_planes) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - for residue in residues: - ^^^^^^^^ - NameError: name 'residues' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - result = builder.finalize(device) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - result = builder.finalize(device) - ^^^^^^^ - NameError: name 'builder' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - builder = ChiralRestraintBuilder(ideal_volume=2.5, sigma=0.2) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - builder = ChiralRestraintBuilder(ideal_volume=2.5, sigma=0.2) - ^^^^^^^^^^^^^^^^^^^^^^ - NameError: name 'ChiralRestraintBuilder' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - for residue in residues: - builder.process_residue(residue, cif_chirals) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - for residue in residues: - ^^^^^^^^ - NameError: name 'residues' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - result = builder.finalize(device) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - result = builder.finalize(device) - ^^^^^^^ - NameError: name 'builder' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - print(result['indices'].shape) # (n_chirals, 4) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - print(result['indices'].shape) # (n_chirals, 4) - ^^^^^^ - NameError: name 'result' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - print(result['ideal_volumes'].shape) # (n_chirals,) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - print(result['ideal_volumes'].shape) # (n_chirals,) - ^^^^^^ - NameError: name 'result' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - builder = InterResidueBondBuilder() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - builder = InterResidueBondBuilder() - ^^^^^^^^^^^^^^^^^^^^^^^ - NameError: name 'InterResidueBondBuilder' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - for res_i, res_next in iterator.get_consecutive_pairs(): - builder.process_peptide_bond(res_i, res_next, trans_link) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - for res_i, res_next in iterator.get_consecutive_pairs(): - ^^^^^^^^ - NameError: name 'iterator' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - result = builder.finalize(device) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - result = builder.finalize(device) - ^^^^^^^ - NameError: name 'builder' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - model.load_pdb_from_file('structure.pdb') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.load_pdb_from_file('structure.pdb') - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1964, in __getattr__ - raise AttributeError( - AttributeError: 'Model' object has no attribute 'load_pdb_from_file' -********************************************************************** -File "None", line ?, in default -Failed example: - restraints = Restraints(model) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - restraints = Restraints(model) - ^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/restraints/restraints_new.py", line 86, in __init__ - self.unique_residues = model.pdb.resname.unique() - ^^^^^^^^^^^^^^^^^ - AttributeError: 'NoneType' object has no attribute 'resname' -********************************************************************** -File "None", line ?, in default -Failed example: - bond_indices = restraints.restraints['bond']['intra']['indices'] -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - bond_indices = restraints.restraints['bond']['intra']['indices'] - ^^^^^^^^^^ - NameError: name 'restraints' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - angle_refs = restraints.restraints['angle']['peptide']['references'] -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - angle_refs = restraints.restraints['angle']['peptide']['references'] - ^^^^^^^^^^ - NameError: name 'restraints' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - torsion_periods = restraints.restraints['torsion']['intra']['periods'] -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - torsion_periods = restraints.restraints['torsion']['intra']['periods'] - ^^^^^^^^^^ - NameError: name 'restraints' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - plane_4atom_indices = restraints.restraints['plane']['4_atoms']['indices'] -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - plane_4atom_indices = restraints.restraints['plane']['4_atoms']['indices'] - ^^^^^^^^^^ - NameError: name 'restraints' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - bond_indices = restraints.bond_indices # intra-residue bonds -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - bond_indices = restraints.bond_indices # intra-residue bonds - ^^^^^^^^^^ - NameError: name 'restraints' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - bond_indices_inter = restraints.bond_indices_inter # peptide bonds -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - bond_indices_inter = restraints.bond_indices_inter # peptide bonds - ^^^^^^^^^^ - NameError: name 'restraints' is not defined -********************************************************************** -1 items had failures: - 34 of 37 in default -37 tests in 1 items. -3 passed and 34 failed. -***Test Failed*** 34 failures. - -Document: api/scaling ---------------------- -********************************************************************** -File "None", line ?, in default -Failed example: - F_calc_scaled = scaler(F_calc, F_obs, s_squared) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - F_calc_scaled = scaler(F_calc, F_obs, s_squared) - ^^^^^^ - NameError: name 'F_calc' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - F_calc_total = solvent(F_calc, F_mask, s_squared) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - F_calc_total = solvent(F_calc, F_mask, s_squared) - ^^^^^^ - NameError: name 'F_calc' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - metrics = scaler.refine_lbfgs(nsteps=5, verbose=True) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - metrics = scaler.refine_lbfgs(nsteps=5, verbose=True) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/scaling/scaler.py", line 613, in refine_lbfgs - optimizer = torch.optim.LBFGS( - ^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/optim/lbfgs.py", line 243, in __init__ - super().__init__(params, defaults) - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/optim/optimizer.py", line 396, in __init__ - raise ValueError("optimizer got an empty parameter list") - ValueError: optimizer got an empty parameter list -********************************************************************** -File "None", line ?, in default -Failed example: - solvent.load_state_dict(torch.load('solvent.pt')) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - solvent.load_state_dict(torch.load('solvent.pt')) - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 1484, in load - with _open_file_like(f, "rb") as opened_file: - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 759, in _open_file_like - return _open_file(name_or_buffer, mode) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 740, in __init__ - super().__init__(open(name, mode)) - ^^^^^^^^^^^^^^^^ - FileNotFoundError: [Errno 2] No such file or directory: 'solvent.pt' -********************************************************************** -File "None", line ?, in default -Failed example: - solvent = SolventModel(model, k_solvent=0.35, b_solvent=46.0) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - solvent = SolventModel(model, k_solvent=0.35, b_solvent=46.0) - ^^^^^ - NameError: name 'model' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - metrics = scaler.refine_lbfgs(nsteps=5, verbose=True) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - metrics = scaler.refine_lbfgs(nsteps=5, verbose=True) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/scaling/scaler.py", line 613, in refine_lbfgs - optimizer = torch.optim.LBFGS( - ^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/optim/lbfgs.py", line 243, in __init__ - super().__init__(params, defaults) - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/optim/optimizer.py", line 396, in __init__ - raise ValueError("optimizer got an empty parameter list") - ValueError: optimizer got an empty parameter list -********************************************************************** -File "None", line ?, in default -Failed example: - solvent.load_state_dict(torch.load('solvent.pt')) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - solvent.load_state_dict(torch.load('solvent.pt')) - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 1484, in load - with _open_file_like(f, "rb") as opened_file: - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 759, in _open_file_like - return _open_file(name_or_buffer, mode) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 740, in __init__ - super().__init__(open(name, mode)) - ^^^^^^^^^^^^^^^^ - FileNotFoundError: [Errno 2] No such file or directory: 'solvent.pt' -********************************************************************** -File "None", line ?, in default -Failed example: - solvent = SolventModel(model, k_solvent=0.35, b_solvent=46.0) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - solvent = SolventModel(model, k_solvent=0.35, b_solvent=46.0) - ^^^^^ - NameError: name 'model' is not defined -********************************************************************** -1 items had failures: - 8 of 17 in default -17 tests in 1 items. -9 passed and 8 failed. -***Test Failed*** 8 failures. - -Document: api/utils -------------------- -********************************************************************** -File "None", line ?, in default -Failed example: - masks['backbone'] = backbone_mask -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - masks['backbone'] = backbone_mask - ^^^^^^^^^^^^^ - NameError: name 'backbone_mask' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - grad_norm = gradnorm(loss, model.parameters()) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - grad_norm = gradnorm(loss, model.parameters()) - ^^^^ - NameError: name 'loss' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - masks['rfree'] = rfree_flags > 0 -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - masks['rfree'] = rfree_flags > 0 - ^^^^^^^^^^^ - NameError: name 'rfree_flags' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - masks.cpu() # Move all to CPU -Expected nothing -Got: - TensorMasks({'valid': shape=torch.Size([100])}, device=cpu) -********************************************************************** -File "None", line ?, in default -Failed example: - model = MyModel() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model = MyModel() - ^^^^^^^ - NameError: name 'MyModel' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - scaler = Scaler() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - scaler = Scaler() - ^^^^^^ - NameError: name 'Scaler' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - scaler._model = ModuleReference(model) # Won't register as submodule -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - scaler._model = ModuleReference(model) # Won't register as submodule - ^^^^^^^^^^^^^^^ - NameError: name 'ModuleReference' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - output = scaler._model.module(input_data) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - output = scaler._model.module(input_data) - ^^^^^^ - NameError: name 'scaler' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - model.load_cif('structure.cif') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.load_cif('structure.cif') - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 252, in load_cif - cif_reader = cif.ModelCIFReader(file) - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/cif_readers.py", line 1263, in __init__ - self.cif = CIFReader(filepath) - ^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/cif_readers.py", line 80, in __init__ - self.load(filepath) - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/cif_readers.py", line 92, in load - with open(filepath, "r", encoding="utf-8", errors="ignore") as f: - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - FileNotFoundError: [Errno 2] No such file or directory: 'structure.cif' -********************************************************************** -File "None", line ?, in default -Failed example: - model.pdb = sanitize_pdb_dataframe(model.pdb, verbose=1) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.pdb = sanitize_pdb_dataframe(model.pdb, verbose=1) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/utils/utils.py", line 797, in sanitize_pdb_dataframe - pdb = pdb.copy() - ^^^^^^^^ - AttributeError: 'NoneType' object has no attribute 'copy' -********************************************************************** -File "None", line ?, in default -Failed example: - mask = parse_phenix_selection("chain A", pdb_df) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - mask = parse_phenix_selection("chain A", pdb_df) - ^^^^^^^^^^^^^^^^^^^^^^ - NameError: name 'parse_phenix_selection' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - mask = parse_phenix_selection("chain A and resseq 10:20", pdb_df) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - mask = parse_phenix_selection("chain A and resseq 10:20", pdb_df) - ^^^^^^^^^^^^^^^^^^^^^^ - NameError: name 'parse_phenix_selection' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - mask = parse_phenix_selection("name CA", pdb_df) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - mask = parse_phenix_selection("name CA", pdb_df) - ^^^^^^^^^^^^^^^^^^^^^^ - NameError: name 'parse_phenix_selection' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - mask = parse_phenix_selection("name CA or name C or name N or name O", pdb_df) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - mask = parse_phenix_selection("name CA or name C or name N or name O", pdb_df) - ^^^^^^^^^^^^^^^^^^^^^^ - NameError: name 'parse_phenix_selection' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - mask = parse_phenix_selection("not resname HOH", pdb_df) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - mask = parse_phenix_selection("not resname HOH", pdb_df) - ^^^^^^^^^^^^^^^^^^^^^^ - NameError: name 'parse_phenix_selection' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - mask = parse_phenix_selection("chain A and (name CA or name CB)", pdb_df) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - mask = parse_phenix_selection("chain A and (name CA or name CB)", pdb_df) - ^^^^^^^^^^^^^^^^^^^^^^ - NameError: name 'parse_phenix_selection' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - mask = create_selection_mask("chain A", pdb_df, mode='set') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - mask = create_selection_mask("chain A", pdb_df, mode='set') - ^^^^^^^^^^^^^^^^^^^^^ - NameError: name 'create_selection_mask' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - mask = create_selection_mask("resseq 10:20", pdb_df, current_mask=mask, mode='add') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - mask = create_selection_mask("resseq 10:20", pdb_df, current_mask=mask, mode='add') - ^^^^^^^^^^^^^^^^^^^^^ - NameError: name 'create_selection_mask' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - mask = create_selection_mask("resname HOH", pdb_df, current_mask=mask, mode='remove') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - mask = create_selection_mask("resname HOH", pdb_df, current_mask=mask, mode='remove') - ^^^^^^^^^^^^^^^^^^^^^ - NameError: name 'create_selection_mask' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - class MyTarget(HyperparameterMixin, nn.Module): - def __init__(self, sigma=1.0, target_value=0.0): - nn.Module.__init__(self) - HyperparameterMixin.__init__(self) - self.register_hyperparameter('sigma', sigma) - self.register_hyperparameter('target_value', target_value) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - class MyTarget(HyperparameterMixin, nn.Module): - ^^^^^^^^^^^^^^^^^^^ - NameError: name 'HyperparameterMixin' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - target = MyTarget(sigma=2.5) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - target = MyTarget(sigma=2.5) - ^^^^^^^^ - NameError: name 'MyTarget' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - dict(target.hyperparameters()) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - dict(target.hyperparameters()) - ^^^^^^ - NameError: name 'target' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - params = module.hyperparameter_dict() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - params = module.hyperparameter_dict() - ^^^^^^ - NameError: name 'module' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - json.dumps(params) # JSON serializable -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - json.dumps(params) # JSON serializable - ^^^^^^ - NameError: name 'params' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - hp_state = module.hyperparameter_state_dict() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - hp_state = module.hyperparameter_state_dict() - ^^^^^^ - NameError: name 'module' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - torch.save(hp_state, 'hyperparameters.pt') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - torch.save(hp_state, 'hyperparameters.pt') - ^^^^^^^^ - NameError: name 'hp_state' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - for name, hp in module.hyperparameters(): - print(f"{name}: {hp.item()}") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - for name, hp in module.hyperparameters(): - ^^^^^^ - NameError: name 'module' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - hp_state = torch.load('hyperparameters.pt') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - hp_state = torch.load('hyperparameters.pt') - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 1484, in load - with _open_file_like(f, "rb") as opened_file: - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 759, in _open_file_like - return _open_file(name_or_buffer, mode) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/serialization.py", line 740, in __init__ - super().__init__(open(name, mode)) - ^^^^^^^^^^^^^^^^ - FileNotFoundError: [Errno 2] No such file or directory: 'hyperparameters.pt' -********************************************************************** -File "None", line ?, in default -Failed example: - module.load_hyperparameter_state_dict(hp_state) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - module.load_hyperparameter_state_dict(hp_state) - ^^^^^^ - NameError: name 'module' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - self.register_hyperparameter('sigma', 2.0) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - self.register_hyperparameter('sigma', 2.0) - ^^^^ - NameError: name 'self' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - self.sigma # Access via property (if defined) or _hp_sigma -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - self.sigma # Access via property (if defined) or _hp_sigma - ^^^^ - NameError: name 'self' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - loss = model(input) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - loss = model(input) - ^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl - return self._call_impl(*args, **kwargs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl - return forward_call(*args, **kwargs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/nn/modules/module.py", line 399, in _forward_unimplemented - raise NotImplementedError( - NotImplementedError: Module [Model] is missing the required "forward" function -********************************************************************** -File "None", line ?, in default -Failed example: - grad_norm = gradnorm(loss, model.parameters()) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - grad_norm = gradnorm(loss, model.parameters()) - ^^^^ - NameError: name 'loss' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - print(f"Gradient norm: {grad_norm:.4f}") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - print(f"Gradient norm: {grad_norm:.4f}") - ^^^^^^^^^ - NameError: name 'grad_norm' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - model = MyModel() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model = MyModel() - ^^^^^^^ - NameError: name 'MyModel' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - scaler = Scaler() -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - scaler = Scaler() - ^^^^^^ - NameError: name 'Scaler' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - scaler._model = ModuleReference(model) # Won't register as submodule -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - scaler._model = ModuleReference(model) # Won't register as submodule - ^^^^^^^^^^^^^^^ - NameError: name 'ModuleReference' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - output = scaler._model.module(input_data) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - output = scaler._model.module(input_data) - ^^^^^^ - NameError: name 'scaler' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - masks['rfree'] = rfree_flags > 0 -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - masks['rfree'] = rfree_flags > 0 - ^^^^^^^^^^^ - NameError: name 'rfree_flags' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - masks.cpu() # Move all to CPU -Expected nothing -Got: - TensorMasks({'valid': shape=torch.Size([100])}, device=cpu) -********************************************************************** -File "None", line ?, in default -Failed example: - model.load_cif('structure.cif') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.load_cif('structure.cif') - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/model/model.py", line 252, in load_cif - cif_reader = cif.ModelCIFReader(file) - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/cif_readers.py", line 1263, in __init__ - self.cif = CIFReader(filepath) - ^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/cif_readers.py", line 80, in __init__ - self.load(filepath) - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/cif_readers.py", line 92, in load - with open(filepath, "r", encoding="utf-8", errors="ignore") as f: - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - FileNotFoundError: [Errno 2] No such file or directory: 'structure.cif' -********************************************************************** -File "None", line ?, in default -Failed example: - model.pdb = sanitize_pdb_dataframe(model.pdb, verbose=1) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - model.pdb = sanitize_pdb_dataframe(model.pdb, verbose=1) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/utils/utils.py", line 797, in sanitize_pdb_dataframe - pdb = pdb.copy() - ^^^^^^^^ - AttributeError: 'NoneType' object has no attribute 'copy' -********************************************************************** -File "None", line ?, in default -Failed example: - mask = parse_phenix_selection("chain A", pdb_df) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - mask = parse_phenix_selection("chain A", pdb_df) - ^^^^^^^^^^^^^^^^^^^^^^ - NameError: name 'parse_phenix_selection' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - mask = parse_phenix_selection("chain A and resseq 10:20", pdb_df) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - mask = parse_phenix_selection("chain A and resseq 10:20", pdb_df) - ^^^^^^^^^^^^^^^^^^^^^^ - NameError: name 'parse_phenix_selection' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - mask = parse_phenix_selection("name CA", pdb_df) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - mask = parse_phenix_selection("name CA", pdb_df) - ^^^^^^^^^^^^^^^^^^^^^^ - NameError: name 'parse_phenix_selection' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - mask = parse_phenix_selection("name CA or name C or name N or name O", pdb_df) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - mask = parse_phenix_selection("name CA or name C or name N or name O", pdb_df) - ^^^^^^^^^^^^^^^^^^^^^^ - NameError: name 'parse_phenix_selection' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - mask = parse_phenix_selection("not resname HOH", pdb_df) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - mask = parse_phenix_selection("not resname HOH", pdb_df) - ^^^^^^^^^^^^^^^^^^^^^^ - NameError: name 'parse_phenix_selection' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - mask = parse_phenix_selection("chain A and (name CA or name CB)", pdb_df) -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - mask = parse_phenix_selection("chain A and (name CA or name CB)", pdb_df) - ^^^^^^^^^^^^^^^^^^^^^^ - NameError: name 'parse_phenix_selection' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - mask = create_selection_mask("chain A", pdb_df, mode='set') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - mask = create_selection_mask("chain A", pdb_df, mode='set') - ^^^^^^^^^^^^^^^^^^^^^ - NameError: name 'create_selection_mask' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - mask = create_selection_mask("resseq 10:20", pdb_df, current_mask=mask, mode='add') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - mask = create_selection_mask("resseq 10:20", pdb_df, current_mask=mask, mode='add') - ^^^^^^^^^^^^^^^^^^^^^ - NameError: name 'create_selection_mask' is not defined -********************************************************************** -File "None", line ?, in default -Failed example: - mask = create_selection_mask("resname HOH", pdb_df, current_mask=mask, mode='remove') -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 1, in - mask = create_selection_mask("resname HOH", pdb_df, current_mask=mask, mode='remove') - ^^^^^^^^^^^^^^^^^^^^^ - NameError: name 'create_selection_mask' is not defined -********************************************************************** -1 items had failures: - 51 of 67 in default -67 tests in 1 items. -16 passed and 51 failed. -***Test Failed*** 51 failures. - -Document: quickstart --------------------- -********************************************************************** -File "quickstart.rst", line 117, in default -Failed example: - from torchref import ModelFT, ROOT_TORCHREF - - model = ModelFT() - model.load_pdb(f"{ROOT_TORCHREF}/example_notebooks/1DAW.pdb") - - print(f"Number of atoms: {model.n_atoms}") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 6, in - print(f"Number of atoms: {model.n_atoms}") - ^^^^^^^^^^^^^ - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1964, in __getattr__ - raise AttributeError( - AttributeError: 'ModelFT' object has no attribute 'n_atoms' -********************************************************************** -File "quickstart.rst", line 299, in default -Failed example: - from torchref import LBFGSRefinement, ROOT_TORCHREF - - refinement = LBFGSRefinement( - data_file=f"{ROOT_TORCHREF}/example_notebooks/1DAW.mtz", - pdb=f"{ROOT_TORCHREF}/example_notebooks/1DAW.pdb", - ) - - # Perturb coordinates to simulate errors - refinement.model.shake_coords(0.1) - - # Run xyz refinement (3 cycles for quick test) - refinement.refine_xyz(n_cycles=1) - - rwork, rfree = refinement.get_rfactor() - print(f"After refinement - Rwork: {rwork:.3f}") -Exception raised: - Traceback (most recent call last): - File "/das/work/p17/p17490/CONDA/torchref/lib/python3.11/doctest.py", line 1355, in __run - exec(compile(example.source, filename, "single", - File "", line 12, in - refinement.refine_xyz(n_cycles=1) - TypeError: LBFGSRefinement.refine_xyz() got an unexpected keyword argument 'n_cycles' diff --git a/paper/extended_figures/exF2/collect_exF2_data.py b/paper/extended_figures/exF2/collect_exF2_data.py index 68adad4..6fda5d0 100644 --- a/paper/extended_figures/exF2/collect_exF2_data.py +++ b/paper/extended_figures/exF2/collect_exF2_data.py @@ -27,9 +27,9 @@ BASE = Path(__file__).resolve().parent AF_ROOT = BASE.parent.parent / "figure2_alphafold_start" CROSSSCORE = AF_ROOT / "runs" / "metrics" / "fig_crossscore.csv" -TORCHREF_ARM = AF_ROOT / "runs" / "torchref_g0p2_a0p02" +TORCHREF_ARM = AF_ROOT / "runs" / "torchref" SCORER = "phenix" # common independent scorer (main-figure scorer) -TORCHREF_ENGINE = "torchref_g0p2_a0p02" +TORCHREF_ENGINE = "torchref" PHENIX_ENGINE = "phenix" REFMAC_ENGINE = "refmac" OUT_CSV = BASE / "data" / "exF2_rfactor_by_resolution.csv" diff --git a/paper/extended_figures/exF3/plot_exF3.py b/paper/extended_figures/exF3/plot_exF3.py index cff1605..0e471b3 100644 --- a/paper/extended_figures/exF3/plot_exF3.py +++ b/paper/extended_figures/exF3/plot_exF3.py @@ -15,7 +15,7 @@ scorers' systematic differences (PHENIX lowest, then REFMAC, then TorchRef — the most conservative); these cancel within a scorer. PHENIX is the main-figure scorer. -TorchRef here is the locked-default arm (torchref_g0p2_a0p02; xray 1 / geometry 0.2 +TorchRef here is the canonical arm (torchref; xray 1 / geometry 0.2 / adp 0.02). Reads figure2_alphafold_start/runs/metrics/fig_crossscore.csv (analysis/aggregate_crossscore.py). @@ -42,7 +42,7 @@ # refined-model engines (exclude the AlphaFold prediction to keep axes focused). # TorchRef = the locked-default arm (xray 1 / geometry 0.2 / adp 0.02). -TORCHREF_ENGINE = "torchref_g0p2_a0p02" +TORCHREF_ENGINE = "torchref" ENGINES = [ ("refmac", "Refmac", "#762a83"), ("phenix", "PHENIX", "#2166ac"), diff --git a/paper/figure2_alphafold_start/analysis/aggregate_weight_grid.py b/paper/figure2_alphafold_start/analysis/aggregate_weight_grid.py index b9a0703..63bfca7 100644 --- a/paper/figure2_alphafold_start/analysis/aggregate_weight_grid.py +++ b/paper/figure2_alphafold_start/analysis/aggregate_weight_grid.py @@ -9,6 +9,7 @@ Usage: ./.dev/bin/python analysis/aggregate_weight_grid.py """ +import argparse import csv import sys from pathlib import Path @@ -17,15 +18,26 @@ import run_af_pipeline as P # noqa: E402 from aggregate_figure_metrics import RE_RWORK, RE_RFREE, parse_geometry # noqa: E402 -MANIFEST = P.RUNS / "metrics" / "wgrid_manifest.csv" -OUT = P.RUNS / "metrics" / "weight_grid.csv" - def ratio(num, den): return (num / den) if (num is not None and den) else None def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--tag", default="", + help="Ablation-grid namespace (matches submit_weight_grid " + "--tag), e.g. 'nosimu'. Reads wgrid__manifest.csv " + "and writes weight_grid_.csv.") + ap.add_argument("--out-root", default=None, + help="Base directory the grid was written to (matches " + "submit_weight_grid --out-root). Default: figure2 runs/.") + args = ap.parse_args() + out_root = Path(args.out_root).resolve() if args.out_root else P.RUNS + suffix = f"_{args.tag}" if args.tag else "" + MANIFEST = out_root / "metrics" / f"wgrid{suffix}_manifest.csv" + OUT = out_root / "metrics" / f"weight_grid{suffix}.csv" + if not MANIFEST.exists(): sys.exit(f"missing {MANIFEST}; run submit_weight_grid.py first") manifest = list(csv.DictReader(open(MANIFEST))) @@ -33,7 +45,7 @@ def main(): rows = [] for m in manifest: arm = m["arm"] - arm_dir = P.RUNS / arm + arm_dir = out_root / arm if not arm_dir.is_dir(): continue for code_dir in sorted(p for p in arm_dir.iterdir() if p.is_dir()): diff --git a/paper/figure2_alphafold_start/analysis/build_torchref_score_worklist.py b/paper/figure2_alphafold_start/analysis/build_torchref_score_worklist.py index 2a9ad9c..4852ffd 100644 --- a/paper/figure2_alphafold_start/analysis/build_torchref_score_worklist.py +++ b/paper/figure2_alphafold_start/analysis/build_torchref_score_worklist.py @@ -28,7 +28,7 @@ MODELS = { "refmac": ("refmac", "refmac/{code}/refined.pdb"), "phenix": ("phenix_norb", "phenix_norb/{code}/{code}_refined_001.pdb"), - "torchref": ("torchref_g0p2_a0p02", "torchref_g0p2_a0p02/{code}/refined.pdb"), + "torchref": ("torchref", "torchref/{code}/refined.pdb"), "prediction": ("af_initial", "../placed/{code}_af.pdb"), } diff --git a/paper/figure2_alphafold_start/analysis/compare_ramachandran.py b/paper/figure2_alphafold_start/analysis/compare_ramachandran.py deleted file mode 100644 index 0a580d9..0000000 --- a/paper/figure2_alphafold_start/analysis/compare_ramachandran.py +++ /dev/null @@ -1,201 +0,0 @@ -#!/usr/bin/env python3 -"""Compare the TorchRef AF-start arm WITH vs WITHOUT the Ramachandran restraint. - -Both arms are identical (-n 10 --mode separate --xray-mode ml, group weights -xray 1 / geometry 0.2 / adp 0.02); the only difference is that the *_norama arm -adds ``--weights '{"geometry/ramachandran": 0}'`` so the Ramachandran component -is skipped (its effective weight 0.2 * 0 = 0). - -Each structure's R-factors and geometry come from the co-located REFMAC 0-cycle -``validate.log`` (apples-to-apples; same parser as aggregate_figure_metrics.py), -so the only thing that moved is what TorchRef refined, not how it was scored. - -Pairs by PDB code, reports medians over the paired (both-arms-present) set and a -paired Δ (norama − rama), and writes a CSV + markdown table. - -Usage ------ - ./.dev/bin/python paper/figure2_alphafold_start/analysis/compare_ramachandran.py - ./.dev/bin/python paper/figure2_alphafold_start/analysis/compare_ramachandran.py \ - --with-arm torchref_g0p2_a0p02 --without-arm torchref_g0p2_a0p02_norama -""" -from __future__ import annotations - -import argparse -import csv -import re -import sys -from pathlib import Path - -import numpy as np - -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) -import run_af_pipeline as P # noqa: E402 - -# Same regexes as aggregate_figure_metrics.py (one source of truth for parsing). -RE_RWORK = re.compile(r"Overall R factor\s+=\s+([\d.]+)") -RE_RFREE = re.compile(r"Free R factor\s+=\s+([\d.]+)") -RE_TAIL3 = re.compile(r"(\d+)\s+([\d.]+)\s+([\d.]+)\s*$") -GEOM_ROWS = { - "Bond distances": ("rmsBOND", "sigBOND"), - "Bond angles": ("rmsANGL", "sigANGL"), - "Chiral centres": ("rmsCHIRAL", "sigCHIRAL"), - "M. chain bond B values": ("rmsB_mc_bond", "sigB_mc_bond"), -} - - -def parse_validate(path: Path): - """R-work/R-free + geometry RMS-delta/Av(sigma) from a REFMAC validate.log.""" - if not path.exists(): - return None - try: - text = path.read_text(errors="replace") - except OSError: - return None - mw, mf = RE_RWORK.search(text), RE_RFREE.search(text) - if not (mw and mf): - return None - rec = {"r_work": float(mw.group(1)), "r_free": float(mf.group(1))} - for line in text.splitlines(): - for label, (rms_col, sig_col) in GEOM_ROWS.items(): - if line.startswith(label): - m = RE_TAIL3.search(line) - if m: - rec[rms_col] = float(m.group(2)) - rec[sig_col] = float(m.group(3)) - return rec - - -def collect(arm, codes): - out = {} - for code in codes: - rec = parse_validate(P.RUNS / arm / code / "validate.log") - if rec is not None: - out[code] = rec - return out - - -def med(vals): - vals = [v for v in vals if v is not None and not np.isnan(v)] - return (float(np.median(vals)), len(vals)) if vals else (float("nan"), 0) - - -def rmsz(rec, rms_col, sig_col): - rms, sig = rec.get(rms_col), rec.get(sig_col) - if rms is None or sig is None or sig <= 0: - return None - return rms / sig - - -def main(): - ap = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter - ) - ap.add_argument("--with-arm", default="torchref_g0p2_a0p02", - help="Arm WITH Ramachandran (the locked default).") - ap.add_argument("--without-arm", default="torchref_g0p2_a0p02_norama", - help="Arm WITHOUT Ramachandran.") - ap.add_argument("--out-csv", default=None, - help="Per-structure CSV (default runs/metrics/ramachandran_compare.csv).") - ap.add_argument("--out-md", default=None, - help="Markdown summary (default runs/metrics/ramachandran_compare.md).") - args = ap.parse_args() - - codes = P.load_solved_codes() - rama = collect(args.with_arm, codes) - nora = collect(args.without_arm, codes) - paired = sorted(set(rama) & set(nora)) - - print(f"WITH-rama arm '{args.with_arm}': {len(rama)} validated") - print(f"WITHOUT-rama '{args.without_arm}': {len(nora)} validated") - print(f"paired (both present): {len(paired)}\n") - if not paired: - print("No paired structures yet — let the *_norama jobs finish, then re-run.") - return - - metrics_dir = P.RUNS / "metrics" - metrics_dir.mkdir(parents=True, exist_ok=True) - out_csv = Path(args.out_csv) if args.out_csv else metrics_dir / "ramachandran_compare.csv" - out_md = Path(args.out_md) if args.out_md else metrics_dir / "ramachandran_compare.md" - - # ── per-structure CSV ──────────────────────────────────────────────────── - fields = ["code", - "rwork_rama", "rfree_rama", "rwork_norama", "rfree_norama", - "d_rwork", "d_rfree", - "rmsz_bond_rama", "rmsz_bond_norama", - "rmsz_angle_rama", "rmsz_angle_norama"] - rows = [] - for code in paired: - a, b = rama[code], nora[code] - rows.append({ - "code": code, - "rwork_rama": a["r_work"], "rfree_rama": a["r_free"], - "rwork_norama": b["r_work"], "rfree_norama": b["r_free"], - "d_rwork": b["r_work"] - a["r_work"], - "d_rfree": b["r_free"] - a["r_free"], - "rmsz_bond_rama": rmsz(a, "rmsBOND", "sigBOND"), - "rmsz_bond_norama": rmsz(b, "rmsBOND", "sigBOND"), - "rmsz_angle_rama": rmsz(a, "rmsANGL", "sigANGL"), - "rmsz_angle_norama": rmsz(b, "rmsANGL", "sigANGL"), - }) - with open(out_csv, "w", newline="") as f: - w = csv.DictWriter(f, fieldnames=fields) - w.writeheader() - w.writerows(rows) - - # ── medians over the paired set ────────────────────────────────────────── - def pm(key): - return med([r[key] for r in rows])[0] - - rw_a, rw_b = pm("rwork_rama"), pm("rwork_norama") - rf_a, rf_b = pm("rfree_rama"), pm("rfree_norama") - gap_a = pm("rfree_rama") - pm("rwork_rama") - gap_b = pm("rfree_norama") - pm("rwork_norama") - d_rw = pm("d_rwork") - d_rf = pm("d_rfree") - n_rf_worse = sum(1 for r in rows if r["d_rfree"] > 0) - - bond_a, bond_b = pm("rmsz_bond_rama"), pm("rmsz_bond_norama") - ang_a, ang_b = pm("rmsz_angle_rama"), pm("rmsz_angle_norama") - - md = [ - "# TorchRef AF-start: Ramachandran restraint on vs off", - "", - f"Paired over **{len(paired)}** structures (both arms REFMAC-0-cycle " - "validated). Both arms: `-n 10 --mode separate --xray-mode ml`, group " - "weights xray 1 / geometry 0.2 / adp 0.02; the only difference is " - "`geometry/ramachandran` weight (default vs 0).", - "", - "## Median R-factors (paired set)", - "", - "| Arm | median R-work | median R-free | R-free − R-work gap |", - "|---|---|---|---|", - f"| with Ramachandran (`{args.with_arm}`) | {rw_a:.4f} | {rf_a:.4f} | {gap_a:.4f} |", - f"| without Ramachandran (`{args.without_arm}`) | {rw_b:.4f} | {rf_b:.4f} | {gap_b:.4f} |", - "", - "## Paired Δ (without − with)", - "", - f"- median ΔR-work: **{d_rw:+.4f}**", - f"- median ΔR-free: **{d_rf:+.4f}** (>0 ⇒ removing Ramachandran hurts R-free)", - f"- structures where R-free got worse without Ramachandran: " - f"**{n_rf_worse}/{len(rows)}** ({100*n_rf_worse/len(rows):.0f}%)", - "", - "## Geometry quality (median RMSZ; ideal ≈ 1.0)", - "", - "| Restraint | with Ramachandran | without Ramachandran |", - "|---|---|---|", - f"| Bond RMSZ | {bond_a:.2f} | {bond_b:.2f} |", - f"| Angle RMSZ | {ang_a:.2f} | {ang_b:.2f} |", - "", - f"Per-structure data: `{out_csv}`", - "", - ] - out_md.write_text("\n".join(md)) - - print("\n".join(md)) - print(f"\nwrote {out_csv}") - print(f"wrote {out_md}") - - -if __name__ == "__main__": - main() diff --git a/paper/figure2_alphafold_start/analysis/submit_weight_grid.py b/paper/figure2_alphafold_start/analysis/submit_weight_grid.py index ac6fc4f..6c3b4fe 100644 --- a/paper/figure2_alphafold_start/analysis/submit_weight_grid.py +++ b/paper/figure2_alphafold_start/analysis/submit_weight_grid.py @@ -72,6 +72,54 @@ def build_script(arm, code, pdb, mtz, outdir, n_cycles, mem, weights): """ +def build_array_script(name, tasks_file, n_tasks, logdir, n_cycles, mem, throttle): + """One SLURM job array: task i reads line i of the (tab-separated) tasks file + `outdirpdbmtzweights_json` and runs refine + REFMAC validation.""" + return f"""#!/bin/bash +#SBATCH --job-name={name} +#SBATCH --partition=hour +#SBATCH --cpus-per-task=4 +#SBATCH --time=01:00:00 +#SBATCH --mem={mem} +#SBATCH --array=0-{n_tasks - 1}%{throttle} +#SBATCH --output={logdir}/task_%a.out +#SBATCH --error={logdir}/task_%a.err + +set -e +export TORCHREF_NUM_THREADS=4 + +LINE=$(sed -n "$((SLURM_ARRAY_TASK_ID + 1))p" "{tasks_file}") +OUTDIR=$(printf '%s' "$LINE" | cut -f1) +PDB=$(printf '%s' "$LINE" | cut -f2) +MTZ=$(printf '%s' "$LINE" | cut -f3) +WEIGHTS=$(printf '%s' "$LINE" | cut -f4) +mkdir -p "$OUTDIR" + +{P.PYTHON} -u {P.REFINE_SCRIPT} \\ + -m "$PDB" -sf "$MTZ" -o "$OUTDIR" \\ + -n {n_cycles} \\ + --mode separate \\ + --xray-mode ml \\ + --sigma-m-scale 1.0 \\ + --weights "$WEIGHTS" + +# REFMAC 0-cycle validation (validate.log -> R-factors + geometry RMS/sigma) +source {P.CCP4_SETUP} +TEMP_DIR=/tmp/validate_{name}_${{SLURM_ARRAY_JOB_ID}}_${{SLURM_ARRAY_TASK_ID}} +mkdir -p $TEMP_DIR && cd $TEMP_DIR +export CCP4_SCR=$TEMP_DIR +cp "$OUTDIR/refined.pdb" input.pdb +cp "$MTZ" input.mtz +refmac5 HKLIN input.mtz HKLOUT output.mtz XYZIN input.pdb XYZOUT output.pdb \\ + > "$OUTDIR/validate.log" 2>&1 << EOF +NCYCLES 0 +MAKE HYDR NO +END +EOF +cd / && rm -rf $TEMP_DIR +""" + + def main(): ap = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter @@ -85,10 +133,33 @@ def main(): ap.add_argument("--codes", nargs="+", default=None) ap.add_argument("--n-cycles", type=int, default=10) ap.add_argument("--mem", default="8G") + ap.add_argument("--tag", default="", + help="Namespace suffix for an ablation grid, e.g. 'nosimu'. " + "Arms become wgrid__g{gi}_a{ai} with their own " + "manifest so they don't collide with the baseline grid.") + ap.add_argument("--disable", nargs="+", default=[], + help="Restraint component weights to force to 0 for this " + "grid, e.g. --disable adp/simu adp/KL.") + ap.add_argument("--out-root", default=None, + help="Base directory for this grid's arm output + manifest " + "(default: figure2 runs/). Use a dedicated dir for " + "ablation grids to keep runs/ uncluttered.") + ap.add_argument("--array", action="store_true", + help="Submit the whole grid as ONE SLURM job array instead " + "of one job per (cell, structure). Much lighter on the " + "scheduler.") + ap.add_argument("--array-throttle", type=int, default=200, + help="Max concurrently-running array tasks (sbatch %%N). " + "At 4 cpus/task, 200 ~ 800 cpus (per-user cap ~1167).") ap.add_argument("--dry-run", action="store_true") ap.add_argument("--force", action="store_true") args = ap.parse_args() + out_root = Path(args.out_root).resolve() if args.out_root else P.RUNS + suffix = f"_{args.tag}" if args.tag else "" + manifest_path = out_root / "metrics" / f"wgrid{suffix}_manifest.csv" + disabled = {k: 0 for k in args.disable} + geoms = np.logspace(np.log10(args.geom_range[0]), np.log10(args.geom_range[1]), args.n_geom) adps = np.logspace(np.log10(args.adp_range[0]), np.log10(args.adp_range[1]), @@ -99,29 +170,34 @@ def main(): codes = codes[:args.limit] # Manifest: arm -> (gi, ai, geometry, adp), exact weights for aggregation. - MANIFEST.parent.mkdir(parents=True, exist_ok=True) - with open(MANIFEST, "w", newline="") as f: + manifest_path.parent.mkdir(parents=True, exist_ok=True) + with open(manifest_path, "w", newline="") as f: w = csv.writer(f) w.writerow(["arm", "gi", "ai", "geometry", "adp"]) for gi, g in enumerate(geoms): for ai, a in enumerate(adps): - w.writerow([f"wgrid_g{gi}_a{ai}", gi, ai, f"{g:.6g}", f"{a:.6g}"]) + w.writerow([f"wgrid{suffix}_g{gi}_a{ai}", gi, ai, + f"{g:.6g}", f"{a:.6g}"]) - print(f"Log weight grid (xray fixed = 1):") + print(f"Log weight grid (xray fixed = 1) tag={args.tag or '(baseline)'}:") print(f" geometry [{args.n_geom}]: {', '.join(f'{g:.4g}' for g in geoms)}") print(f" adp [{args.n_adp}]: {', '.join(f'{a:.4g}' for a in adps)}") + if disabled: + print(f" DISABLED components (weight=0): {', '.join(disabled)}") print(f" {args.n_geom * args.n_adp} combos x {len(codes)} structures = " f"{args.n_geom * args.n_adp * len(codes)} jobs max") - print(f" manifest: {MANIFEST}\n") + print(f" manifest: {manifest_path}\n") - first, total_sub = True, 0 + # Collect the (cell x structure) work list, skipping already-complete cells. + tasks = [] # each: (outdir, pdb, mtz, weights_json) + skip = miss = 0 for gi, g in enumerate(geoms): for ai, a in enumerate(adps): - arm = f"wgrid_g{gi}_a{ai}" - arm_dir = P.RUNS / arm - tmp = arm_dir / "tmp_scripts" - weights = {"xray": 1, "geometry": float(g), "adp": float(a)} - sub = skip = miss = 0 + arm = f"wgrid{suffix}_g{gi}_a{ai}" + arm_dir = out_root / arm + weights = {"xray": 1, "geometry": float(g), "adp": float(a), + **disabled} + wjson = json.dumps(weights) for code in codes: pdb, mtz = P.PLACED / f"{code}_af.pdb", P._mtz(code) if not pdb.exists() or not mtz.exists(): @@ -131,19 +207,56 @@ def main(): if P._refmac_complete(outdir / "validate.log") and not args.force: skip += 1 continue - if not args.dry_run: - outdir.mkdir(parents=True, exist_ok=True) - script = build_script(arm, code, pdb, mtz, outdir, args.n_cycles, - args.mem, weights) - if first and args.dry_run: - print("── example sbatch script ──") - print(script) - print("───────────────────────────") - first = False - if P._sbatch(script, tmp / f"ref_{code}.sh", args.dry_run) or args.dry_run: - sub += 1 - total_sub += sub - print(f"TOTAL submitted={total_sub} (over {args.n_geom * args.n_adp} arms)") + tasks.append((str(outdir), str(pdb), str(mtz), wjson)) + print(f"work list: {len(tasks)} tasks (skip={skip} complete, miss={miss})") + + if not tasks: + print("nothing to do.") + return + + if args.array: + # One job array for the whole grid: task i <- line i of the tasks file. + tmp = out_root / "tmp_scripts" + logdir = tmp / f"logs{suffix}" + tasks_file = tmp / f"tasks{suffix}.tsv" + array_script = tmp / f"array{suffix}.sh" + name = f"af_wgrid{suffix or '_base'}" + script = build_array_script(name, tasks_file, len(tasks), logdir, + args.n_cycles, args.mem, args.array_throttle) + if args.dry_run: + print(f"\n[DRY-RUN] would write {len(tasks)} tasks -> {tasks_file}") + print(f"[DRY-RUN] would submit array 0-{len(tasks) - 1}%" + f"{args.array_throttle}\n── array script ──\n{script}") + return + tmp.mkdir(parents=True, exist_ok=True) + logdir.mkdir(parents=True, exist_ok=True) + with open(tasks_file, "w") as f: + for outdir, pdb, mtz, wjson in tasks: + f.write(f"{outdir}\t{pdb}\t{mtz}\t{wjson}\n") + array_script.write_text(script) + jid = P._sbatch(script, array_script, dry_run=False) + print(f"submitted array job {jid}: {len(tasks)} tasks " + f"(throttle %{args.array_throttle})") + return + + # Per-job fallback (one sbatch per task). + first = 0 + for outdir, pdb, mtz, wjson in tasks: + outdir_p = Path(outdir) + arm = outdir_p.parent.name + code = outdir_p.name + if not args.dry_run: + outdir_p.mkdir(parents=True, exist_ok=True) + script = build_script(arm, code, pdb, mtz, outdir, args.n_cycles, + args.mem, json.loads(wjson)) + if args.dry_run and first == 0: + print("── example sbatch script ──") + print(script) + print("───────────────────────────") + first += 1 + P._sbatch(script, outdir_p.parent / "tmp_scripts" / f"ref_{code}.sh", + args.dry_run) + print(f"TOTAL submitted={len(tasks)} (over {args.n_geom * args.n_adp} arms)") if __name__ == "__main__": diff --git a/paper/figure2_alphafold_start/full_mvd.log b/paper/figure2_alphafold_start/full_mvd.log deleted file mode 100644 index d5c4dbd..0000000 --- a/paper/figure2_alphafold_start/full_mvd.log +++ /dev/null @@ -1,44 +0,0 @@ -Data: -------------------------------------------------------------------------- - Miller array info: /das/work/p17/p17490/Peter/Library/work_trees_torchref/dev/paper/data/1A0F/1A0F.mtz:FP,SIGFP,systematic_absences_eliminated - Observation type: xray.amplitude - Type of data: double, size=25970 - Type of sigmas: double, size=25970 - Number of Miller indices: 25970 - Anomalous flag: False - Unit cell: (90.65, 95.39, 51.21, 90, 90, 90) - Space group: P 21 21 21 (No. 19) - Systematic absences: 0 - Centric reflections: 2574 - Resolution range: 45.325 2.02435 - Completeness in resolution range: 0.875413 - Completeness with d_max=infinity: 0.875354 - Wavelength: 0.0000 -R-free-flags: ------------------------------------------------------------------ - flag value: 0 -Model vs Data: ----------------------------------------------------------------- - Resolution Compl Nwork Nfree R_work kiso kani kmask - 45.325-14.495 94.29 97 2 0.2775 688.519 655.651 1.284 1.002 0.293 - 14.491-11.472 98.99 96 2 0.3639 700.173 597.889 0.958 1.009 0.287 - 11.444-9.039 98.94 185 2 0.2887 745.013 670.776 1.083 1.017 0.290 - 9.024-7.135 97.88 365 5 0.1987 614.362 574.569 1.197 1.016 0.300 - 7.131-5.634 97.82 701 18 0.2216 469.403 454.948 1.256 1.017 0.311 - 5.631-4.446 98.05 1435 27 0.1924 589.301 568.877 1.284 1.013 0.309 - 4.444-3.511 97.02 2783 54 0.1852 564.616 552.915 1.381 1.007 0.300 - 3.510-2.772 96.29 5553 109 0.2318 337.327 328.672 1.382 0.996 0.082 - 2.771-2.188 89.30 10350 205 0.2276 201.210 193.843 1.364 0.975 0.000 - 2.188-2.024 65.77 3898 76 0.2207 135.273 129.088 1.342 0.953 0.000 - - r_work: 0.2174 - r_free: 0.2426 - - Number of F-obs outliers: 7 -Information extracted from PDB file header: ------------------------------------ - program_name : REFMAC - year : 2026 - r_work : 0.22009 - r_free : 0.24854 - high_resolution : 2.02 - low_resolution : 45.33 - sigma_cutoff : None - matthews_coeff : None - solvent_cont : None diff --git a/paper/figure3_performance/SF_calc_comparison/.gitignore b/paper/figure3_performance/SF_calc_comparison/.gitignore deleted file mode 100644 index 2467d5a..0000000 --- a/paper/figure3_performance/SF_calc_comparison/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# Derived benchmark data — regenerated by run_benchmarks.py / plot_results.py. -# Keep the scripts (*.py, *.sbatch, *.sh, README.md) tracked; ignore outputs. -results_*/ -output/ - -# SLURM job logs (the repo-root rule only matches a file named ".out", not "*.out") -*.out -*.err diff --git a/paper/figure3_performance/SF_calc_comparison/README.md b/paper/figure3_performance/SF_calc_comparison/README.md deleted file mode 100644 index e4f7325..0000000 --- a/paper/figure3_performance/SF_calc_comparison/README.md +++ /dev/null @@ -1,107 +0,0 @@ -# SFcalculator vs TorchRef SF_DS / SF_FFT — performance benchmark - -Direct performance comparison (requested in paper review) between three -differentiable structure-factor calculators, plus a classical reference: - -| Method | What it is | -|--------|------------| -| **TorchRef SF_FFT** | TorchRef FFT-based structure factors (`SfFFT`) | -| **TorchRef SF_DS** | TorchRef direct-summation structure factors (`SfDS`) | -| **SFcalculator** | `SFC_Torch.SFcalculator` (Hekstra lab), `calc_fprotein` | -| **cctbx** (ref) | `iotbx` `xray_structure.structure_factors().f_calc()`, CPU only | - -Metrics, per structure, on **GPU** and **CPU (16 threads)**: -- **forward** time (SF calculation only) -- **forward + backward** time (gradient w.r.t. atom xyz + isotropic B-factors) -- **peak memory** (forward and forward+backward, measured separately) - -## What is compared (fairness) - -- **Bare protein structure factor only** — no bulk-solvent mask, no scaling. - SFcalc `calc_fprotein` is the matching quantity to TorchRef's model SF. -- **float32 / complex64** for all three differentiable methods (TorchRef default - == SFcalc default). cctbx is double precision — treat it as a separate-precision - classical reference, not a like-for-like point. -- **Same reflections to `d_min`** with full crystal symmetry and Cromer-Mann / - ITC92 atomic form factors. TorchRef computes over the MTZ reflection list; - SFcalc computes over its ASU list (`Hasu_array`) — each run records its own - `n_reflections` in the JSON / `summary.csv` for transparency. -- **Gradients** flow through atom positions + isotropic B-factors for all three - (`loss = |F|.sum()`). - -### Measurement methodology -- **One metric per process.** Each `(method, structure)` is benchmarked by - separate worker processes — one for timing, one for forward peak memory, one - for forward+backward peak memory — so the three figures are independent and a - warmed allocator from one never contaminates another. -- **Always warmed.** Every process runs `--n-warmup` calls before measuring, so - compiled/Triton kernels and lazy buffers are built first (a cold first call - would misreport both time and memory). -- **Memory.** GPU: warm up, then `reset_peak_memory_stats` + one steady-state - call → `max_memory_allocated` (excludes CUDA context, includes the call's - tensors). CPU: baseline RSS after model build, then peak RSS across - warmup+measured via `psutil` (robust to torch's CPU caching allocator, which - does not return freed blocks to the OS). -- **Failures are recorded, not dropped.** Each cell carries a `status` - (`ok` / `oom` / `error` / `timeout`) per metric in the JSON and `summary.csv`, - so e.g. SFcalc running out of GPU memory on large structures shows up - explicitly rather than as a silent gap. - -### Caveats -- **SF_DS auto-batches** to a memory budget (`SfDS(max_memory_gb=...)`, default - 2 GB). Numbers reflect that as-shipped behavior: its time is for batched - execution. This is how a user actually runs it. -- **cctbx `f_calc` is single-threaded** regardless of `OMP_NUM_THREADS`; its CPU - number is effectively single-core, and it is forward-only (no autodiff/memory). -- **SFcalc** has no internal batching and builds dense `(N_atoms, N_HKL)` - tensors; it can exhaust GPU memory on large structures (recorded as `oom`). - -## Structures (mixed size series, from `tests/files/`) - -3GR5 (1.3k atoms, iso) · 1DAW (3k, iso) · 6G9X (5.9k, **aniso**) · 5BOV (11.4k, -**aniso**). Override with `--structures`. - -The series mixes isotropic and anisotropic (ANISOU) structures so the -direct-summation **anisotropic fast paths** (and the Triton DS kernel) are -exercised, not just the isotropic path. - -## Running (SLURM) - -```bash -cd paper/figure3_performance/SF_calc_comparison -bash submit_all.sh -``` - -This submits a CPU job (`--partition=hour`, 16 cores), a GPU job -(`--partition=gpu-day --gres=gpu:1`), and a plotting job that runs after both, -all writing into a shared `results_/` directory. Watch with -`squeue -u $USER`. - -Individual pieces: -```bash -sbatch submit_cpu.sbatch # CPU only -sbatch submit_gpu.sbatch # GPU only -.dev/bin/python run_benchmarks.py --device cpu # run directly (no SLURM) -.dev/bin/python plot_results.py --results-dir results_ -``` - -## Outputs - -``` -results_/ - cpu/ {STRUCT}_{method}.json summary.csv - gpu/ {STRUCT}_{method}.json summary.csv - figure3_sf_comparison_cpu.png - figure3_sf_comparison_gpu.png -``` - -`summary.csv` columns: `device, structure, n_atoms, n_reflections, d_min, -method, dtype, torch_threads, fwd_{mean,min,max}, fwd_bwd_{mean,min,max}, -mem_fwd_mb, mem_fwd_bwd_mb`. - -## Files - -- `benchmark_worker.py` — benchmarks one `(method, structure)` in its own process. -- `run_benchmarks.py` — orchestrates all method×structure runs for one device. -- `submit_cpu.sbatch`, `submit_gpu.sbatch`, `submit_all.sh` — SLURM submission. -- `plot_results.py` — grouped bar charts per device. diff --git a/paper/figure3_performance/SF_calc_comparison/bench_iso_vs_aniso.py b/paper/figure3_performance/SF_calc_comparison/bench_iso_vs_aniso.py deleted file mode 100644 index 212a1ed..0000000 --- a/paper/figure3_performance/SF_calc_comparison/bench_iso_vs_aniso.py +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env python -"""Benchmark: isotropic vs anisotropic structure-factor calculation. - -Compares the cost of the ISO vs ANISO path for both TorchRef SF pipelines -(``sf_fft`` = electron-density + FFT, ``sf_ds`` = direct summation) on the SAME -atoms and the SAME reflections, so the time delta is purely the per-atom math -overhead of anisotropic ADPs (3x3 covariance + inverse) over isotropic ones. - -Every atom is run twice: - * ISO — isotropic B-factor ``b`` (the structure's B_eq). - * ANISO — anisotropic ``U = (b / 8*pi^2) * I`` (iso-equivalent diagonal U). -Because the ANISO U is the iso-equivalent, both paths produce the SAME structure -factors (checked), isolating the kernel cost rather than a different workload. - -Reuses the timing helpers from ``benchmark_worker.py``. - -Usage: - TORCHREF_NUM_THREADS=8 .dev/bin/python bench_iso_vs_aniso.py \ - --device cuda --structures 1DAW 2DQ6 --n-iterations 20 --n-warmup 5 -""" - -import argparse -import math -import os - -import torch - -from benchmark_worker import ( - _resolve_files, - _summarize_times, - _time_iterations, - _time_iterations_gpu, -) - -EIGHT_PI_SQ = 8.0 * math.pi**2 - - -def _all_atoms(M): - """Concatenate iso + aniso atoms into one set with both ADP representations. - - Returns xyz, occ, A, B (all atoms) plus matched ``b`` (isotropic) and ``u`` - (anisotropic, 6-comp) where the iso-origin atoms get ``U = b/(8*pi^2)*I`` and - the aniso-origin atoms get a B_eq = 8*pi^2*tr(U)/3. - """ - xi, adpi, occi, Ai, Bi = M.get_iso() - xa, ua, occa, Aa, Ba = M.get_aniso() - - def cat(a, b): - if a is None or len(a) == 0: - return b - if b is None or len(b) == 0: - return a - return torch.cat([a, b], dim=0) - - # iso-origin atoms -> diagonal U; aniso-origin atoms -> B_eq - u_from_iso = torch.zeros(len(xi), 6, device=xi.device, dtype=xi.dtype) - u_from_iso[:, 0] = u_from_iso[:, 1] = u_from_iso[:, 2] = adpi / EIGHT_PI_SQ - if xa is not None and len(xa) > 0: - b_from_aniso = EIGHT_PI_SQ * (ua[:, 0] + ua[:, 1] + ua[:, 2]) / 3.0 - else: - b_from_aniso = None - - xyz = cat(xi, xa) - occ = cat(occi, occa) - A = cat(Ai, Aa) - B = cat(Bi, Ba) - b = cat(adpi, b_from_aniso) - u = cat(u_from_iso, ua) - return xyz, occ, A, B, b, u - - -def _empty(device, dtype): - return ( - torch.zeros(0, 3, device=device, dtype=dtype), - torch.zeros(0, device=device, dtype=dtype), - torch.zeros(0, device=device, dtype=dtype), - torch.zeros(0, 5, device=device, dtype=dtype), - torch.zeros(0, 5, device=device, dtype=dtype), - ) - - -def _build(engine, hkl, xyz, occ, A, B, b, u, mode): - """Return a (fwd, fwd_bwd) pair for ``mode`` in {'iso','aniso'}.""" - device, dtype = xyz.device, xyz.dtype - ex, eadp, eocc, eA, eB = _empty(device, dtype) - - if mode == "iso": - xyz_p = xyz.clone().detach().requires_grad_(True) - adp_p = b.clone().detach().requires_grad_(True) - occ_d, A_d, B_d = occ.detach(), A.detach(), B.detach() - grads = (xyz_p, adp_p) - - def call(): - return engine.compute_structure_factors( - hkl, xyz_p, adp_p, occ_d, A_d, B_d - )[0] - else: # aniso - xyz_p = xyz.clone().detach().requires_grad_(True) - u_p = u.clone().detach().requires_grad_(True) - occ_d, A_d, B_d = occ.detach(), A.detach(), B.detach() - grads = (xyz_p, u_p) - - def call(): - return engine.compute_structure_factors( - hkl, ex, eadp, eocc, eA, eB, - xyz_p, u_p, occ_d, A_d, B_d, - )[0] - - def fwd(): - return call() - - def fwd_bwd(): - for t in grads: - t.grad = None - sf = call() - loss = sf.abs().sum() - loss.backward() - return loss - - return fwd, fwd_bwd - - -def bench_structure(method, structure, device_str, n_iter, n_warmup): - from torchref import ModelFT, ReflectionData - from torchref.model import SfDS - - device = torch.device(device_str) - is_gpu = device.type == "cuda" - timer = _time_iterations_gpu if is_gpu else _time_iterations - - pdb, mtz = _resolve_files(structure) - data = ReflectionData(device=device).load_mtz(mtz) - d_min = float(data.d_min) - M = ModelFT(max_res=d_min, device=device, radius_angstrom=3.0).load_pdb(pdb) - hkl = data()[0] - xyz, occ, A, B, b, u = _all_atoms(M) - n_atoms, n_refl = int(xyz.shape[0]), int(hkl.shape[0]) - - engine = M.fft if method == "sf_fft" else SfDS(M.cell, M.spacegroup, device=device) - - out = {"method": method, "structure": structure, "n_atoms": n_atoms, - "n_refl": n_refl, "d_min": d_min} - - # correctness: iso vs aniso F should agree (iso-equivalent U) - with torch.no_grad(): - f_iso = _build(engine, hkl, xyz, occ, A, B, b, u, "iso")[0]() - f_ani = _build(engine, hkl, xyz, occ, A, B, b, u, "aniso")[0]() - out["F_rel_err"] = ((f_iso - f_ani).abs().max() / f_iso.abs().max()).item() - - for mode in ("iso", "aniso"): - fwd, fwd_bwd = _build(engine, hkl, xyz, occ, A, B, b, u, mode) - with torch.no_grad(): - for _ in range(n_warmup): - fwd() - if is_gpu: - torch.cuda.synchronize() - out[f"{mode}_fwd"] = _summarize_times(timer(fwd, n_iter))["mean_time"] - for _ in range(n_warmup): - fwd_bwd() - if is_gpu: - torch.cuda.synchronize() - out[f"{mode}_fwd_bwd"] = _summarize_times(timer(fwd_bwd, n_iter))["mean_time"] - - return out - - -def main(): - p = argparse.ArgumentParser(description="iso vs aniso SF benchmark") - p.add_argument("--device", default="cuda", choices=["cpu", "cuda"]) - p.add_argument("--methods", nargs="+", default=["sf_fft", "sf_ds"]) - p.add_argument("--structures", nargs="+", default=["1DAW"]) - p.add_argument("--n-iterations", type=int, default=20) - p.add_argument("--n-warmup", type=int, default=5) - args = p.parse_args() - - if args.device == "cuda" and not torch.cuda.is_available(): - raise SystemExit("CUDA requested but not available") - - hdr = (f"{'method':7s} {'struct':6s} {'atoms':>6s} {'refl':>6s} " - f"{'iso_fwd':>9s} {'ani_fwd':>9s} {'x':>5s} " - f"{'iso_f+b':>9s} {'ani_f+b':>9s} {'x':>5s} {'F_relerr':>9s}") - print(f"\n=== iso vs aniso SF [{args.device}] " - f"(times in ms, x = aniso/iso) ===") - print(hdr) - print("-" * len(hdr)) - for method in args.methods: - for s in args.structures: - try: - r = bench_structure(method, s, args.device, - args.n_iterations, args.n_warmup) - except Exception as e: - print(f"{method:7s} {s:6s} ERROR: {type(e).__name__}: {e}") - continue - print( - f"{r['method']:7s} {r['structure']:6s} {r['n_atoms']:6d} " - f"{r['n_refl']:6d} " - f"{r['iso_fwd']*1e3:9.3f} {r['aniso_fwd']*1e3:9.3f} " - f"{r['aniso_fwd']/r['iso_fwd']:5.2f} " - f"{r['iso_fwd_bwd']*1e3:9.3f} {r['aniso_fwd_bwd']*1e3:9.3f} " - f"{r['aniso_fwd_bwd']/r['iso_fwd_bwd']:5.2f} " - f"{r['F_rel_err']:9.1e}" - ) - - -if __name__ == "__main__": - main() diff --git a/paper/figure3_performance/SF_calc_comparison/benchmark_worker.py b/paper/figure3_performance/SF_calc_comparison/benchmark_worker.py deleted file mode 100644 index 22e8146..0000000 --- a/paper/figure3_performance/SF_calc_comparison/benchmark_worker.py +++ /dev/null @@ -1,493 +0,0 @@ -#!/usr/bin/env python -""" -Worker for the SFcalculator vs TorchRef (SF_DS / SF_FFT) structure-factor benchmark. - -Benchmarks ONE (method, structure) on ONE device, in its own process so peak -memory is cleanly attributable. Measures: - - forward time (structure-factor calc only, under no_grad) - - forward+backward time (gradient w.r.t. xyz + B-factors) - - peak memory (forward and forward+backward, measured separately) - -All three differentiable methods compute the BARE PROTEIN structure factor (no -bulk solvent, no scaling) over the reflections to d_min, with full crystal -symmetry, in float32 / complex64. cctbx is a classical double-precision CPU -reference (forward only). - -TORCHREF_NUM_THREADS must be set in the environment BEFORE this script runs. - -Usage (called by run_benchmarks.py, not usually directly): - TORCHREF_NUM_THREADS=16 python benchmark_worker.py \ - --method sf_fft --structure 1DAW --device cpu \ - --n-iterations 10 --n-warmup 3 --output out.json -""" - -import argparse -import io -import json -import os -import sys -import threading -import time -import warnings - -# Read thread count before importing torch (mirrors fcalc_benchmark worker). -N_THREADS = int(os.environ.get("TORCHREF_NUM_THREADS", 1)) - -# Repo root = .../comparison_SFcalc (this file is paper/figure3_performance/SF_calc_comparison/) -REPO_ROOT = os.path.abspath( - os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "..") -) -PDB_DIR = os.path.join(REPO_ROOT, "tests", "files", "pdb") -MTZ_DIR = os.path.join(REPO_ROOT, "tests", "files", "mtz") - -METHODS = ("sf_fft", "sf_ds", "sfcalc", "cctbx") - - -# --------------------------------------------------------------------------- # -# Timing helpers (reused verbatim from fcalc_benchmark/benchmark_worker.py) -# --------------------------------------------------------------------------- # -def _time_iterations(func, n_iterations): - """Time repeated calls to func on CPU.""" - times = [] - for _ in range(n_iterations): - t0 = time.perf_counter() - func() - t1 = time.perf_counter() - times.append(t1 - t0) - return times - - -def _time_iterations_gpu(func, n_iterations): - """Time repeated calls to func on GPU with proper synchronization.""" - import torch - - times = [] - for _ in range(n_iterations): - torch.cuda.synchronize() - t0 = time.perf_counter() - func() - torch.cuda.synchronize() - t1 = time.perf_counter() - times.append(t1 - t0) - return times - - -def _summarize_times(times): - """Compute summary statistics from a list of iteration times.""" - total = sum(times) - return { - "iteration_times": times, - "total_time": total, - "mean_time": total / len(times), - "min_time": min(times), - "max_time": max(times), - } - - -# --------------------------------------------------------------------------- # -# Memory helpers -# --------------------------------------------------------------------------- # -class _RSSPeakSampler: - """Background thread sampling process RSS to estimate CPU peak memory. - - Reports peak RSS observed minus a baseline captured at start(), so the - figure reflects the memory the timed call itself adds (Python + torch + - loaded model are already resident at baseline). - """ - - def __init__(self, interval=0.005): - import psutil - - self._proc = psutil.Process() - self._interval = interval - self._stop = threading.Event() - self._thread = None - self.baseline = 0 - self.peak = 0 - - def __enter__(self): - self.baseline = self._proc.memory_info().rss - self.peak = self.baseline - self._stop.clear() - self._thread = threading.Thread(target=self._run, daemon=True) - self._thread.start() - return self - - def _run(self): - while not self._stop.is_set(): - rss = self._proc.memory_info().rss - if rss > self.peak: - self.peak = rss - time.sleep(self._interval) - - def __exit__(self, *exc): - self._stop.set() - if self._thread is not None: - self._thread.join() - # One final reading in case the peak occurred between samples. - rss = self._proc.memory_info().rss - if rss > self.peak: - self.peak = rss - - @property - def delta_bytes(self): - return max(0, self.peak - self.baseline) - - -def _trim_cpu_allocator(): - """Best effort: collect garbage and return freed heap to the OS. - - Without this, freed CPU tensors stay cached by the allocator and inflate - the RSS baseline, contaminating the peak-minus-baseline measurement. - """ - import ctypes - import gc - - gc.collect() - try: - ctypes.CDLL("libc.so.6").malloc_trim(0) - except Exception: - pass - - -def _measure_peak_memory(func, device, is_gpu, n_warmup): - """Return peak memory (bytes) for func, measured AFTER warmup. - - Runs in a dedicated process (one metric per process) so the measurement is - independent. Kernels are warmed first (Triton/compiled kernels and lazy - buffers must be built before measuring — a cold first call would misreport). - - - GPU: warm up, then reset the allocator high-water mark and measure one - steady-state call via ``max_memory_allocated`` (excludes the CUDA context - and cached kernels, includes the call's tensors). - - CPU: capture the RSS baseline after model build, then take the peak RSS - reached across warmup + the measured call (robust to torch's CPU caching - allocator, which does not return freed blocks to the OS). - """ - import torch - - if is_gpu: - for _ in range(n_warmup): - func() - torch.cuda.synchronize() - torch.cuda.empty_cache() - torch.cuda.reset_peak_memory_stats(device) - func() - torch.cuda.synchronize() - return int(torch.cuda.max_memory_allocated(device)) - else: - _trim_cpu_allocator() - with _RSSPeakSampler() as sampler: # baseline = RSS before warmup - for _ in range(n_warmup): - func() - func() - return int(sampler.delta_bytes) - - -# --------------------------------------------------------------------------- # -# Common data loading -# --------------------------------------------------------------------------- # -def _resolve_files(structure): - pdb = os.path.join(PDB_DIR, f"{structure}.pdb") - mtz = os.path.join(MTZ_DIR, f"{structure}.mtz") - for p in (pdb, mtz): - if not os.path.exists(p): - raise FileNotFoundError(p) - return pdb, mtz - - -# --------------------------------------------------------------------------- # -# Per-method benchmark builders -# -# Each returns: (fwd, fwd_bwd, n_atoms, n_reflections, d_min, dtype_str) -# fwd -> callable computing the SF (no grad needed; caller wraps no_grad) -# fwd_bwd -> callable computing SF, loss = |F|.sum(), loss.backward() -# --------------------------------------------------------------------------- # -def _build_torchref(method, pdb, mtz, device): - """Build fwd / fwd_bwd closures for sf_fft or sf_ds.""" - import torch - from torchref import ModelFT, ReflectionData - - data = ReflectionData(device=device).load_mtz(mtz) - d_min = float(data.d_min) - M = ModelFT(max_res=d_min, device=device, radius_angstrom=3.0).load_pdb(pdb) - hkl, _, _, _ = data() - - n_atoms = int(M.xyz().shape[0]) - n_reflections = int(hkl.shape[0]) - - xyz_i, adp_i, occ_i, A_i, B_i = M.get_iso() - aniso_ref = M.get_aniso() # (xyz, u, occ, A, B); tensors may be None - - # Only xyz + ADP carry gradients (matches SFcalc: positions + B-factors). - xyz_i = xyz_i.clone().detach().requires_grad_(True) - adp_i = adp_i.clone().detach().requires_grad_(True) - occ_i = occ_i.clone().detach() - A_i = A_i.clone().detach() - B_i = B_i.clone().detach() - iso = (xyz_i, adp_i, occ_i, A_i, B_i) - - def _clone_aniso(t, grad=False): - if t is None: - return None - t = t.clone().detach() - return t.requires_grad_(True) if grad else t - - aniso = tuple( - _clone_aniso(t, grad=(idx in (0, 1))) # xyz_aniso, u_aniso - for idx, t in enumerate(aniso_ref) - ) - - if method == "sf_fft": - engine = M.fft - else: # sf_ds - from torchref.model import SfDS - - engine = SfDS(M.cell, M.spacegroup, device=device) - - dtype_str = str(iso[0].dtype) - - def fwd(): - sf, _ = engine.compute_structure_factors(hkl, *iso, *aniso) - return sf - - def fwd_bwd(): - for t in (xyz_i, adp_i): - if t.grad is not None: - t.grad = None - sf, _ = engine.compute_structure_factors(hkl, *iso, *aniso) - loss = sf.abs().sum() - loss.backward() - return loss - - return fwd, fwd_bwd, n_atoms, n_reflections, d_min, dtype_str - - -def _build_sfcalc(pdb, mtz, device): - """Build fwd / fwd_bwd closures for SFcalculator (SFC_Torch).""" - import torch - from SFC_Torch import SFcalculator - - sfc = SFcalculator( - pdb, mtz, device=torch.device(device), set_experiment=False - ) - # Trigger any lazy form-factor / symmetry setup with one bare call. - sfc.calc_fprotein() - - # Gradients on positions + isotropic B-factors (matches the torch methods). - sfc.atom_pos_orth.requires_grad_(True) - sfc.atom_b_iso.requires_grad_(True) - - n_atoms = int(sfc.atom_pos_orth.shape[0]) - # SFcalc computes over the ASU reflection list (Hasu_array), then maps to - # the MTZ HKL set; report the count it actually computes. - n_reflections = int(sfc.Hasu_array.shape[0]) - d_min = float(sfc.dHKL.min()) if hasattr(sfc, "dHKL") else float("nan") - dtype_str = str(sfc.atom_pos_orth.dtype) - - def fwd(): - return sfc.calc_fprotein(Return=True) - - def fwd_bwd(): - if sfc.atom_pos_orth.grad is not None: - sfc.atom_pos_orth.grad = None - if sfc.atom_b_iso.grad is not None: - sfc.atom_b_iso.grad = None - Fp = sfc.calc_fprotein(Return=True) - loss = Fp.abs().sum() - loss.backward() - return loss - - return fwd, fwd_bwd, n_atoms, n_reflections, d_min, dtype_str - - -def _build_cctbx(pdb, mtz): - """Build fwd closure for cctbx (classical reference, CPU, forward only).""" - from iotbx import pdb as iotbx_pdb - from torchref import ReflectionData - - # Use the same d_min as the torch methods for a comparable workload. - data = ReflectionData().load_mtz(mtz) - d_min = float(data.d_min) - n_reflections = int(data()[0].shape[0]) - - pdb_input = iotbx_pdb.input(file_name=pdb) - xray_structure = pdb_input.xray_structure_simple() - n_atoms = int(xray_structure.scatterers().size()) - - def fwd(): - return xray_structure.structure_factors(d_min=d_min).f_calc() - - return fwd, n_atoms, n_reflections, d_min, "float64(cctbx)" - - -# --------------------------------------------------------------------------- # -# Main benchmark -# -# One METRIC per process so each is measured independently from a cold -# allocator (critical for honest CPU memory numbers): -# mode "time" -> forward + forward/backward timing -# mode "mem_fwd" -> peak memory of a single forward call -# mode "mem_fwd_bwd" -> peak memory of a single forward+backward call -# --------------------------------------------------------------------------- # -# One metric per process. time_fwd / time_fwd_bwd are split so a backward OOM -# does not also lose the forward timing. ("time" kept as a combined alias.) -MODES = ("time", "time_fwd", "time_fwd_bwd", "mem_fwd", "mem_fwd_bwd") - - -def _fwd_no_grad(fwd): - import torch - - with torch.no_grad(): - fwd() - - -def _build(method, structure, device): - """Return (fwd, fwd_bwd_or_None, n_atoms, n_refl, d_min, dtype).""" - pdb, mtz = _resolve_files(structure) - if method == "cctbx": - fwd, n_atoms, n_refl, d_min, dtype_str = _build_cctbx(pdb, mtz) - return fwd, None, n_atoms, n_refl, d_min, dtype_str - if method in ("sf_fft", "sf_ds"): - return _build_torchref(method, pdb, mtz, device) - if method == "sfcalc": - return _build_sfcalc(pdb, mtz, device) - raise ValueError(f"Unknown method: {method}") - - -def run_benchmark(method, structure, device_str, n_iterations, n_warmup, mode): - import torch - - device = torch.device(device_str) - is_gpu = device.type == "cuda" - timer = _time_iterations_gpu if is_gpu else _time_iterations - - result = { - "method": method, - "structure": structure, - "device": device_str, - "mode": mode, - "n_threads": N_THREADS, - "n_iterations": n_iterations, - "n_warmup": n_warmup, - "torch_threads": torch.get_num_threads(), - "status": "ok", - } - if is_gpu: - result["gpu_name"] = torch.cuda.get_device_name(0) - if method == "cctbx" and is_gpu: - raise ValueError("cctbx benchmark is CPU only") - - fwd, fwd_bwd, n_atoms, n_refl, d_min, dtype_str = _build( - method, structure, device - ) - result.update( - {"n_atoms": n_atoms, "n_reflections": n_refl, "d_min": d_min, - "dtype": dtype_str} - ) - - if mode in ("time", "time_fwd"): - with torch.no_grad(): - for _ in range(n_warmup): - fwd() - if is_gpu: - torch.cuda.synchronize() - result["fwd"] = _summarize_times(timer(fwd, n_iterations)) - - if mode in ("time", "time_fwd_bwd"): - if fwd_bwd is not None: - fwd_bwd() # warmup (also builds caches) - for _ in range(max(0, n_warmup - 1)): - fwd_bwd() - if is_gpu: - torch.cuda.synchronize() - result["fwd_bwd"] = _summarize_times(timer(fwd_bwd, n_iterations)) - else: - result["fwd_bwd"] = None - - if mode == "mem_fwd": - result["mem_fwd_bytes"] = _measure_peak_memory( - lambda: _fwd_no_grad(fwd), device, is_gpu, n_warmup - ) - - elif mode == "mem_fwd_bwd": - if fwd_bwd is None: - result["mem_fwd_bwd_bytes"] = None # classical: no autodiff - else: - result["mem_fwd_bwd_bytes"] = _measure_peak_memory( - fwd_bwd, device, is_gpu, n_warmup - ) - - return result - - -def _classify_error(exc): - """Return 'oom' for out-of-memory errors, else 'error'.""" - import torch - - if isinstance(exc, torch.cuda.OutOfMemoryError): - return "oom" - msg = str(exc).lower() - if "out of memory" in msg or "can't allocate" in msg or "cannot allocate" in msg: - return "oom" - return "error" - - -def main(): - parser = argparse.ArgumentParser(description="SF comparison benchmark worker") - parser.add_argument("--method", required=True, choices=METHODS) - parser.add_argument("--structure", required=True) - parser.add_argument("--device", default="cpu", choices=["cpu", "cuda"]) - parser.add_argument("--mode", default="time", choices=MODES) - parser.add_argument("--n-iterations", type=int, default=10) - parser.add_argument("--n-warmup", type=int, default=3) - parser.add_argument("--output", required=True) - args = parser.parse_args() - - warnings.filterwarnings("ignore", message="TorchRef.*threads") - warnings.filterwarnings("ignore", message=".*deprecated.*") - - # Suppress verbose library loading chatter on stdout (keep stderr live). - old_stdout = sys.stdout - sys.stdout = io.StringIO() - status = "ok" - try: - result = run_benchmark( - args.method, args.structure, args.device, - args.n_iterations, args.n_warmup, args.mode, - ) - except Exception as exc: # capture (incl. OOM) rather than silently dropping - sys.stdout = old_stdout - status = _classify_error(exc) - result = { - "method": args.method, "structure": args.structure, - "device": args.device, "mode": args.mode, "status": status, - "error": f"{type(exc).__name__}: {exc}", - } - finally: - sys.stdout = old_stdout - - with open(args.output, "w") as f: - json.dump(result, f, indent=2) - - if status != "ok": - print(f" {args.method:7s} {args.structure} [{args.device}] " - f"{args.mode}: {status.upper()}") - return - - parts = [] - if "fwd" in result: - parts.append(f"fwd={result['fwd']['mean_time']:.4f}s") - if result.get("fwd_bwd"): - parts.append(f"fwd+bwd={result['fwd_bwd']['mean_time']:.4f}s") - if "mem_fwd_bytes" in result: - parts.append(f"mem(fwd)={result['mem_fwd_bytes'] / 1e6:.0f}MB") - if "mem_fwd_bwd_bytes" in result: - v = result["mem_fwd_bwd_bytes"] - parts.append(f"mem(fwd+bwd)={v / 1e6:.0f}MB" if v is not None else "mem(fwd+bwd)=n/a") - print(f" {args.method:7s} {args.structure} [{args.device}] " - f"{args.mode}: {' '.join(parts)}") - - -if __name__ == "__main__": - main() diff --git a/paper/figure3_performance/SF_calc_comparison/plot_results.py b/paper/figure3_performance/SF_calc_comparison/plot_results.py deleted file mode 100644 index 6aaac69..0000000 --- a/paper/figure3_performance/SF_calc_comparison/plot_results.py +++ /dev/null @@ -1,231 +0,0 @@ -#!/usr/bin/env python -""" -Plot the SFcalculator vs TorchRef (SF_DS / SF_FFT) benchmark. - -Reads results_//*.json and produces, per device, a 3-panel grouped -bar chart: forward time, forward+backward time, and peak memory; x-axis is the -structure (ordered by atom count), one bar group per method. - -Usage: - python plot_results.py --results-dir results_20260608_120000 -""" - -import argparse -import json -from pathlib import Path - -import matplotlib -matplotlib.use("Agg") -import matplotlib.pyplot as plt -plt.rcParams["font.family"] = "serif" -plt.rcParams["font.serif"] = ["STIXGeneral"] -plt.rcParams["mathtext.fontset"] = "stix" -plt.rcParams["font.size"] = 13 -plt.rcParams["axes.labelsize"] = 15 -plt.rcParams["axes.titlesize"] = 16 -import numpy as np - -# Method display order, labels, colors. -METHOD_ORDER = ["sf_fft", "sf_ds", "sfcalc", "cctbx"] -METHOD_LABEL = { - "sf_fft": "TorchRef SF_FFT", - "sf_ds": "TorchRef SF_DS", - "sfcalc": "SFcalculator", - "cctbx": "cctbx (ref)", -} -METHOD_COLOR = { - "sf_fft": "#2563eb", # blue - "sf_ds": "#7c3aed", # purple - "sfcalc": "#e67e22", # orange - "cctbx": "#6b7280", # gray -} - - -def load_device(device_dir: Path) -> list[dict]: - runs = [] - for jf in sorted(device_dir.glob("*.json")): - with open(jf) as f: - runs.append(json.load(f)) - return runs - - -def _structures_sorted(runs): - # Order structures by atom count (ascending). - by_struct = {} - for r in runs: - by_struct.setdefault(r["structure"], r.get("n_atoms", 0)) - return sorted(by_struct, key=lambda s: by_struct[s]) - - -def _value(runs, structure, method, kind): - """kind in {'fwd', 'fwd_bwd', 'mem_fwd', 'mem_fwd_bwd'}; returns float or nan.""" - for r in runs: - if r["structure"] == structure and r["method"] == method: - if kind == "fwd": - return r["fwd"]["mean_time"] if r.get("fwd") else np.nan - if kind == "fwd_bwd": - return r["fwd_bwd"]["mean_time"] if r.get("fwd_bwd") else np.nan - if kind == "mem_fwd": - v = r.get("mem_fwd_bytes") - return v / 1e6 if v is not None else np.nan - if kind == "mem_fwd_bwd": - v = r.get("mem_fwd_bwd_bytes") - return v / 1e6 if v is not None else np.nan - return np.nan - - -def _bar_panel(ax, runs, structures, methods, kind, ylabel, title): - n_m = len(methods) - width = 0.8 / n_m - x = np.arange(len(structures)) - # Log scale first so axis limits derive only from finite bar heights. - ax.set_yscale("log") - nan_marks = [] # (x_position,) for n/a annotations, drawn after autoscale - for i, m in enumerate(methods): - vals = [_value(runs, s, m, kind) for s in structures] - offset = (i - (n_m - 1) / 2) * width - ax.bar( - x + offset, vals, width, label=METHOD_LABEL[m], - color=METHOD_COLOR[m], edgecolor="black", linewidth=0.4, - ) - nan_marks += [xi for xi, v in zip(x + offset, vals) if np.isnan(v)] - ax.set_xticks(x) - ax.set_xticklabels(structures) - ax.set_ylabel(ylabel) - ax.set_title(title) - ax.grid(axis="y", which="both", ls=":", alpha=0.4) - # Annotate missing (OOM / skipped) cells. Use a blended transform so the - # text sits just above the x-axis in AXES-fraction y — never at data y=0, - # which on a log axis is -inf and would explode the figure size. - trans = ax.get_xaxis_transform() # x in data coords, y in axes fraction - for xi in nan_marks: - ax.text(xi, 0.02, "n/a", transform=trans, ha="center", va="bottom", - fontsize=8, rotation=90, color="gray") - - -def _norm_bar_panel(ax, runs, structures, methods, kind, title, ref="sf_fft"): - """Bar panel of each method's value divided by the reference method's. - - The reference (SF_FFT) is the baseline = 1.0 (drawn as a dashed line). Cells - where either the method or the reference is missing are marked 'n/a'. - """ - n_m = len(methods) - width = 0.8 / n_m - x = np.arange(len(structures)) - ax.set_yscale("log") - base = {s: _value(runs, s, ref, kind) for s in structures} - nan_marks = [] - for i, m in enumerate(methods): - ratios = [] - for s in structures: - v, b = _value(runs, s, m, kind), base[s] - ratios.append(v / b if (np.isfinite(v) and np.isfinite(b) and b > 0) - else np.nan) - offset = (i - (n_m - 1) / 2) * width - ax.bar(x + offset, ratios, width, label=METHOD_LABEL[m], - color=METHOD_COLOR[m], edgecolor="black", linewidth=0.4) - nan_marks += [xi for xi, r in zip(x + offset, ratios) if np.isnan(r)] - ax.axhline(1.0, color="black", ls="--", lw=1.0, zorder=0) - ax.set_xticks(x) - ax.set_xticklabels(structures) - ax.set_ylabel(f"× relative to {METHOD_LABEL[ref]}") - ax.set_title(title) - ax.grid(axis="y", which="both", ls=":", alpha=0.4) - trans = ax.get_xaxis_transform() - for xi in nan_marks: - ax.text(xi, 0.02, "n/a", transform=trans, ha="center", va="bottom", - fontsize=8, rotation=90, color="gray") - - -def plot_device_normalized(runs, device, out_path, ref="sf_fft"): - """Performance normalized to TorchRef SF_FFT (separate figure).""" - if not any(r["method"] == ref for r in runs): - print(f"Reference method {ref} absent for {device}; skipping normalized plot.") - return - structures = _structures_sorted(runs) - present = {r["method"] for r in runs} - # Plot every method except the reference itself. - methods = [m for m in METHOD_ORDER if m in present and m != ref] - diff_methods = [m for m in methods if m != "cctbx"] - - fig, axes = plt.subplots(1, 3, figsize=(17, 5.2)) - _norm_bar_panel(axes[0], runs, structures, methods, "fwd", "Forward", ref) - _norm_bar_panel(axes[1], runs, structures, diff_methods, "fwd_bwd", - "Forward + backward", ref) - _norm_bar_panel(axes[2], runs, structures, diff_methods, "mem_fwd_bwd", - "Peak memory (fwd + bwd)", ref) - - handles, labels = axes[0].get_legend_handles_labels() - fig.legend(handles, labels, loc="lower center", ncol=max(1, len(methods)), - frameon=False, bbox_to_anchor=(0.5, -0.02)) - dev_label = "GPU" if device == "gpu" else "CPU (16 threads)" - gpu_name = next((r.get("gpu_name") for r in runs if r.get("gpu_name")), None) - if gpu_name: - dev_label += f" — {gpu_name}" - fig.suptitle(f"Performance relative to TorchRef SF_FFT (higher = worse) " - f"[{dev_label}]", fontsize=17) - fig.tight_layout(rect=[0, 0.04, 1, 0.96]) - fig.savefig(out_path, dpi=200, bbox_inches="tight") - plt.close(fig) - print(f"Wrote {out_path}") - - -def plot_device(runs, device, out_path): - structures = _structures_sorted(runs) - present = {r["method"] for r in runs} - methods = [m for m in METHOD_ORDER if m in present] - - fig, axes = plt.subplots(1, 3, figsize=(17, 5.2)) - _bar_panel(axes[0], runs, structures, methods, "fwd", - "time per call (s)", "Forward") - # Backward / memory: only the differentiable methods (drop cctbx). - diff_methods = [m for m in methods if m != "cctbx"] - _bar_panel(axes[1], runs, structures, diff_methods, "fwd_bwd", - "time per call (s)", "Forward + backward") - _bar_panel(axes[2], runs, structures, diff_methods, "mem_fwd_bwd", - "peak memory (MB)", "Peak memory (fwd + bwd)") - - handles, labels = axes[0].get_legend_handles_labels() - fig.legend(handles, labels, loc="lower center", ncol=len(methods), - frameon=False, bbox_to_anchor=(0.5, -0.02)) - dev_label = "GPU" if device == "gpu" else "CPU (16 threads)" - gpu_name = next((r.get("gpu_name") for r in runs if r.get("gpu_name")), None) - if gpu_name: - dev_label += f" — {gpu_name}" - fig.suptitle(f"Structure-factor calculation: SFcalculator vs TorchRef " - f"[{dev_label}]", fontsize=17) - fig.tight_layout(rect=[0, 0.04, 1, 0.96]) - fig.savefig(out_path, dpi=200, bbox_inches="tight") - plt.close(fig) - print(f"Wrote {out_path}") - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--results-dir", required=True) - args = parser.parse_args() - root = Path(args.results_dir) - - # Device subdir name (as written by run_benchmarks: --device value) -> label. - devices = {"cpu": "cpu", "cuda": "gpu"} - any_plotted = False - for subdir, label in devices.items(): - device_dir = root / subdir - if not device_dir.is_dir(): - continue - runs = load_device(device_dir) - if not runs: - print(f"No JSON results in {device_dir}, skipping.") - continue - plot_device(runs, label, root / f"figure3_sf_comparison_{label}.png") - plot_device_normalized( - runs, label, root / f"figure3_sf_comparison_{label}_normalized.png" - ) - any_plotted = True - - if not any_plotted: - print(f"No results found under {root}/(cpu|cuda).") - - -if __name__ == "__main__": - main() diff --git a/paper/figure3_performance/SF_calc_comparison/run_benchmarks.py b/paper/figure3_performance/SF_calc_comparison/run_benchmarks.py deleted file mode 100644 index a56c017..0000000 --- a/paper/figure3_performance/SF_calc_comparison/run_benchmarks.py +++ /dev/null @@ -1,258 +0,0 @@ -#!/usr/bin/env python -""" -Orchestrator for the SFcalculator vs TorchRef (SF_DS / SF_FFT) benchmark. - -Launches one benchmark_worker.py subprocess per (structure x method) so each -method's peak memory is measured in a clean process. Collects per-run JSONs and -writes a combined summary.csv. - -Run via SLURM (see submit_cpu.sbatch / submit_gpu.sbatch): - python run_benchmarks.py --device cpu - python run_benchmarks.py --device cuda - -cctbx is benchmarked on CPU only. -""" - -import argparse -import csv -import json -import os -import subprocess -import sys -from datetime import datetime -from pathlib import Path - -SCRIPT_DIR = Path(__file__).parent.resolve() -WORKER = SCRIPT_DIR / "benchmark_worker.py" -PLOT_SCRIPT = SCRIPT_DIR / "plot_results.py" -PYTHON = sys.executable - -# Mixed size series: 3GR5/1DAW are isotropic; 6G9X/5BOV carry anisotropic -# (ANISOU) atoms to exercise the direct-summation aniso fast paths. Spread spans -# ~1.3k -> 11.4k atoms. -DEFAULT_STRUCTURES = ["3GR5", "1DAW", "6G9X", "5BOV"] -# Order matters only for the summary; "heavier" methods last. -TORCH_METHODS = ["sf_fft", "sf_ds", "sfcalc"] - - -def _clear_stale_extension_locks(): - """Remove stale torch C++ extension lock files left by killed processes.""" - cache_dir = Path.home() / ".cache" / "torch_extensions" - if not cache_dir.exists(): - return - for lock_file in cache_dir.rglob("lock"): - try: - lock_file.unlink() - print(f"Removed stale lock: {lock_file}") - except OSError: - pass - - -def run_worker(method, structure, device, n_iterations, n_warmup, output_file, - timeout, mode): - """Launch one worker subprocess for one metric-mode. Returns parsed dict.""" - cmd = [ - PYTHON, str(WORKER), - "--method", method, - "--structure", structure, - "--device", device, - "--mode", mode, - "--n-iterations", str(n_iterations), - "--n-warmup", str(n_warmup), - "--output", str(output_file), - ] - try: - result = subprocess.run( - cmd, stdout=subprocess.PIPE, stderr=None, text=True, timeout=timeout - ) - if result.stdout.strip(): - print(result.stdout.strip()) - if result.returncode != 0: - # Worker crashed without writing JSON (e.g. killed/segfault). - print(f" CRASH ({method}/{structure}/{mode}): rc={result.returncode}") - return {"status": "crash", "mode": mode} - with open(output_file) as f: - return json.load(f) - except subprocess.TimeoutExpired: - print(f" TIMEOUT ({method}/{structure}/{mode}): exceeded {timeout}s") - return {"status": "timeout", "mode": mode} - except Exception as e: - print(f" EXCEPTION ({method}/{structure}/{mode}): {e}") - return {"status": "error", "mode": mode} - - -def run_cell(method, structure, device, n_iterations, n_warmup, out_dir, timeout): - """Run all metric-modes for one (method, structure) and merge into one dict. - - Each metric runs in its own worker process (cold allocator) so timing and - the two memory figures are measured independently. - """ - # Split metrics so one OOM (e.g. backward) doesn't lose the others. - modes = ["time_fwd", "time_fwd_bwd", "mem_fwd", "mem_fwd_bwd"] - if method == "cctbx": - modes = ["time_fwd"] # classical reference: forward time only - - raw_dir = out_dir / "raw" - raw_dir.mkdir(parents=True, exist_ok=True) - merged = {"method": method, "structure": structure, "device": device} - statuses = {} - for mode in modes: - out_file = raw_dir / f"{structure}_{method}_{mode}.json" - r = run_worker(method, structure, device, n_iterations, n_warmup, - out_file, timeout, mode) - statuses[mode] = (r or {}).get("status", "missing") - if not r: - continue - # Copy metadata (first non-empty wins) and the metric this mode produced. - for k in ("n_atoms", "n_reflections", "d_min", "dtype", "gpu_name", - "torch_threads", "n_iterations", "n_warmup"): - if k in r and k not in merged: - merged[k] = r[k] - for k in ("fwd", "fwd_bwd", "mem_fwd_bytes", "mem_fwd_bwd_bytes"): - if k in r: - merged[k] = r[k] - merged["status"] = statuses - return merged - - -def _cell(stats): - if not stats: - return ("", "", "") - return ( - f"{stats['mean_time']:.6f}", - f"{stats['min_time']:.6f}", - f"{stats['max_time']:.6f}", - ) - - -def _fmt_status(status): - """Compact per-mode status, e.g. 'ok' or 'time=ok;mem_fwd=oom'.""" - if isinstance(status, dict): - if all(v == "ok" for v in status.values()): - return "ok" - return ";".join(f"{k}={v}" for k, v in status.items() if v != "ok") - return str(status) - - -def write_summary_csv(results, output_path): - fieldnames = [ - "device", "structure", "n_atoms", "n_reflections", "d_min", "method", - "dtype", "torch_threads", "status", - "fwd_mean", "fwd_min", "fwd_max", - "fwd_bwd_mean", "fwd_bwd_min", "fwd_bwd_max", - "mem_fwd_mb", "mem_fwd_bwd_mb", - ] - with open(output_path, "w", newline="") as f: - writer = csv.DictWriter(f, fieldnames=fieldnames) - writer.writeheader() - for r in results: - fwd_m, fwd_lo, fwd_hi = _cell(r.get("fwd")) - fb_m, fb_lo, fb_hi = _cell(r.get("fwd_bwd")) - mf = r.get("mem_fwd_bytes") - mfb = r.get("mem_fwd_bwd_bytes") - d_min = r.get("d_min") - writer.writerow({ - "device": r["device"], - "structure": r["structure"], - "n_atoms": r.get("n_atoms", ""), - "n_reflections": r.get("n_reflections", ""), - "d_min": f"{d_min:.4f}" if isinstance(d_min, (int, float)) else "", - "method": r["method"], - "dtype": r.get("dtype", ""), - "torch_threads": r.get("torch_threads", ""), - "status": _fmt_status(r.get("status", "")), - "fwd_mean": fwd_m, "fwd_min": fwd_lo, "fwd_max": fwd_hi, - "fwd_bwd_mean": fb_m, "fwd_bwd_min": fb_lo, "fwd_bwd_max": fb_hi, - "mem_fwd_mb": f"{mf / 1e6:.1f}" if mf is not None else "", - "mem_fwd_bwd_mb": f"{mfb / 1e6:.1f}" if mfb is not None else "", - }) - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--device", default="cpu", choices=["cpu", "cuda"]) - parser.add_argument("--structures", nargs="+", default=DEFAULT_STRUCTURES) - parser.add_argument("--n-iterations", type=int, default=10) - parser.add_argument("--n-warmup", type=int, default=3) - parser.add_argument( - "--timeout", type=int, default=1800, - help="Per-worker timeout in seconds (default 1800). SF_DS on large " - "structures can be slow; timed-out runs are skipped, not fatal.", - ) - parser.add_argument( - "--output-dir", default=None, - help="Default: results_// under this folder.", - ) - parser.add_argument("--no-plot", action="store_true") - parser.add_argument( - "--methods", nargs="+", default=None, - help="Override methods to run (default: sf_fft sf_ds sfcalc, + cctbx on cpu).", - ) - args = parser.parse_args() - - _clear_stale_extension_locks() - - if args.output_dir: - out_root = Path(args.output_dir) - else: - ts = datetime.now().strftime("%Y%m%d_%H%M%S") - out_root = SCRIPT_DIR / f"results_{ts}" - out_dir = out_root / args.device - out_dir.mkdir(parents=True, exist_ok=True) - - if args.methods: - methods = list(args.methods) - else: - methods = list(TORCH_METHODS) - if args.device == "cpu": - methods.append("cctbx") - - print("=" * 78) - print("SFcalculator vs TorchRef SF_DS / SF_FFT — benchmark") - print("=" * 78) - print(f"Device: {args.device}") - print(f"Structures: {args.structures}") - print(f"Methods: {methods}") - print(f"Iterations: {args.n_iterations} (+ {args.n_warmup} warmup)") - print(f"Output dir: {out_dir}") - print(f"Python: {PYTHON}") - print("=" * 78, flush=True) - - results = [] - total = len(args.structures) * len(methods) - i = 0 - for structure in args.structures: - for method in methods: - i += 1 - print(f"[{i}/{total}] {method} / {structure} ...", flush=True) - r = run_cell( - method, structure, args.device, - args.n_iterations, args.n_warmup, out_dir, args.timeout, - ) - results.append(r) - # Persist the merged per-cell result too. - with open(out_dir / f"{structure}_{method}.json", "w") as f: - json.dump(r, f, indent=2) - print(flush=True) - - if not results: - print("No successful runs. Exiting.") - sys.exit(1) - - csv_path = out_dir / "summary.csv" - write_summary_csv(results, csv_path) - print(f"Summary written to: {csv_path}") - - if not args.no_plot: - print("\nGenerating plots...") - try: - subprocess.run( - [PYTHON, str(PLOT_SCRIPT), "--results-dir", str(out_root)], - timeout=120, - ) - except Exception as e: - print(f"Plot generation failed (run plot_results.py manually): {e}") - - -if __name__ == "__main__": - main() diff --git a/paper/figure3_performance/SF_calc_comparison/submit_all.sh b/paper/figure3_performance/SF_calc_comparison/submit_all.sh deleted file mode 100644 index 9183b28..0000000 --- a/paper/figure3_performance/SF_calc_comparison/submit_all.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -# Submit the CPU + GPU SF-comparison benchmarks, then a dependent plotting job. -# Both device jobs write into a shared timestamped results dir so the plotter -# can combine them into per-device figures. -set -euo pipefail - -BENCH_DIR="/das/work/p17/p17490/Peter/Library/work_trees_torchref/comparison_SFcalc/paper/figure3_performance/SF_calc_comparison" -PYTHON="/das/work/p17/p17490/Peter/Library/work_trees_torchref/comparison_SFcalc/.dev/bin/python" -cd "${BENCH_DIR}" - -TS="$(date +%Y%m%d_%H%M%S)" -OUTPUT_DIR="${BENCH_DIR}/results_${TS}" -mkdir -p "${OUTPUT_DIR}" -echo "Shared results dir: ${OUTPUT_DIR}" - -CPU_JID=$(OUTPUT_DIR="${OUTPUT_DIR}" sbatch --parsable submit_cpu.sbatch) -echo "Submitted CPU job: ${CPU_JID}" - -GPU_JID=$(OUTPUT_DIR="${OUTPUT_DIR}" sbatch --parsable submit_gpu.sbatch) -echo "Submitted GPU job: ${GPU_JID}" - -# Plot once both device jobs have finished (afterany: plot whatever completed). -PLOT_JID=$(sbatch --parsable \ - --job-name=sf_cmp_plot \ - --partition=hour \ - --time=00:15:00 \ - --cpus-per-task=2 \ - --mem=8G \ - --output="${BENCH_DIR}/sf_cmp_plot_%j.out" \ - --dependency="afterany:${CPU_JID}:${GPU_JID}" \ - --wrap "${PYTHON} ${BENCH_DIR}/plot_results.py --results-dir ${OUTPUT_DIR}") -echo "Submitted plot job: ${PLOT_JID} (after ${CPU_JID}, ${GPU_JID})" - -echo -echo "Watch with: squeue -u \$USER" -echo "Results will be in: ${OUTPUT_DIR}/{cpu,gpu}/summary.csv" -echo "Figures in: ${OUTPUT_DIR}/figure3_sf_comparison_{cpu,gpu}.png" diff --git a/paper/figure3_performance/SF_calc_comparison/submit_bench_iso_aniso_cpu.sbatch b/paper/figure3_performance/SF_calc_comparison/submit_bench_iso_aniso_cpu.sbatch deleted file mode 100644 index 853d472..0000000 --- a/paper/figure3_performance/SF_calc_comparison/submit_bench_iso_aniso_cpu.sbatch +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=iso_aniso_cpu -#SBATCH --partition=day -#SBATCH --time=04:00:00 -#SBATCH --cpus-per-task=16 -#SBATCH --mem=64G -#SBATCH --ntasks=1 -#SBATCH --output=%x_%j.out - -set -euo pipefail - -REPO_ROOT="/das/work/p17/p17490/Peter/Library/work_trees_torchref/comparison_SFcalc" -BENCH_DIR="${REPO_ROOT}/paper/figure3_performance/SF_calc_comparison" -PYTHON="${REPO_ROOT}/.dev/bin/python" - -cd "${BENCH_DIR}" - -export TORCHREF_NUM_THREADS=16 -export OMP_NUM_THREADS=16 -export MKL_NUM_THREADS=16 -export OPENBLAS_NUM_THREADS=16 - -echo "=== iso vs aniso SF benchmark — CPU (16 threads) ===" -echo "Host: $(hostname)" -echo - -# FFT (electron-density) path: iso separable vs new aniso box kernel — both structures. -"${PYTHON}" bench_iso_vs_aniso.py \ - --device cpu \ - --methods sf_fft \ - --structures 1DAW 4BX9 \ - --n-iterations 10 \ - --n-warmup 3 - -# Direct summation is much heavier on CPU; benchmark the small structure only. -"${PYTHON}" bench_iso_vs_aniso.py \ - --device cpu \ - --methods sf_ds \ - --structures 1DAW \ - --n-iterations 5 \ - --n-warmup 2 diff --git a/paper/figure3_performance/SF_calc_comparison/submit_bench_iso_aniso_gpu.sbatch b/paper/figure3_performance/SF_calc_comparison/submit_bench_iso_aniso_gpu.sbatch deleted file mode 100644 index 317ccbe..0000000 --- a/paper/figure3_performance/SF_calc_comparison/submit_bench_iso_aniso_gpu.sbatch +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=iso_aniso_gpu -#SBATCH --partition=gpu -#SBATCH --time=01:00:00 -#SBATCH --gres=gpu:1 -#SBATCH --cpus-per-task=8 -#SBATCH --mem=48G -#SBATCH --ntasks=1 -#SBATCH --output=%x_%j.out - -set -euo pipefail - -REPO_ROOT="/das/work/p17/p17490/Peter/Library/work_trees_torchref/comparison_SFcalc" -BENCH_DIR="${REPO_ROOT}/paper/figure3_performance/SF_calc_comparison" -PYTHON="${REPO_ROOT}/.dev/bin/python" - -cd "${BENCH_DIR}" - -export TORCHREF_NUM_THREADS=8 -export OMP_NUM_THREADS=8 -export MKL_NUM_THREADS=8 - -echo "=== iso vs aniso SF benchmark — GPU ===" -echo "Host: $(hostname)" -nvidia-smi | head -15 || true -echo - -"${PYTHON}" bench_iso_vs_aniso.py \ - --device cuda \ - --methods sf_fft sf_ds \ - --structures 1DAW 4BX9 \ - --n-iterations 20 \ - --n-warmup 5 diff --git a/paper/figure3_performance/SF_calc_comparison/submit_cpu.sbatch b/paper/figure3_performance/SF_calc_comparison/submit_cpu.sbatch deleted file mode 100644 index 01f84fe..0000000 --- a/paper/figure3_performance/SF_calc_comparison/submit_cpu.sbatch +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=sf_cmp_cpu -#SBATCH --partition=day -#SBATCH --time=06:00:00 -#SBATCH --cpus-per-task=16 -#SBATCH --mem=128G -#SBATCH --ntasks=1 -#SBATCH --output=%x_%j.out - -set -euo pipefail - -REPO_ROOT="/das/work/p17/p17490/Peter/Library/work_trees_torchref/comparison_SFcalc" -BENCH_DIR="${REPO_ROOT}/paper/figure3_performance/SF_calc_comparison" -PYTHON="${REPO_ROOT}/.dev/bin/python" - -cd "${BENCH_DIR}" - -# 16-thread CPU benchmark. Note: cctbx structure_factors() is single-threaded -# regardless of these settings (documented in README). -export TORCHREF_NUM_THREADS=16 -export OMP_NUM_THREADS=16 -export MKL_NUM_THREADS=16 -export OPENBLAS_NUM_THREADS=16 - -echo "=== SF comparison — CPU (16 threads) ===" -echo "Host: $(hostname)" -echo "Output dir: ${OUTPUT_DIR:-auto}" -echo - -"${PYTHON}" run_benchmarks.py \ - --device cpu \ - --n-iterations 10 \ - --n-warmup 3 \ - ${OUTPUT_DIR:+--output-dir "${OUTPUT_DIR}"} \ - --no-plot diff --git a/paper/figure3_performance/SF_calc_comparison/submit_gpu.sbatch b/paper/figure3_performance/SF_calc_comparison/submit_gpu.sbatch deleted file mode 100644 index b2b889a..0000000 --- a/paper/figure3_performance/SF_calc_comparison/submit_gpu.sbatch +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=sf_cmp_gpu -#SBATCH --partition=gpu -#SBATCH --time=02:00:00 -#SBATCH --gres=gpu:a100:1 -#SBATCH --cpus-per-task=16 -#SBATCH --mem=64G -#SBATCH --ntasks=1 -#SBATCH --output=%x_%j.out - -set -euo pipefail - -REPO_ROOT="/das/work/p17/p17490/Peter/Library/work_trees_torchref/comparison_SFcalc" -BENCH_DIR="${REPO_ROOT}/paper/figure3_performance/SF_calc_comparison" -PYTHON="${REPO_ROOT}/.dev/bin/python" - -cd "${BENCH_DIR}" - -export TORCHREF_NUM_THREADS=16 -export OMP_NUM_THREADS=16 -export MKL_NUM_THREADS=16 - -echo "=== SF comparison — GPU ===" -echo "Host: $(hostname)" -nvidia-smi | head -20 || true -echo "Output dir: ${OUTPUT_DIR:-auto}" -echo - -"${PYTHON}" run_benchmarks.py \ - --device cuda \ - --n-iterations 10 \ - --n-warmup 3 \ - ${OUTPUT_DIR:+--output-dir "${OUTPUT_DIR}"} \ - --no-plot diff --git a/paper/figure3_performance/fcalc_benchmark/benchmark_cpu.py b/paper/figure3_performance/fcalc_benchmark/benchmark_cpu.py index c395368..f1098be 100644 --- a/paper/figure3_performance/fcalc_benchmark/benchmark_cpu.py +++ b/paper/figure3_performance/fcalc_benchmark/benchmark_cpu.py @@ -20,7 +20,7 @@ d_min = data.d_min print(f"d_min: {d_min}") -M = ModelFT(max_res=d_min, device=device,radius_angstrom=3.0).load_pdb(pdb_file) +M = ModelFT(max_res=d_min, device=device,radius_angstrom=4.0).load_pdb(pdb_file) hkl, _, _, _ = data() M(hkl, recalc=True) diff --git a/paper/figure3_performance/fcalc_benchmark/benchmark_thread_scaling.py b/paper/figure3_performance/fcalc_benchmark/benchmark_thread_scaling.py index 24b1e4e..b72c605 100644 --- a/paper/figure3_performance/fcalc_benchmark/benchmark_thread_scaling.py +++ b/paper/figure3_performance/fcalc_benchmark/benchmark_thread_scaling.py @@ -33,6 +33,29 @@ PYTHON = sys.executable +def _find_repo_root() -> Path: + """Locate the repo root (marker: pyproject.toml); honor TORCHREF_REPO_ROOT.""" + env_root = os.environ.get("TORCHREF_REPO_ROOT") + if env_root: + return Path(env_root).resolve() + for ancestor in [SCRIPT_DIR, *SCRIPT_DIR.parents]: + if (ancestor / "pyproject.toml").is_file(): + return ancestor + raise RuntimeError("Could not locate repo root (no pyproject.toml found).") + + +def discover_structures() -> list[str]: + """All test structures with a matching tests/files/{pdb,mtz}/{ID} pair. + + Mirrors tests/conftest.py::all_structure_pairs — intersection of PDB and + MTZ stems. Naturally excludes 1AK5 (pdb is 1AK5_with_H) and 7L84 (no mtz). + """ + files_dir = _find_repo_root() / "tests" / "files" + pdb_ids = {p.stem for p in (files_dir / "pdb").glob("*.pdb")} + mtz_ids = {p.stem for p in (files_dir / "mtz").glob("*.mtz")} + return sorted(pdb_ids & mtz_ids) + + def _clear_stale_extension_locks(): """Remove stale lock files from torch C++ extension cache. @@ -73,7 +96,8 @@ def get_default_max_threads() -> int: def run_worker(n_threads: int, n_iterations: int, n_warmup: int, - output_file: Path, device: str = "cpu") -> dict | None: + output_file: Path, structure: str = "1DAW", + device: str = "cpu", timeout: int = 600) -> dict | None: """Launch a benchmark worker subprocess with the given thread count.""" env = os.environ.copy() env["TORCHREF_NUM_THREADS"] = str(n_threads) @@ -88,6 +112,7 @@ def run_worker(n_threads: int, n_iterations: int, n_warmup: int, "--n_iterations", str(n_iterations), "--n_warmup", str(n_warmup), "--device", device, + "--structure", structure, "--output", str(output_file), ] @@ -98,7 +123,7 @@ def run_worker(n_threads: int, n_iterations: int, n_warmup: int, stdout=subprocess.PIPE, stderr=None, # pass stderr through to terminal for live progress text=True, - timeout=600, # 10 min timeout per run + timeout=timeout, ) # Print worker stdout (final summary) if result.stdout.strip(): @@ -111,127 +136,140 @@ def run_worker(n_threads: int, n_iterations: int, n_warmup: int, return json.load(f) except subprocess.TimeoutExpired: - print(f" TIMEOUT: exceeded 600s") + print(f" TIMEOUT: exceeded {timeout}s") return None except Exception as e: print(f" EXCEPTION: {e}") return None -def write_summary_csv(cpu_results: list[dict], gpu_result: dict | None, - output_path: Path): - """Write combined results to a CSV file.""" - fieldnames = [ - "device", "n_threads", - "torchref_mean", "torchref_min", "torchref_max", - "torchref_speedup", "torchref_efficiency", - "torchref_fwd_graph_mean", "torchref_fwd_graph_min", "torchref_fwd_graph_max", - "torchref_fwd_graph_speedup", - "torchref_bwd_only_mean", "torchref_bwd_only_min", "torchref_bwd_only_max", - "torchref_bwd_only_speedup", - "torchref_fwd_bwd_mean", "torchref_fwd_bwd_min", "torchref_fwd_bwd_max", - "torchref_fwd_bwd_speedup", - "cctbx_mean", "cctbx_min", "cctbx_max", - "cctbx_speedup", "cctbx_efficiency", - "n_iterations", "n_atoms", "n_reflections", "d_min", - ] - # Find single-thread baselines - tr_baseline = cc_baseline = fb_baseline = fg_baseline = bo_baseline = None +FIELDNAMES = [ + "structure", "device", "n_threads", + "torchref_mean", "torchref_min", "torchref_max", + "torchref_speedup", "torchref_efficiency", + "torchref_fwd_graph_mean", "torchref_fwd_graph_min", "torchref_fwd_graph_max", + "torchref_fwd_graph_speedup", + "torchref_bwd_only_mean", "torchref_bwd_only_min", "torchref_bwd_only_max", + "torchref_bwd_only_speedup", + "torchref_fwd_bwd_mean", "torchref_fwd_bwd_min", "torchref_fwd_bwd_max", + "torchref_fwd_bwd_speedup", + "cctbx_mean", "cctbx_min", "cctbx_max", + "cctbx_speedup", "cctbx_efficiency", + "n_iterations", "n_atoms", "n_reflections", "d_min", +] + + +def _row_for(r: dict, baselines: dict) -> dict: + """Build one CSV row from a result dict, using per-structure baselines. + + `baselines` maps metric prefix -> single-thread mean_time for this + structure (None if no 1-thread CPU run was recorded). Speedups are + relative to that baseline. GPU rows use n_threads=0 and omit cctbx. + """ + is_gpu = r["device"] != "cpu" + nt = 0 if is_gpu else r["n_threads"] + + def _speedup(prefix): + base = baselines.get(prefix) + return base / r[prefix]["mean_time"] if base else float("nan") + + row = { + "structure": r["structure"], + "device": "gpu" if is_gpu else "cpu", + "n_threads": nt, + "n_iterations": r["n_iterations"], + "n_atoms": r["n_atoms"], + "n_reflections": r["n_reflections"], + "d_min": r["d_min"], + } + for prefix in ("torchref", "torchref_fwd_graph", + "torchref_bwd_only", "torchref_fwd_bwd"): + stats = r[prefix] + sp = _speedup(prefix) + row[f"{prefix}_mean"] = f"{stats['mean_time']:.6f}" + row[f"{prefix}_min"] = f"{stats['min_time']:.6f}" + row[f"{prefix}_max"] = f"{stats['max_time']:.6f}" + row[f"{prefix}_speedup"] = f"{sp:.3f}" + # Efficiency only meaningful for the no_grad CPU forward. + tr_sp = _speedup("torchref") + row["torchref_efficiency"] = f"{tr_sp / nt:.3f}" if not is_gpu else "" + + # cctbx is CPU-only. + if not is_gpu and "cctbx" in r: + cc = r["cctbx"] + cc_base = baselines.get("cctbx") + cc_sp = cc_base / cc["mean_time"] if cc_base else float("nan") + row["cctbx_mean"] = f"{cc['mean_time']:.6f}" + row["cctbx_min"] = f"{cc['min_time']:.6f}" + row["cctbx_max"] = f"{cc['max_time']:.6f}" + row["cctbx_speedup"] = f"{cc_sp:.3f}" + row["cctbx_efficiency"] = f"{cc_sp / nt:.3f}" + else: + for k in ("cctbx_mean", "cctbx_min", "cctbx_max", + "cctbx_speedup", "cctbx_efficiency"): + row[k] = "" + return row + + +def _structure_baselines(cpu_results: list[dict]) -> dict: + """Per-structure 1-thread baselines: {structure: {prefix: mean_time}}.""" + baselines = {} for r in cpu_results: if r["n_threads"] == 1: - tr_baseline = r["torchref"]["mean_time"] - fg_baseline = r["torchref_fwd_graph"]["mean_time"] - bo_baseline = r["torchref_bwd_only"]["mean_time"] - fb_baseline = r["torchref_fwd_bwd"]["mean_time"] - cc_baseline = r["cctbx"]["mean_time"] - break + baselines[r["structure"]] = { + prefix: r[prefix]["mean_time"] + for prefix in ("torchref", "torchref_fwd_graph", + "torchref_bwd_only", "torchref_fwd_bwd", "cctbx") + if prefix in r + } + return baselines + + +def _aggregate_and_write(output_dir: Path): + """Rebuild the combined summary.csv from every per-structure JSON in output_dir. + + Used after per-structure / gpu-only shard jobs (run on separate nodes) have + each written their flat ``{structure}_threads_NN.json`` / ``{structure}_gpu.json`` + files; this gathers them all into one canonical summary.csv. + """ + cpu_results, gpu_results = [], [] + for p in sorted(output_dir.glob("*_threads_*.json")): + try: + with open(p) as f: + cpu_results.append(json.load(f)) + except Exception as e: + print(f" skip {p.name}: {e}") + for p in sorted(output_dir.glob("*_gpu.json")): + try: + with open(p) as f: + gpu_results.append(json.load(f)) + except Exception as e: + print(f" skip {p.name}: {e}") + csv_path = output_dir / "summary.csv" + write_summary_csv(cpu_results, gpu_results, csv_path) + n_struct = len({r["structure"] for r in cpu_results + gpu_results}) + print(f"Aggregated {len(cpu_results)} CPU + {len(gpu_results)} GPU JSONs " + f"({n_struct} structures) -> {csv_path}") + + +def write_summary_csv(cpu_results: list[dict], gpu_results: list[dict], + output_path: Path): + """Write combined multi-structure results to a CSV file. + + `cpu_results` / `gpu_results` are flat lists across all structures; each + dict carries its own "structure". Speedups are computed per structure + against that structure's single-thread CPU run. + """ + baselines = _structure_baselines(cpu_results) rows = [] - for r in sorted(cpu_results, key=lambda x: x["n_threads"]): - tr = r["torchref"] - fg = r["torchref_fwd_graph"] - bo = r["torchref_bwd_only"] - fb = r["torchref_fwd_bwd"] - cc = r["cctbx"] - nt = r["n_threads"] - tr_speedup = tr_baseline / tr["mean_time"] if tr_baseline else float("nan") - fg_speedup = fg_baseline / fg["mean_time"] if fg_baseline else float("nan") - bo_speedup = bo_baseline / bo["mean_time"] if bo_baseline else float("nan") - fb_speedup = fb_baseline / fb["mean_time"] if fb_baseline else float("nan") - cc_speedup = cc_baseline / cc["mean_time"] if cc_baseline else float("nan") - rows.append({ - "device": "cpu", - "n_threads": nt, - "torchref_mean": f"{tr['mean_time']:.6f}", - "torchref_min": f"{tr['min_time']:.6f}", - "torchref_max": f"{tr['max_time']:.6f}", - "torchref_speedup": f"{tr_speedup:.3f}", - "torchref_efficiency": f"{tr_speedup / nt:.3f}", - "torchref_fwd_graph_mean": f"{fg['mean_time']:.6f}", - "torchref_fwd_graph_min": f"{fg['min_time']:.6f}", - "torchref_fwd_graph_max": f"{fg['max_time']:.6f}", - "torchref_fwd_graph_speedup": f"{fg_speedup:.3f}", - "torchref_bwd_only_mean": f"{bo['mean_time']:.6f}", - "torchref_bwd_only_min": f"{bo['min_time']:.6f}", - "torchref_bwd_only_max": f"{bo['max_time']:.6f}", - "torchref_bwd_only_speedup": f"{bo_speedup:.3f}", - "torchref_fwd_bwd_mean": f"{fb['mean_time']:.6f}", - "torchref_fwd_bwd_min": f"{fb['min_time']:.6f}", - "torchref_fwd_bwd_max": f"{fb['max_time']:.6f}", - "torchref_fwd_bwd_speedup": f"{fb_speedup:.3f}", - "cctbx_mean": f"{cc['mean_time']:.6f}", - "cctbx_min": f"{cc['min_time']:.6f}", - "cctbx_max": f"{cc['max_time']:.6f}", - "cctbx_speedup": f"{cc_speedup:.3f}", - "cctbx_efficiency": f"{cc_speedup / nt:.3f}", - "n_iterations": r["n_iterations"], - "n_atoms": r["n_atoms"], - "n_reflections": r["n_reflections"], - "d_min": r["d_min"], - }) - - if gpu_result: - tr = gpu_result["torchref"] - fg = gpu_result["torchref_fwd_graph"] - bo = gpu_result["torchref_bwd_only"] - fb = gpu_result["torchref_fwd_bwd"] - tr_speedup = tr_baseline / tr["mean_time"] if tr_baseline else float("nan") - fg_speedup = fg_baseline / fg["mean_time"] if fg_baseline else float("nan") - bo_speedup = bo_baseline / bo["mean_time"] if bo_baseline else float("nan") - fb_speedup = fb_baseline / fb["mean_time"] if fb_baseline else float("nan") - rows.append({ - "device": "gpu", - "n_threads": 0, - "torchref_mean": f"{tr['mean_time']:.6f}", - "torchref_min": f"{tr['min_time']:.6f}", - "torchref_max": f"{tr['max_time']:.6f}", - "torchref_speedup": f"{tr_speedup:.3f}", - "torchref_efficiency": "", - "torchref_fwd_graph_mean": f"{fg['mean_time']:.6f}", - "torchref_fwd_graph_min": f"{fg['min_time']:.6f}", - "torchref_fwd_graph_max": f"{fg['max_time']:.6f}", - "torchref_fwd_graph_speedup": f"{fg_speedup:.3f}", - "torchref_bwd_only_mean": f"{bo['mean_time']:.6f}", - "torchref_bwd_only_min": f"{bo['min_time']:.6f}", - "torchref_bwd_only_max": f"{bo['max_time']:.6f}", - "torchref_bwd_only_speedup": f"{bo_speedup:.3f}", - "torchref_fwd_bwd_mean": f"{fb['mean_time']:.6f}", - "torchref_fwd_bwd_min": f"{fb['min_time']:.6f}", - "torchref_fwd_bwd_max": f"{fb['max_time']:.6f}", - "torchref_fwd_bwd_speedup": f"{fb_speedup:.3f}", - "cctbx_mean": "", - "cctbx_min": "", - "cctbx_max": "", - "cctbx_speedup": "", - "cctbx_efficiency": "", - "n_iterations": gpu_result["n_iterations"], - "n_atoms": gpu_result["n_atoms"], - "n_reflections": gpu_result["n_reflections"], - "d_min": gpu_result["d_min"], - }) + for r in sorted(cpu_results, key=lambda x: (x["structure"], x["n_threads"])): + rows.append(_row_for(r, baselines.get(r["structure"], {}))) + for r in sorted(gpu_results, key=lambda x: x["structure"]): + rows.append(_row_for(r, baselines.get(r["structure"], {}))) with open(output_path, "w", newline="") as f: - writer = csv.DictWriter(f, fieldnames=fieldnames) + writer = csv.DictWriter(f, fieldnames=FIELDNAMES) writer.writeheader() writer.writerows(rows) @@ -275,15 +313,58 @@ def main(): action="store_true", help="Skip GPU benchmark even if a GPU is available.", ) + parser.add_argument( + "--structures", + type=str, + nargs="+", + default=None, + help="Structures to benchmark (resolved from tests/files/{pdb,mtz}/). " + "Default: all matching test-data pairs.", + ) + parser.add_argument( + "--timeout", + type=int, + default=600, + help="Per-worker subprocess timeout in seconds (default: 600).", + ) parser.add_argument( "--output_dir", type=str, default=None, help="Directory for results. Default: results_/", ) + parser.add_argument( + "--gpu_only", + action="store_true", + help="Only run the GPU point per structure (skip the CPU thread sweep). " + "For running the GPU measurement on a GPU node while the CPU " + "thread-scaling runs as separate exclusive-node jobs.", + ) + parser.add_argument( + "--no_summary", + action="store_true", + help="Skip writing summary.csv and auto-plotting at the end (for shard " + "jobs that only emit per-structure JSONs; a later --aggregate job " + "builds the combined summary).", + ) + parser.add_argument( + "--aggregate", + action="store_true", + help="Do not benchmark: load every per-structure JSON in --output_dir, " + "rebuild the combined summary.csv, then exit.", + ) args = parser.parse_args() + if args.aggregate: + if not args.output_dir: + print("--aggregate requires --output_dir") + sys.exit(1) + _aggregate_and_write(Path(args.output_dir)) + return + + structures = args.structures if args.structures else discover_structures() + # Determine thread counts to test if args.threads: thread_counts = sorted(set(args.threads)) @@ -315,6 +396,7 @@ def main(): print("=" * 78) print("TorchRef vs cctbx — Fcalc Benchmark") print("=" * 78) + print(f"Structures: {structures}") print(f"CPU threads: {thread_counts}") print(f"GPU: {'yes' if has_gpu else 'no'}") print(f"Iterations: {args.n_iterations} (+ {args.n_warmup} warmup)") @@ -323,58 +405,62 @@ def main(): print("=" * 78) print() - total_runs = len(thread_counts) + (1 if has_gpu else 0) - - # --- CPU thread scaling --- - cpu_results = [] - for i, n_threads in enumerate(thread_counts): - progress = f"[{i + 1}/{total_runs}]" - print(f"{progress} CPU with {n_threads} thread(s)...", flush=True) - - output_file = output_dir / f"threads_{n_threads:02d}.json" - result = run_worker(n_threads, args.n_iterations, args.n_warmup, output_file) - - if result: - cpu_results.append(result) - print() - - # --- GPU benchmark --- - gpu_result = None - if has_gpu: - progress = f"[{total_runs}/{total_runs}]" - print(f"{progress} GPU benchmark...", flush=True) - - output_file = output_dir / "gpu.json" - gpu_result = run_worker( - 1, args.n_iterations, args.n_warmup, output_file, device="cuda" - ) - print() - - if not cpu_results and not gpu_result: + runs_per_struct = len(thread_counts) + (1 if has_gpu else 0) + total_runs = runs_per_struct * len(structures) + run_idx = 0 + + cpu_results = [] # flat across structures + gpu_results = [] # flat across structures + + for structure in structures: + print(f"### Structure {structure} ###", flush=True) + + # --- CPU thread scaling --- + for n_threads in ([] if args.gpu_only else thread_counts): + run_idx += 1 + print(f"[{run_idx}/{total_runs}] {structure} CPU with " + f"{n_threads} thread(s)...", flush=True) + output_file = output_dir / f"{structure}_threads_{n_threads:02d}.json" + result = run_worker(n_threads, args.n_iterations, args.n_warmup, + output_file, structure=structure, + timeout=args.timeout) + if result: + cpu_results.append(result) + print() + + # --- GPU benchmark --- + if has_gpu: + run_idx += 1 + print(f"[{run_idx}/{total_runs}] {structure} GPU benchmark...", + flush=True) + output_file = output_dir / f"{structure}_gpu.json" + gpu_result = run_worker(1, args.n_iterations, args.n_warmup, + output_file, structure=structure, + device="cuda", timeout=args.timeout) + if gpu_result: + gpu_results.append(gpu_result) + print() + + if not cpu_results and not gpu_results: print("No successful runs. Exiting.") sys.exit(1) + if args.no_summary: + print(f"\n--no_summary: wrote per-structure JSONs to {output_dir} " + f"(run --aggregate to build summary.csv).") + return + # Write combined summary csv_path = output_dir / "summary.csv" - write_summary_csv(cpu_results, gpu_result, csv_path) + write_summary_csv(cpu_results, gpu_results, csv_path) - # Find baselines for display - tr_baseline = cc_baseline = fb_baseline = fg_baseline = bo_baseline = None - for r in cpu_results: - if r["n_threads"] == 1: - tr_baseline = r["torchref"]["mean_time"] - fg_baseline = r["torchref_fwd_graph"]["mean_time"] - bo_baseline = r["torchref_bwd_only"]["mean_time"] - fb_baseline = r["torchref_fwd_bwd"]["mean_time"] - cc_baseline = r["cctbx"]["mean_time"] - break - - # Print summary table + # Print a compact per-structure summary table + baselines = _structure_baselines(cpu_results) print("=" * 120) print("Summary") print("=" * 120) header = ( - f"{'Device':>10s} {'Threads':>8s} | " + f"{'Structure':>10s} {'Device':>8s} {'Thr':>5s} | " f"{'Fwd':>10s} {'Sp':>6s} | " f"{'Fwd(graph)':>10s} {'Sp':>6s} | " f"{'Bwd':>10s} {'Sp':>6s} | " @@ -384,44 +470,29 @@ def main(): print(header) print("-" * len(header)) - for r in sorted(cpu_results, key=lambda x: x["n_threads"]): - tr = r["torchref"] - fg = r["torchref_fwd_graph"] - bo = r["torchref_bwd_only"] - fb = r["torchref_fwd_bwd"] - cc = r["cctbx"] - nt = r["n_threads"] - tr_sp = tr_baseline / tr["mean_time"] if tr_baseline else float("nan") - fg_sp = fg_baseline / fg["mean_time"] if fg_baseline else float("nan") - bo_sp = bo_baseline / bo["mean_time"] if bo_baseline else float("nan") - fb_sp = fb_baseline / fb["mean_time"] if fb_baseline else float("nan") - cc_sp = cc_baseline / cc["mean_time"] if cc_baseline else float("nan") - print( - f"{'CPU':>10s} {nt:>8d} | " - f"{tr['mean_time']:>9.4f}s {tr_sp:>5.1f}x | " - f"{fg['mean_time']:>9.4f}s {fg_sp:>5.1f}x | " - f"{bo['mean_time']:>9.4f}s {bo_sp:>5.1f}x | " - f"{fb['mean_time']:>9.4f}s {fb_sp:>5.1f}x | " - f"{cc['mean_time']:>9.4f}s {cc_sp:>5.1f}x" - ) - - if gpu_result: - tr = gpu_result["torchref"] - fg = gpu_result["torchref_fwd_graph"] - bo = gpu_result["torchref_bwd_only"] - fb = gpu_result["torchref_fwd_bwd"] - gpu_name = gpu_result.get("gpu_name", "GPU") - tr_sp = tr_baseline / tr["mean_time"] if tr_baseline else float("nan") - fg_sp = fg_baseline / fg["mean_time"] if fg_baseline else float("nan") - bo_sp = bo_baseline / bo["mean_time"] if bo_baseline else float("nan") - fb_sp = fb_baseline / fb["mean_time"] if fb_baseline else float("nan") + def _sp(r, prefix, struct): + base = baselines.get(struct, {}).get(prefix) + return base / r[prefix]["mean_time"] if base else float("nan") + + all_rows = sorted(cpu_results, key=lambda x: (x["structure"], x["n_threads"])) + all_rows += sorted(gpu_results, key=lambda x: x["structure"]) + for r in all_rows: + s = r["structure"] + is_gpu = r["device"] != "cpu" + dev = r.get("gpu_name", "GPU") if is_gpu else "CPU" + nt = "-" if is_gpu else str(r["n_threads"]) + tr, fg = r["torchref"], r["torchref_fwd_graph"] + bo, fb = r["torchref_bwd_only"], r["torchref_fwd_bwd"] + cc = r.get("cctbx") + cc_str = (f"{cc['mean_time']:>9.4f}s {_sp(r, 'cctbx', s):>5.1f}x" + if cc else f"{'':>10s} {'':>6s}") print( - f"{gpu_name:>10s} {'':>8s} | " - f"{tr['mean_time']:>9.4f}s {tr_sp:>5.1f}x | " - f"{fg['mean_time']:>9.4f}s {fg_sp:>5.1f}x | " - f"{bo['mean_time']:>9.4f}s {bo_sp:>5.1f}x | " - f"{fb['mean_time']:>9.4f}s {fb_sp:>5.1f}x | " - f"{'':>10s} {'':>6s}" + f"{s:>10s} {dev:>8s} {nt:>5s} | " + f"{tr['mean_time']:>9.4f}s {_sp(r, 'torchref', s):>5.1f}x | " + f"{fg['mean_time']:>9.4f}s {_sp(r, 'torchref_fwd_graph', s):>5.1f}x | " + f"{bo['mean_time']:>9.4f}s {_sp(r, 'torchref_bwd_only', s):>5.1f}x | " + f"{fb['mean_time']:>9.4f}s {_sp(r, 'torchref_fwd_bwd', s):>5.1f}x | " + f"{cc_str}" ) print() @@ -433,7 +504,7 @@ def main(): try: subprocess.run( [PYTHON, str(PLOT_SCRIPT), "--results-dir", str(output_dir)], - timeout=60, + timeout=120, ) except Exception as e: print(f"Plot generation failed: {e}") diff --git a/paper/figure3_performance/fcalc_benchmark/benchmark_worker.py b/paper/figure3_performance/fcalc_benchmark/benchmark_worker.py index 960f7cf..7e7474d 100644 --- a/paper/figure3_performance/fcalc_benchmark/benchmark_worker.py +++ b/paper/figure3_performance/fcalc_benchmark/benchmark_worker.py @@ -16,6 +16,7 @@ import sys import time import warnings +from pathlib import Path # Verify TORCHREF_NUM_THREADS is set before any imports n_threads = int(os.environ.get("TORCHREF_NUM_THREADS", 1)) @@ -23,9 +24,35 @@ import torch from torchref import ModelFT, ReflectionData -DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data") -MTZ_FILE = os.path.join(DATA_DIR, "1DAW.mtz") -PDB_FILE = os.path.join(DATA_DIR, "1DAW.pdb") + +def _find_repo_root() -> Path: + """Walk up from this file to find the repo root (marker: pyproject.toml). + + `__file__` can be relative or staged under sbatch, so we anchor on a + stable marker. Override with TORCHREF_REPO_ROOT if needed. + """ + env_root = os.environ.get("TORCHREF_REPO_ROOT") + if env_root: + return Path(env_root).resolve() + p = Path(os.path.abspath(__file__)).parent + for ancestor in [p, *p.parents]: + if (ancestor / "pyproject.toml").is_file(): + return ancestor + raise RuntimeError( + "Could not locate repo root (no pyproject.toml found walking up " + f"from {p}). Set TORCHREF_REPO_ROOT explicitly." + ) + + +def _resolve_files(structure: str) -> tuple[str, str]: + """Resolve (pdb, mtz) paths for a structure from the test-data suite.""" + files_dir = _find_repo_root() / "tests" / "files" + pdb = files_dir / "pdb" / f"{structure}.pdb" + mtz = files_dir / "mtz" / f"{structure}.mtz" + for path in (pdb, mtz): + if not path.exists(): + raise FileNotFoundError(path) + return str(pdb), str(mtz) def _time_iterations(func, n_iterations: int) -> list[float]: @@ -64,16 +91,19 @@ def _summarize_times(times: list[float]) -> dict: } -def run_benchmark(n_iterations: int, n_warmup: int, device_str: str = "cpu") -> dict: +def run_benchmark(n_iterations: int, n_warmup: int, device_str: str = "cpu", + structure: str = "1DAW") -> dict: """Run structure factor calculation benchmark and return timing results.""" device = torch.device(device_str) is_gpu = device.type == "cuda" timer = _time_iterations_gpu if is_gpu else _time_iterations + pdb_file, mtz_file = _resolve_files(structure) + # Load data - data = ReflectionData(device=device).load_mtz(MTZ_FILE) + data = ReflectionData(device=device).load_mtz(mtz_file) d_min = data.d_min - M = ModelFT(max_res=d_min, device=device, radius_angstrom=3.0).load_pdb(PDB_FILE) + M = ModelFT(max_res=d_min, device=device).load_pdb(pdb_file) hkl, _, _, _ = data() n_atoms = M.xyz().shape[0] @@ -148,6 +178,7 @@ def _forward_backward(): result = { "device": device_str, + "structure": structure, "n_threads": n_threads, "n_iterations": n_iterations, "n_warmup": n_warmup, @@ -168,7 +199,7 @@ def _forward_backward(): if not is_gpu: from iotbx import pdb as iotbx_pdb - pdb_input = iotbx_pdb.input(file_name=PDB_FILE) + pdb_input = iotbx_pdb.input(file_name=pdb_file) xray_structure = pdb_input.xray_structure_simple() # Warmup @@ -196,6 +227,11 @@ def main(): "--device", type=str, default="cpu", choices=["cpu", "cuda"], help="Device to benchmark on (default: cpu)", ) + parser.add_argument( + "--structure", type=str, default="1DAW", + help="Structure to benchmark; resolved from tests/files/{pdb,mtz}/ " + "(default: 1DAW)", + ) parser.add_argument("--output", type=str, required=True, help="Output JSON file") args = parser.parse_args() @@ -207,7 +243,9 @@ def main(): old_stdout = sys.stdout sys.stdout = io.StringIO() try: - results = run_benchmark(args.n_iterations, args.n_warmup, args.device) + results = run_benchmark( + args.n_iterations, args.n_warmup, args.device, args.structure + ) finally: sys.stdout = old_stdout @@ -221,7 +259,7 @@ def main(): fb = results["torchref_fwd_bwd"] device_label = results.get("gpu_name", f"CPU x{results['n_threads']}") msg = ( - f" {device_label} " + f" [{results['structure']}] {device_label} " f"fwd: {tr['mean_time']:.4f}s " f"fwd(graph): {fg['mean_time']:.4f}s " f"bwd: {bo['mean_time']:.4f}s " diff --git a/paper/figure3_performance/fcalc_benchmark/submit_fig3a.sbatch b/paper/figure3_performance/fcalc_benchmark/submit_fig3a.sbatch new file mode 100644 index 0000000..6bea566 --- /dev/null +++ b/paper/figure3_performance/fcalc_benchmark/submit_fig3a.sbatch @@ -0,0 +1,27 @@ +#!/bin/bash +#SBATCH --job-name=fig3a_fcalc +#SBATCH --partition=gpu +#SBATCH --time=08:00:00 +#SBATCH --gres=gpu:nvidia_a100-pcie-40gb:1 +#SBATCH --cpus-per-task=16 +#SBATCH --mem=64G +#SBATCH --ntasks=1 +#SBATCH --output=%x_%j.out + +set -euo pipefail + +REPO_ROOT="/das/work/units/LBR-FEL/p17490/Peter/Library/work_trees_torchref/dev" +BENCH_DIR="${REPO_ROOT}/paper/figure3_performance/fcalc_benchmark" +DATA_DIR="${REPO_ROOT}/paper/figure3_performance/data/fcalc" +PYTHON="${REPO_ROOT}/.dev/bin/python" + +cd "${BENCH_DIR}" + +echo "=== Fig3a fcalc thread scaling ===" +echo "Host: $(hostname)" +nvidia-smi | head -20 || true +echo "Output dir: ${DATA_DIR}" +echo + +# All 10 test structures (auto-discovered from tests/files/{pdb,mtz}/). +"${PYTHON}" benchmark_thread_scaling.py --output_dir "${DATA_DIR}" --timeout 1200 diff --git a/paper/figure3_performance/fcalc_benchmark/submit_fig3a_nodes.sh b/paper/figure3_performance/fcalc_benchmark/submit_fig3a_nodes.sh new file mode 100644 index 0000000..0dcedcb --- /dev/null +++ b/paper/figure3_performance/fcalc_benchmark/submit_fig3a_nodes.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# Fig3a (fcalc) benchmark — one EXCLUSIVE node per structure. +# +# Why: this is a PERFORMANCE benchmark, so co-locating CPU-heavy jobs on a node +# corrupts the timing (shared memory bandwidth / L3). Each structure's CPU +# thread-scaling therefore runs as its own job on its OWN exclusive node (clean +# timing + full parallelism). The single GPU point per structure is CUDA-event +# timed (contention-insensitive), so all structures' GPU points run in one GPU +# job. A dependent job aggregates every per-structure JSON into summary.csv and +# renders the figure. +set -euo pipefail + +REPO="/das/work/units/LBR-FEL/p17490/Peter/Library/work_trees_torchref/dev" +BENCH="${REPO}/paper/figure3_performance/fcalc_benchmark" +PY="${REPO}/.dev/bin/python" +TS="$(date +%Y%m%d_%H%M%S)" +OUT="${REPO}/paper/figure3_performance/data/fcalc/results_${TS}" +THREADS="1 2 4 8 16 32" +mkdir -p "${OUT}" +cd "${BENCH}" + +"${PY}" -c "import benchmark_thread_scaling as b; print('\n'.join(b.discover_structures()))" \ + > "${OUT}/structures.txt" +N=$(wc -l < "${OUT}/structures.txt"); ARR="0-$((N - 1))" +echo "Structures (${N}): $(tr '\n' ' ' < "${OUT}/structures.txt")" +echo "Output: ${OUT}" + +# --- CPU thread-scaling: one exclusive node per structure --- +CPU=$(sbatch --parsable --array="${ARR}" --job-name=fig3a_cpu \ + --partition=day --exclusive --time=04:00:00 \ + --output="${BENCH}/fig3a_cpu_%A_%a.out" \ + --wrap "set -e; STRUCT=\$(sed -n \"\$((SLURM_ARRAY_TASK_ID+1))p\" ${OUT}/structures.txt); \ +echo \"node \$(hostname) structure \${STRUCT}\"; \ +cd ${BENCH} && ${PY} benchmark_thread_scaling.py --structures \${STRUCT} --no_gpu \ +--threads ${THREADS} --n_iterations 10 --n_warmup 3 --no_summary \ +--output_dir ${OUT} --timeout 1800") +echo "CPU array: ${CPU} [${ARR}] (exclusive nodes)" + +# --- GPU point: all structures, one A100 job --- +GPU=$(sbatch --parsable --job-name=fig3a_gpu \ + --partition=gpu --gres=gpu:nvidia_a100-pcie-40gb:1 --cpus-per-task=8 --mem=32G \ + --time=02:00:00 --output="${BENCH}/fig3a_gpu_%j.out" \ + --wrap "cd ${BENCH} && ${PY} benchmark_thread_scaling.py --gpu_only \ +--n_iterations 10 --n_warmup 3 --no_summary --output_dir ${OUT} --timeout 1800") +echo "GPU job: ${GPU}" + +# --- Aggregate once everything finishes (build summary.csv only) --- +# NOTE: the Figure 3a speed panel needs BOTH this fcalc summary AND the +# SFcalculator-GPU data from SF_calc_comparison, so the figure is NOT rendered +# here. After this run and the SF run both finish, render it with: +# plot_figure3a.py --fcalc-dir ${OUT} --sf-dir +PLOT=$(sbatch --parsable --job-name=fig3a_agg --partition=hour --time=00:20:00 \ + --mem=8G --dependency="afterany:${CPU}:${GPU}" \ + --output="${BENCH}/fig3a_agg_%j.out" \ + --wrap "cd ${BENCH} && ${PY} benchmark_thread_scaling.py --aggregate --output_dir ${OUT}") +echo "Aggregate job: ${PLOT} (after ${CPU}, ${GPU})" +echo +echo "Watch: squeue -u \$USER | summary: ${OUT}/summary.csv" +echo "Then render Fig3a: ${PY} plot_figure3a.py --fcalc-dir ${OUT} --sf-dir " diff --git a/paper/figure3_performance/refinement_cycle_benchmark/benchmark_thread_scaling.py b/paper/figure3_performance/refinement_cycle_benchmark/benchmark_thread_scaling.py index df80870..56b539e 100644 --- a/paper/figure3_performance/refinement_cycle_benchmark/benchmark_thread_scaling.py +++ b/paper/figure3_performance/refinement_cycle_benchmark/benchmark_thread_scaling.py @@ -88,12 +88,26 @@ def get_default_max_threads() -> int: return os.cpu_count() or 16 +def discover_structures() -> list[str]: + """All test structures with a matching tests/files/{pdb,mtz}/{ID} pair. + + Mirrors tests/conftest.py::all_structure_pairs — intersection of PDB and + MTZ stems. Naturally excludes 1AK5 (pdb is 1AK5_with_H) and 7L84 (no mtz). + """ + files_dir = REPO_ROOT / "tests" / "files" + pdb_ids = {p.stem for p in (files_dir / "pdb").glob("*.pdb")} + mtz_ids = {p.stem for p in (files_dir / "mtz").glob("*.mtz")} + return sorted(pdb_ids & mtz_ids) + + def run_worker( n_threads: int, n_iterations: int, n_warmup: int, output_file: Path, + structure: str = "1DAW", device: str = "cpu", + timeout: int = 900, ) -> dict | None: """Launch a benchmark worker subprocess with the given thread count.""" env = os.environ.copy() @@ -108,6 +122,7 @@ def run_worker( "--n_iterations", str(n_iterations), "--n_warmup", str(n_warmup), "--device", device, + "--structure", structure, "--output", str(output_file), ] @@ -118,7 +133,7 @@ def run_worker( stdout=subprocess.PIPE, stderr=None, # pass stderr through to terminal text=True, - timeout=900, # 15 min timeout (refinement init is slower) + timeout=timeout, ) if result.stdout.strip(): print(result.stdout.strip()) @@ -130,7 +145,7 @@ def run_worker( return json.load(f) except subprocess.TimeoutExpired: - print(f" TIMEOUT: exceeded 900s") + print(f" TIMEOUT: exceeded {timeout}s") return None except Exception as e: print(f" EXCEPTION: {e}") @@ -155,66 +170,106 @@ def _group_per_target(per_target: dict, mode: str = "fwd_bwd") -> dict: return groups -def write_summary_csv( - cpu_results: list[dict], gpu_result: dict | None, output_path: Path -): - """Write combined results to a CSV file.""" - fieldnames = [ - "device", "n_threads", - "agg_fwd_no_grad_mean", "agg_fwd_no_grad_min", "agg_fwd_no_grad_max", - "agg_fwd_no_grad_speedup", - "agg_fwd_graph_mean", "agg_fwd_graph_min", "agg_fwd_graph_max", - "agg_fwd_graph_speedup", - "agg_bwd_only_mean", "agg_bwd_only_min", "agg_bwd_only_max", - "agg_bwd_only_speedup", - "agg_fwd_bwd_mean", "agg_fwd_bwd_min", "agg_fwd_bwd_max", - "agg_fwd_bwd_speedup", - "target_xray_mean", "target_geometry_total_mean", "target_adp_total_mean", - "n_iterations", "n_atoms", "n_reflections", "d_min", - ] - - # Find single-thread baselines +FIELDNAMES = [ + "structure", "device", "n_threads", + "agg_fwd_no_grad_mean", "agg_fwd_no_grad_min", "agg_fwd_no_grad_max", + "agg_fwd_no_grad_speedup", + "agg_fwd_graph_mean", "agg_fwd_graph_min", "agg_fwd_graph_max", + "agg_fwd_graph_speedup", + "agg_bwd_only_mean", "agg_bwd_only_min", "agg_bwd_only_max", + "agg_bwd_only_speedup", + "agg_fwd_bwd_mean", "agg_fwd_bwd_min", "agg_fwd_bwd_max", + "agg_fwd_bwd_speedup", + "target_xray_mean", "target_geometry_total_mean", "target_adp_total_mean", + "n_iterations", "n_atoms", "n_reflections", "d_min", +] + +_AGG_KEYS = [ + ("agg_fwd_no_grad", "aggregate_fwd_no_grad"), + ("agg_fwd_graph", "aggregate_fwd_graph"), + ("agg_bwd_only", "aggregate_bwd_only"), + ("agg_fwd_bwd", "aggregate_fwd_bwd"), +] + + +def _structure_baselines(cpu_results: list[dict]) -> dict: + """Per-structure 1-thread baselines: {structure: {full_key: mean_time}}.""" baselines = {} for r in cpu_results: if r["n_threads"] == 1: - for key in ["aggregate_fwd_no_grad", "aggregate_fwd_graph", - "aggregate_bwd_only", "aggregate_fwd_bwd"]: - baselines[key] = r[key]["mean_time"] - break + baselines[r["structure"]] = { + full: r[full]["mean_time"] for _, full in _AGG_KEYS + } + return baselines + + +def _aggregate_and_write(output_dir: Path): + """Rebuild the combined summary.csv from every per-structure JSON in output_dir. + + Gathers the flat ``{structure}_threads_NN.json`` / ``{structure}_gpu.json`` + files written by per-structure / gpu-only shard jobs (run on separate nodes). + """ + cpu_results, gpu_results = [], [] + for p in sorted(output_dir.glob("*_threads_*.json")): + try: + with open(p) as f: + cpu_results.append(json.load(f)) + except Exception as e: + print(f" skip {p.name}: {e}") + for p in sorted(output_dir.glob("*_gpu.json")): + try: + with open(p) as f: + gpu_results.append(json.load(f)) + except Exception as e: + print(f" skip {p.name}: {e}") + csv_path = output_dir / "summary.csv" + write_summary_csv(cpu_results, gpu_results, csv_path) + n_struct = len({r["structure"] for r in cpu_results + gpu_results}) + print(f"Aggregated {len(cpu_results)} CPU + {len(gpu_results)} GPU JSONs " + f"({n_struct} structures) -> {csv_path}") - def _speedup(key, mean_time): - base = baselines.get(key) + +def write_summary_csv( + cpu_results: list[dict], gpu_results: list[dict], output_path: Path +): + """Write combined multi-structure results to a CSV file. + + `cpu_results` / `gpu_results` are flat lists across all structures; each + dict carries its own "structure". Speedups are computed per structure + against that structure's single-thread CPU run. + """ + baselines = _structure_baselines(cpu_results) + + def _speedup(structure, key, mean_time): + base = baselines.get(structure, {}).get(key) if base and mean_time > 0: return base / mean_time return float("nan") rows = [] - all_results = sorted(cpu_results, key=lambda x: x["n_threads"]) - if gpu_result: - all_results.append(gpu_result) + all_results = sorted(cpu_results, key=lambda x: (x["structure"], x["n_threads"])) + all_results += sorted(gpu_results, key=lambda x: x["structure"]) for r in all_results: is_gpu = r["device"] != "cpu" nt = 0 if is_gpu else r["n_threads"] - dev = "gpu" if is_gpu else "cpu" + structure = r["structure"] per_target_groups = _group_per_target(r.get("per_target", {})) row = { - "device": dev, + "structure": structure, + "device": "gpu" if is_gpu else "cpu", "n_threads": nt, } - for short, full in [ - ("agg_fwd_no_grad", "aggregate_fwd_no_grad"), - ("agg_fwd_graph", "aggregate_fwd_graph"), - ("agg_bwd_only", "aggregate_bwd_only"), - ("agg_fwd_bwd", "aggregate_fwd_bwd"), - ]: + for short, full in _AGG_KEYS: stats = r[full] row[f"{short}_mean"] = f"{stats['mean_time']:.6f}" row[f"{short}_min"] = f"{stats['min_time']:.6f}" row[f"{short}_max"] = f"{stats['max_time']:.6f}" - row[f"{short}_speedup"] = f"{_speedup(full, stats['mean_time']):.3f}" + row[f"{short}_speedup"] = ( + f"{_speedup(structure, full, stats['mean_time']):.3f}" + ) row["target_xray_mean"] = f"{per_target_groups.get('xray', 0):.6f}" row["target_geometry_total_mean"] = f"{per_target_groups.get('geometry', 0):.6f}" @@ -226,7 +281,7 @@ def _speedup(key, mean_time): rows.append(row) with open(output_path, "w", newline="") as f: - writer = csv.DictWriter(f, fieldnames=fieldnames) + writer = csv.DictWriter(f, fieldnames=FIELDNAMES) writer.writeheader() writer.writerows(rows) @@ -260,13 +315,46 @@ def main(): "--no_gpu", action="store_true", help="Skip GPU benchmark even if a GPU is available.", ) + parser.add_argument( + "--structures", type=str, nargs="+", default=None, + help="Structures to benchmark (resolved from tests/files/{pdb,mtz}/). " + "Default: all matching test-data pairs.", + ) + parser.add_argument( + "--timeout", type=int, default=900, + help="Per-worker subprocess timeout in seconds (default: 900). " + "Large structures + per-target breakdown can be slow.", + ) parser.add_argument( "--output_dir", type=str, default=None, help="Directory for results. Default: results_/", ) + parser.add_argument( + "--gpu_only", action="store_true", + help="Only run the GPU point per structure (skip the CPU thread sweep).", + ) + parser.add_argument( + "--no_summary", action="store_true", + help="Skip writing summary.csv and auto-plotting (shard jobs; a later " + "--aggregate job builds the combined summary).", + ) + parser.add_argument( + "--aggregate", action="store_true", + help="Do not benchmark: load every per-structure JSON in --output_dir, " + "rebuild the combined summary.csv, then exit.", + ) args = parser.parse_args() + if args.aggregate: + if not args.output_dir: + print("--aggregate requires --output_dir") + sys.exit(1) + _aggregate_and_write(Path(args.output_dir).resolve()) + return + + structures = args.structures if args.structures else discover_structures() + # Determine thread counts if args.threads: thread_counts = sorted(set(args.threads)) @@ -300,6 +388,7 @@ def main(): print("=" * 78) print("TorchRef — Refinement Cycle Benchmark") print("=" * 78) + print(f"Structures: {structures}") print(f"CPU threads: {thread_counts}") print(f"GPU: {'yes' if has_gpu else 'no'}") print(f"Iterations: {args.n_iterations} (+ {args.n_warmup} warmup)") @@ -308,57 +397,62 @@ def main(): print("=" * 78) print() - total_runs = len(thread_counts) + (1 if has_gpu else 0) - - # --- CPU thread scaling --- - cpu_results = [] - for i, nt in enumerate(thread_counts): - progress = f"[{i + 1}/{total_runs}]" - print(f"{progress} CPU with {nt} thread(s)...", flush=True) - - output_file = output_dir / f"threads_{nt:02d}.json" - result = run_worker(nt, args.n_iterations, args.n_warmup, output_file) - - if result: - cpu_results.append(result) - print() - - # --- GPU benchmark --- - gpu_result = None - if has_gpu: - progress = f"[{total_runs}/{total_runs}]" - print(f"{progress} GPU benchmark...", flush=True) - - output_file = output_dir / "gpu.json" - gpu_result = run_worker( - 1, args.n_iterations, args.n_warmup, output_file, device="cuda" - ) - print() - - if not cpu_results and not gpu_result: + runs_per_struct = len(thread_counts) + (1 if has_gpu else 0) + total_runs = runs_per_struct * len(structures) + run_idx = 0 + + cpu_results = [] # flat across structures + gpu_results = [] # flat across structures + + for structure in structures: + print(f"### Structure {structure} ###", flush=True) + + # --- CPU thread scaling --- + for nt in ([] if args.gpu_only else thread_counts): + run_idx += 1 + print(f"[{run_idx}/{total_runs}] {structure} CPU with " + f"{nt} thread(s)...", flush=True) + output_file = output_dir / f"{structure}_threads_{nt:02d}.json" + result = run_worker(nt, args.n_iterations, args.n_warmup, + output_file, structure=structure, + timeout=args.timeout) + if result: + cpu_results.append(result) + print() + + # --- GPU benchmark --- + if has_gpu: + run_idx += 1 + print(f"[{run_idx}/{total_runs}] {structure} GPU benchmark...", + flush=True) + output_file = output_dir / f"{structure}_gpu.json" + gpu_result = run_worker(1, args.n_iterations, args.n_warmup, + output_file, structure=structure, + device="cuda", timeout=args.timeout) + if gpu_result: + gpu_results.append(gpu_result) + print() + + if not cpu_results and not gpu_results: print("No successful runs. Exiting.") sys.exit(1) + if args.no_summary: + print(f"\n--no_summary: wrote per-structure JSONs to {output_dir} " + f"(run --aggregate to build summary.csv).") + return + # Write combined summary csv_path = output_dir / "summary.csv" - write_summary_csv(cpu_results, gpu_result, csv_path) - - # Find baselines for display - baselines = {} - for r in cpu_results: - if r["n_threads"] == 1: - baselines["fwd"] = r["aggregate_fwd_no_grad"]["mean_time"] - baselines["fg"] = r["aggregate_fwd_graph"]["mean_time"] - baselines["bwd"] = r["aggregate_bwd_only"]["mean_time"] - baselines["fb"] = r["aggregate_fwd_bwd"]["mean_time"] - break + write_summary_csv(cpu_results, gpu_results, csv_path) - # Print summary table + # Print a compact per-structure summary table + baselines = _structure_baselines(cpu_results) print("=" * 110) print("Summary") print("=" * 110) header = ( - f"{'Device':>10s} {'Threads':>8s} | " + f"{'Structure':>10s} {'Device':>8s} {'Thr':>5s} | " f"{'Fwd':>10s} {'Sp':>6s} | " f"{'Fwd(graph)':>10s} {'Sp':>6s} | " f"{'Bwd':>10s} {'Sp':>6s} | " @@ -367,38 +461,27 @@ def main(): print(header) print("-" * len(header)) - def _sp(key, val): - base = baselines.get(key) - if base and val > 0: - return base / val - return float("nan") - - for r in sorted(cpu_results, key=lambda x: x["n_threads"]): - fwd = r["aggregate_fwd_no_grad"]["mean_time"] - fg = r["aggregate_fwd_graph"]["mean_time"] - bwd = r["aggregate_bwd_only"]["mean_time"] - fb = r["aggregate_fwd_bwd"]["mean_time"] - nt = r["n_threads"] - print( - f"{'CPU':>10s} {nt:>8d} | " - f"{fwd:>9.4f}s {_sp('fwd', fwd):>5.1f}x | " - f"{fg:>9.4f}s {_sp('fg', fg):>5.1f}x | " - f"{bwd:>9.4f}s {_sp('bwd', bwd):>5.1f}x | " - f"{fb:>9.4f}s {_sp('fb', fb):>5.1f}x" - ) + def _sp(r, full): + base = baselines.get(r["structure"], {}).get(full) + val = r[full]["mean_time"] + return base / val if (base and val > 0) else float("nan") - if gpu_result: - fwd = gpu_result["aggregate_fwd_no_grad"]["mean_time"] - fg = gpu_result["aggregate_fwd_graph"]["mean_time"] - bwd = gpu_result["aggregate_bwd_only"]["mean_time"] - fb = gpu_result["aggregate_fwd_bwd"]["mean_time"] - gpu_name = gpu_result.get("gpu_name", "GPU") + all_rows = sorted(cpu_results, key=lambda x: (x["structure"], x["n_threads"])) + all_rows += sorted(gpu_results, key=lambda x: x["structure"]) + for r in all_rows: + is_gpu = r["device"] != "cpu" + dev = r.get("gpu_name", "GPU") if is_gpu else "CPU" + nt = "-" if is_gpu else str(r["n_threads"]) print( - f"{gpu_name:>10s} {'':>8s} | " - f"{fwd:>9.4f}s {_sp('fwd', fwd):>5.1f}x | " - f"{fg:>9.4f}s {_sp('fg', fg):>5.1f}x | " - f"{bwd:>9.4f}s {_sp('bwd', bwd):>5.1f}x | " - f"{fb:>9.4f}s {_sp('fb', fb):>5.1f}x" + f"{r['structure']:>10s} {dev:>8s} {nt:>5s} | " + f"{r['aggregate_fwd_no_grad']['mean_time']:>9.4f}s " + f"{_sp(r, 'aggregate_fwd_no_grad'):>5.1f}x | " + f"{r['aggregate_fwd_graph']['mean_time']:>9.4f}s " + f"{_sp(r, 'aggregate_fwd_graph'):>5.1f}x | " + f"{r['aggregate_bwd_only']['mean_time']:>9.4f}s " + f"{_sp(r, 'aggregate_bwd_only'):>5.1f}x | " + f"{r['aggregate_fwd_bwd']['mean_time']:>9.4f}s " + f"{_sp(r, 'aggregate_fwd_bwd'):>5.1f}x" ) print() @@ -410,7 +493,7 @@ def _sp(key, val): try: subprocess.run( [PYTHON, str(PLOT_SCRIPT), "--results-dir", str(output_dir)], - timeout=60, + timeout=120, ) except Exception as e: print(f"Plot generation failed: {e}") diff --git a/paper/figure3_performance/refinement_cycle_benchmark/benchmark_worker.py b/paper/figure3_performance/refinement_cycle_benchmark/benchmark_worker.py index 6254c26..1c9dedd 100644 --- a/paper/figure3_performance/refinement_cycle_benchmark/benchmark_worker.py +++ b/paper/figure3_performance/refinement_cycle_benchmark/benchmark_worker.py @@ -47,9 +47,15 @@ def _find_repo_root() -> Path: ) -DATA_DIR = _find_repo_root() / "paper" / "figure3_performance" / "data" -MTZ_FILE = str(DATA_DIR / "1DAW.mtz") -PDB_FILE = str(DATA_DIR / "1DAW.pdb") +def _resolve_files(structure: str) -> tuple[str, str]: + """Resolve (pdb, mtz) paths for a structure from the test-data suite.""" + files_dir = _find_repo_root() / "tests" / "files" + pdb = files_dir / "pdb" / f"{structure}.pdb" + mtz = files_dir / "mtz" / f"{structure}.mtz" + for path in (pdb, mtz): + if not path.exists(): + raise FileNotFoundError(path) + return str(pdb), str(mtz) def _time_iterations(func, n_iterations: int) -> list[float]: @@ -88,18 +94,21 @@ def _summarize_times(times: list[float]) -> dict: } -def run_benchmark(n_iterations: int, n_warmup: int, device_str: str = "cpu") -> dict: +def run_benchmark(n_iterations: int, n_warmup: int, device_str: str = "cpu", + structure: str = "1DAW") -> dict: """Run refinement cycle benchmark and return timing results.""" device = torch.device(device_str) is_gpu = device.type == "cuda" timer = _time_iterations_gpu if is_gpu else _time_iterations + pdb_file, mtz_file = _resolve_files(structure) + # ---- SETUP (untimed) ---- # Pass device at construction so all tensors (model, scaler, restraints) # are created on the correct device from the start. refinement = LBFGSRefinement( - data_file=MTZ_FILE, - pdb=PDB_FILE, + data_file=mtz_file, + pdb=pdb_file, device=device, target_mode="ml", ) @@ -235,6 +244,7 @@ def _bwd(t=target): # ---- Assemble results ---- result = { "device": device_str, + "structure": structure, "n_threads": n_threads, "n_iterations": n_iterations, "n_warmup": n_warmup, @@ -269,6 +279,11 @@ def main(): "--device", type=str, default="cpu", choices=["cpu", "cuda"], help="Device to benchmark on (default: cpu)", ) + parser.add_argument( + "--structure", type=str, default="1DAW", + help="Structure to benchmark; resolved from tests/files/{pdb,mtz}/ " + "(default: 1DAW)", + ) parser.add_argument("--output", type=str, required=True, help="Output JSON file") args = parser.parse_args() @@ -281,7 +296,9 @@ def main(): old_stdout = sys.stdout sys.stdout = io.StringIO() try: - results = run_benchmark(args.n_iterations, args.n_warmup, args.device) + results = run_benchmark( + args.n_iterations, args.n_warmup, args.device, args.structure + ) finally: sys.stdout = old_stdout @@ -294,7 +311,7 @@ def main(): bwd = results["aggregate_bwd_only"] fb = results["aggregate_fwd_bwd"] msg = ( - f" {device_label} " + f" [{results['structure']}] {device_label} " f"fwd: {fwd['mean_time']:.4f}s " f"bwd: {bwd['mean_time']:.4f}s " f"fwd+bwd: {fb['mean_time']:.4f}s" diff --git a/paper/figure3_performance/refinement_cycle_benchmark/rerun_ml.log b/paper/figure3_performance/refinement_cycle_benchmark/rerun_ml.log deleted file mode 100644 index dc03a36..0000000 --- a/paper/figure3_performance/refinement_cycle_benchmark/rerun_ml.log +++ /dev/null @@ -1,68 +0,0 @@ -============================================================================== -TorchRef — Refinement Cycle Benchmark -============================================================================== -CPU threads: [1, 2, 4, 8, 16] -GPU: yes -Iterations: 10 (+ 3 warmup) -Output dir: /das/work/units/LBR-FEL/p17490/Peter/Library/work_trees_torchref/dev/paper/figure3_performance/data/refinement_cycle/results_20260618_141803 -Python: /das/work/units/LBR-FEL/p17490/Peter/Library/work_trees_torchref/dev/.dev/bin/python -============================================================================== - -[1/6] CPU with 1 thread(s)... -/das/work/units/LBR-FEL/p17490/Peter/Library/work_trees_torchref/dev/paper/figure3_performance/refinement_cycle_benchmark/benchmark_worker.py:28: UserWarning: TorchRef using user-specified 1 threads from TORCHREF_NUM_THREADS. - from torchref.refinement import LBFGSRefinement -/das/work/units/LBR-FEL/p17490/Peter/Library/work_trees_torchref/dev/paper/figure3_performance/refinement_cycle_benchmark/benchmark_worker.py:112: DeprecationWarning: Calling ReflectionData (data()) is deprecated; use the data.work / data.free / data.validation accessor, or data.get_corrected_data() for the full scaled arrays. - hkl, _, _, _ = refinement.reflection_data() -CPU x1 fwd: 0.1131s bwd: 0.1096s fwd+bwd: 0.2208s | top: xray=168.3ms, geometry/nonbonded=39.4ms, geometry/planarity=4.0ms - -[2/6] CPU with 2 thread(s)... -/das/work/units/LBR-FEL/p17490/Peter/Library/work_trees_torchref/dev/paper/figure3_performance/refinement_cycle_benchmark/benchmark_worker.py:28: UserWarning: TorchRef using user-specified 2 threads from TORCHREF_NUM_THREADS. - from torchref.refinement import LBFGSRefinement -/das/work/units/LBR-FEL/p17490/Peter/Library/work_trees_torchref/dev/paper/figure3_performance/refinement_cycle_benchmark/benchmark_worker.py:112: DeprecationWarning: Calling ReflectionData (data()) is deprecated; use the data.work / data.free / data.validation accessor, or data.get_corrected_data() for the full scaled arrays. - hkl, _, _, _ = refinement.reflection_data() -CPU x2 fwd: 0.0794s bwd: 0.0711s fwd+bwd: 0.1515s | top: xray=112.3ms, geometry/nonbonded=25.9ms, geometry/planarity=4.0ms - -[3/6] CPU with 4 thread(s)... -/das/work/units/LBR-FEL/p17490/Peter/Library/work_trees_torchref/dev/paper/figure3_performance/refinement_cycle_benchmark/benchmark_worker.py:28: UserWarning: TorchRef using user-specified 4 threads from TORCHREF_NUM_THREADS. - from torchref.refinement import LBFGSRefinement -/das/work/units/LBR-FEL/p17490/Peter/Library/work_trees_torchref/dev/paper/figure3_performance/refinement_cycle_benchmark/benchmark_worker.py:112: DeprecationWarning: Calling ReflectionData (data()) is deprecated; use the data.work / data.free / data.validation accessor, or data.get_corrected_data() for the full scaled arrays. - hkl, _, _, _ = refinement.reflection_data() -CPU x4 fwd: 0.0682s bwd: 0.0569s fwd+bwd: 0.1211s | top: xray=81.1ms, geometry/nonbonded=25.5ms, geometry/planarity=4.0ms - -[4/6] CPU with 8 thread(s)... -/das/work/units/LBR-FEL/p17490/Peter/Library/work_trees_torchref/dev/paper/figure3_performance/refinement_cycle_benchmark/benchmark_worker.py:28: UserWarning: TorchRef using user-specified 8 threads from TORCHREF_NUM_THREADS. - from torchref.refinement import LBFGSRefinement -/das/work/units/LBR-FEL/p17490/Peter/Library/work_trees_torchref/dev/paper/figure3_performance/refinement_cycle_benchmark/benchmark_worker.py:112: DeprecationWarning: Calling ReflectionData (data()) is deprecated; use the data.work / data.free / data.validation accessor, or data.get_corrected_data() for the full scaled arrays. - hkl, _, _, _ = refinement.reflection_data() -CPU x8 fwd: 0.0542s bwd: 0.0412s fwd+bwd: 0.0960s | top: xray=66.5ms, geometry/nonbonded=16.8ms, geometry/planarity=4.0ms - -[5/6] CPU with 16 thread(s)... -/das/work/units/LBR-FEL/p17490/Peter/Library/work_trees_torchref/dev/paper/figure3_performance/refinement_cycle_benchmark/benchmark_worker.py:28: UserWarning: TorchRef using user-specified 16 threads from TORCHREF_NUM_THREADS. - from torchref.refinement import LBFGSRefinement -/das/work/units/LBR-FEL/p17490/Peter/Library/work_trees_torchref/dev/paper/figure3_performance/refinement_cycle_benchmark/benchmark_worker.py:112: DeprecationWarning: Calling ReflectionData (data()) is deprecated; use the data.work / data.free / data.validation accessor, or data.get_corrected_data() for the full scaled arrays. - hkl, _, _, _ = refinement.reflection_data() -CPU x16 fwd: 0.0442s bwd: 0.0355s fwd+bwd: 0.0809s | top: xray=55.8ms, geometry/nonbonded=13.7ms, geometry/planarity=3.9ms - -[6/6] GPU benchmark... -/das/work/units/LBR-FEL/p17490/Peter/Library/work_trees_torchref/dev/paper/figure3_performance/refinement_cycle_benchmark/benchmark_worker.py:28: UserWarning: TorchRef using user-specified 1 threads from TORCHREF_NUM_THREADS. - from torchref.refinement import LBFGSRefinement -/das/work/units/LBR-FEL/p17490/Peter/Library/work_trees_torchref/dev/paper/figure3_performance/refinement_cycle_benchmark/benchmark_worker.py:112: DeprecationWarning: Calling ReflectionData (data()) is deprecated; use the data.work / data.free / data.validation accessor, or data.get_corrected_data() for the full scaled arrays. - hkl, _, _, _ = refinement.reflection_data() -Saved: /das/work/units/LBR-FEL/p17490/Peter/Library/work_trees_torchref/dev/paper/figure3_performance/output/figure3b_profiling.png -NVIDIA A100-PCIE-40GB fwd: 0.0087s bwd: 0.0058s fwd+bwd: 0.0153s | top: xray=7.9ms, geometry/nonbonded=2.0ms, geometry/planarity=1.7ms - -============================================================================================================== -Summary -============================================================================================================== - Device Threads | Fwd Sp | Fwd(graph) Sp | Bwd Sp | Fwd+Bwd Sp ---------------------------------------------------------------------------------------------------- - CPU 1 | 0.1131s 1.0x | 0.1172s 1.0x | 0.1096s 1.0x | 0.2208s 1.0x - CPU 2 | 0.0794s 1.4x | 0.0828s 1.4x | 0.0711s 1.5x | 0.1515s 1.5x - CPU 4 | 0.0682s 1.7x | 0.0717s 1.6x | 0.0569s 1.9x | 0.1211s 1.8x - CPU 8 | 0.0542s 2.1x | 0.0569s 2.1x | 0.0412s 2.7x | 0.0960s 2.3x - CPU 16 | 0.0442s 2.6x | 0.0482s 2.4x | 0.0355s 3.1x | 0.0809s 2.7x -NVIDIA A100-PCIE-40GB | 0.0087s 12.9x | 0.0094s 12.5x | 0.0058s 18.8x | 0.0153s 14.4x - -Results saved to: /das/work/units/LBR-FEL/p17490/Peter/Library/work_trees_torchref/dev/paper/figure3_performance/data/refinement_cycle/results_20260618_141803/summary.csv - -Generating plots... diff --git a/paper/figure3_performance/refinement_cycle_benchmark/submit_fig3b.sbatch b/paper/figure3_performance/refinement_cycle_benchmark/submit_fig3b.sbatch new file mode 100644 index 0000000..2cbbc1e --- /dev/null +++ b/paper/figure3_performance/refinement_cycle_benchmark/submit_fig3b.sbatch @@ -0,0 +1,30 @@ +#!/bin/bash +#SBATCH --job-name=fig3b_refcycle +#SBATCH --partition=gpu +#SBATCH --time=24:00:00 +#SBATCH --gres=gpu:nvidia_a100-pcie-40gb:1 +#SBATCH --cpus-per-task=16 +#SBATCH --mem=64G +#SBATCH --ntasks=1 +#SBATCH --output=%x_%j.out + +set -euo pipefail + +REPO_ROOT="/das/work/units/LBR-FEL/p17490/Peter/Library/work_trees_torchref/dev" +BENCH_DIR="${REPO_ROOT}/paper/figure3_performance/refinement_cycle_benchmark" +TS="$(date +%Y%m%d_%H%M%S)" +OUT_DIR="${REPO_ROOT}/paper/figure3_performance/data/refinement_cycle/results_${TS}" +PYTHON="${REPO_ROOT}/.dev/bin/python" + +cd "${BENCH_DIR}" + +echo "=== Fig3b refinement-cycle thread scaling ===" +echo "Host: $(hostname)" +nvidia-smi | head -20 || true +echo "Output dir: ${OUT_DIR}" +echo + +# All 10 test structures (auto-discovered from tests/files/{pdb,mtz}/). The +# per-target breakdown on large structures (5BOV/2DQ6) is the long pole, so a +# generous per-worker timeout is given. +"${PYTHON}" benchmark_thread_scaling.py --output_dir "${OUT_DIR}" --timeout 3600 diff --git a/paper/figure3_performance/refinement_cycle_benchmark/submit_fig3b_nodes.sh b/paper/figure3_performance/refinement_cycle_benchmark/submit_fig3b_nodes.sh new file mode 100644 index 0000000..80d1fa4 --- /dev/null +++ b/paper/figure3_performance/refinement_cycle_benchmark/submit_fig3b_nodes.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Fig3b (refinement-cycle) benchmark — one EXCLUSIVE node per structure. +# +# Same rationale as Fig3a: clean per-structure CPU timing requires a dedicated +# node (no co-tenant memory-bandwidth contention). The refinement-cycle worker +# (loss + gradient over all targets, with a per-target breakdown) is the heavy +# one, so the CPU jobs get more walltime. GPU points run in one A100 job; a +# dependent job aggregates + plots. +set -euo pipefail + +REPO="/das/work/units/LBR-FEL/p17490/Peter/Library/work_trees_torchref/dev" +BENCH="${REPO}/paper/figure3_performance/refinement_cycle_benchmark" +PY="${REPO}/.dev/bin/python" +TS="$(date +%Y%m%d_%H%M%S)" +OUT="${REPO}/paper/figure3_performance/data/refinement_cycle/results_${TS}" +THREADS="1 2 4 8 16 32" +mkdir -p "${OUT}" +cd "${BENCH}" + +"${PY}" -c "import benchmark_thread_scaling as b; print('\n'.join(b.discover_structures()))" \ + > "${OUT}/structures.txt" +N=$(wc -l < "${OUT}/structures.txt"); ARR="0-$((N - 1))" +echo "Structures (${N}): $(tr '\n' ' ' < "${OUT}/structures.txt")" +echo "Output: ${OUT}" + +# --- CPU thread-scaling: one exclusive node per structure (heavy: longer walltime) --- +CPU=$(sbatch --parsable --array="${ARR}" --job-name=fig3b_cpu \ + --partition=day --exclusive --time=12:00:00 \ + --output="${BENCH}/fig3b_cpu_%A_%a.out" \ + --wrap "set -e; STRUCT=\$(sed -n \"\$((SLURM_ARRAY_TASK_ID+1))p\" ${OUT}/structures.txt); \ +echo \"node \$(hostname) structure \${STRUCT}\"; \ +cd ${BENCH} && ${PY} benchmark_thread_scaling.py --structures \${STRUCT} --no_gpu \ +--threads ${THREADS} --n_iterations 10 --n_warmup 3 --no_summary \ +--output_dir ${OUT} --timeout 3600") +echo "CPU array: ${CPU} [${ARR}] (exclusive nodes)" + +# --- GPU point: all structures, one A100 job --- +GPU=$(sbatch --parsable --job-name=fig3b_gpu \ + --partition=gpu --gres=gpu:nvidia_a100-pcie-40gb:1 --cpus-per-task=8 --mem=32G \ + --time=03:00:00 --output="${BENCH}/fig3b_gpu_%j.out" \ + --wrap "cd ${BENCH} && ${PY} benchmark_thread_scaling.py --gpu_only \ +--n_iterations 10 --n_warmup 3 --no_summary --output_dir ${OUT} --timeout 3600") +echo "GPU job: ${GPU}" + +# --- Aggregate + plot once everything finishes --- +PLOT=$(sbatch --parsable --job-name=fig3b_plot --partition=hour --time=00:20:00 \ + --mem=8G --dependency="afterany:${CPU}:${GPU}" \ + --output="${BENCH}/fig3b_plot_%j.out" \ + --wrap "cd ${BENCH} && ${PY} benchmark_thread_scaling.py --aggregate --output_dir ${OUT} \ +&& ${PY} ../plot_figure3b.py --results-dir ${OUT}") +echo "Plot job: ${PLOT} (after ${CPU}, ${GPU})" +echo +echo "Watch: squeue -u \$USER | summary: ${OUT}/summary.csv | figure: ${BENCH%/*}/output/" diff --git a/paper/figure4_difference_refinement/refinement_output/out.log b/paper/figure4_difference_refinement/refinement_output/out.log deleted file mode 100644 index cbe43f0..0000000 --- a/paper/figure4_difference_refinement/refinement_output/out.log +++ /dev/null @@ -1,214 +0,0 @@ -/var/spool/slurmd/job891785/slurm_script:5: UserWarning: TorchRef auto-configured 16 threads. Set TORCHREF_NUM_THREADS to override. - from torchref.cli.collection_difference_refine import main -======================================================================== -TorchRef Collection Difference Refinement -======================================================================== -Dark model: figure4_difference_refinement/data/8QL2_no_altloc.pdb -Light model: figure4_difference_refinement/work_no_altloc.pdb -Dark data: figure4_difference_refinement/data/8QL2-sf.cif -Light data: figure4_difference_refinement/data/7YYZ-light.mtz -Fractions: dark=0.78, light=0.22 (frozen) -Output: figure4_difference_refinement/refinement_output -Device: cuda -Resolution cutoff: 2.20 A -CIF restraints: figure4_difference_refinement/data/IBL_grade.cif -Weight schedule: [1.0] x 10 cycles -LBFGS steps/weight: 2 (max_iter=100) - -Target weights: - similarity: 3.0 - xray/difference: 1.0 - xray/ml: 0.0 -======================================================================== - -Setting up models... -Loaded 16157 atoms -Loaded 8204 atoms -Loading reflection data... -/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/cif_readers.py:695: RuntimeWarning: invalid value encountered in cast - self.data["R-free-flags"] = rfree_characters.to_numpy().astype(np.int32) -Filtering: 57575/124404 reflections in range [inf - 2.2] Å (56814 valid after all masks) -Filtering: 56820/123649 reflections in range [inf - 2.2] Å (56796 valid after all masks) -Added dataset 'dark' (124404 reflections) -Added dataset 'light' (124404 reflections) - R-free flags: dark=2814 free, light=2000 free, agreement=123590/124404 (99.3%) -Setting up joint scaler... -Initialized ScalerBase with 20 bins. -Parametrization built for 7 unique atom types -Parametrization built for 7 unique atom types -Joint initial scale from 2 data-model pairs (20 bins). - Created 2 component solvent models. -/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/.dev/lib/python3.11/site-packages/torch/masked/maskedtensor/core.py:247: UserWarning: It is not recommended to create a MaskedTensor with a tensor that requires_grad. To avoid this, you can use data.detach().clone() - return MaskedTensor(data, mask) -Joint solvent screening: k_sol=0.3596, B_sol=55.0, NLL=53.6408 -Refining scales jointly with LBFGS... -Joint scale refinement complete. rwork: 0.1645, rfree: 0.1838 - Initial R-factor (dark vs dark data): R_work=0.1646 R_free=0.1837 - Initial R-factor (mixed vs light data): R_work=0.1749 R_free=0.1902 - Initial R-factor (dark vs light data): R_work=0.1766 R_free=0.1919 - -Initializing TotalGeometryTarget with component targets... -Initializing TotalADPTarget with component targets... -Building restraints... -Found 109 link definitions -Built 1014 peptide bond restraints -Built 3085 peptide angle restraints -Built 1014 peptide plane restraints - -Building VDW (non-bonded) restraints... - Symmetry expansion: 46 valid (symop, offset) combos - Grid: [12, 15, 13], 1755/2340 cells occupied, max 621 entries/cell - Built 146467 VDW restraints, 4496 symmetry contacts - Hydrogen topology: 7688 riding H atoms - H candidate pairs: 332395 (78907 H-H, 253488 H-heavy, 7378 symmetry) -================================================================================ -Restraints Summary (New Implementation) -================================================================================ -CIF file: ['figure4_difference_refinement/data/IBL_grade.cif'] -Residue types in dictionary: 23 - -INTRA-RESIDUE RESTRAINTS: --------------------------------------------------------------------------------- - Bonds: 7127 - Angles: 7975 - Torsions: 2018 - Planes: 2485 - Chirals: 1234 - -INTER-RESIDUE RESTRAINTS: --------------------------------------------------------------------------------- - Peptide bonds: 1014 - Peptide angles: 3085 - Disulfide bonds: 0 - Disulfide angles: 0 - Disulfide torsions: 0 - -BACKBONE TORSIONS: --------------------------------------------------------------------------------- - Phi: 1014 - Psi: 1014 - Omega: 1014 - Ramachandran: 1010 - -VDW RESTRAINTS: --------------------------------------------------------------------------------- - Non-bonded contacts: 146467 (141971 intra-ASU, 4496 symmetry) -================================================================================ -/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/refinement/loss_state.py:356: UserWarning: LossState.register_target: target 'geometry/bond' returned tensor(s) with no grad_fn — the loss does not track gradients. Register targets while every parameter the loss should depend on has requires_grad=True. The target is still registered; this is only a performance hint, not a correctness issue. - self.register_target( -/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/refinement/loss_state.py:356: UserWarning: LossState.register_target: target 'geometry/angle' returned tensor(s) with no grad_fn — the loss does not track gradients. Register targets while every parameter the loss should depend on has requires_grad=True. The target is still registered; this is only a performance hint, not a correctness issue. - self.register_target( -/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/refinement/loss_state.py:356: UserWarning: LossState.register_target: target 'geometry/torsion' returned tensor(s) with no grad_fn — the loss does not track gradients. Register targets while every parameter the loss should depend on has requires_grad=True. The target is still registered; this is only a performance hint, not a correctness issue. - self.register_target( -/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/refinement/loss_state.py:356: UserWarning: LossState.register_target: target 'geometry/planarity' returned tensor(s) with no grad_fn — the loss does not track gradients. Register targets while every parameter the loss should depend on has requires_grad=True. The target is still registered; this is only a performance hint, not a correctness issue. - self.register_target( -/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/refinement/loss_state.py:356: UserWarning: LossState.register_target: target 'geometry/chiral' returned tensor(s) with no grad_fn — the loss does not track gradients. Register targets while every parameter the loss should depend on has requires_grad=True. The target is still registered; this is only a performance hint, not a correctness issue. - self.register_target( -/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/refinement/loss_state.py:356: UserWarning: LossState.register_target: target 'geometry/nonbonded' returned tensor(s) with no grad_fn — the loss does not track gradients. Register targets while every parameter the loss should depend on has requires_grad=True. The target is still registered; this is only a performance hint, not a correctness issue. - self.register_target( -/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/refinement/loss_state.py:356: UserWarning: LossState.register_target: target 'geometry/ramachandran' returned tensor(s) with no grad_fn — the loss does not track gradients. Register targets while every parameter the loss should depend on has requires_grad=True. The target is still registered; this is only a performance hint, not a correctness issue. - self.register_target( -/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/refinement/loss_state.py:356: UserWarning: LossState.register_target: target 'adp/simu' returned tensor(s) with no grad_fn — the loss does not track gradients. Register targets while every parameter the loss should depend on has requires_grad=True. The target is still registered; this is only a performance hint, not a correctness issue. - self.register_target( -/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/refinement/loss_state.py:356: UserWarning: LossState.register_target: target 'adp/locality' returned tensor(s) with no grad_fn — the loss does not track gradients. Register targets while every parameter the loss should depend on has requires_grad=True. The target is still registered; this is only a performance hint, not a correctness issue. - self.register_target( -/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/cli/collection_difference_refine.py:268: UserWarning: LossState.register_target: target 'similarity' returned tensor(s) with no grad_fn — the loss does not track gradients. Register targets while every parameter the loss should depend on has requires_grad=True. The target is still registered; this is only a performance hint, not a correctness issue. - state.register_target("similarity", similarity_target) -Initial loss breakdown: -LossState Summary: - - -[1/10] cycle 1/10, diff_weight=1.0 - LBFGS step 1/2, loss: 28225.0078 - LBFGS step 2/2, loss: 27944.7734 - Updated all component solvent masks. - R-factor (dark): Rwork=0.1646, Rfree=0.1837 - R-factor (light): Rwork=0.1729, Rfree=0.1911 -[2/10] cycle 2/10, diff_weight=1.0 - LBFGS step 1/2, loss: 28192.0703 - LBFGS step 2/2, loss: 28175.4297 - Updated all component solvent masks. - R-factor (dark): Rwork=0.1646, Rfree=0.1830 - R-factor (light): Rwork=0.1730, Rfree=0.1906 -[3/10] cycle 3/10, diff_weight=1.0 - LBFGS step 1/2, loss: 28171.0781 - LBFGS step 2/2, loss: 28180.8906 - Updated all component solvent masks. - R-factor (dark): Rwork=0.1646, Rfree=0.1840 - R-factor (light): Rwork=0.1730, Rfree=0.1911 -[4/10] cycle 4/10, diff_weight=1.0 - LBFGS step 1/2, loss: 28169.0391 - LBFGS step 2/2, loss: 28190.5742 - Updated all component solvent masks. - R-factor (dark): Rwork=0.1645, Rfree=0.1839 - R-factor (light): Rwork=0.1729, Rfree=0.1909 -[5/10] cycle 5/10, diff_weight=1.0 - LBFGS step 1/2, loss: 28165.3359 - LBFGS step 2/2, loss: 28141.4297 - Updated all component solvent masks. - R-factor (dark): Rwork=0.1647, Rfree=0.1836 - R-factor (light): Rwork=0.1730, Rfree=0.1906 -[6/10] cycle 6/10, diff_weight=1.0 - LBFGS step 1/2, loss: 28144.9102 - LBFGS step 2/2, loss: 27978.5547 - Updated all component solvent masks. - R-factor (dark): Rwork=0.1646, Rfree=0.1846 - R-factor (light): Rwork=0.1730, Rfree=0.1917 -[7/10] cycle 7/10, diff_weight=1.0 - LBFGS step 1/2, loss: 27949.4922 - LBFGS step 2/2, loss: 27955.9219 - Updated all component solvent masks. - R-factor (dark): Rwork=0.1645, Rfree=0.1836 - R-factor (light): Rwork=0.1729, Rfree=0.1910 -[8/10] cycle 8/10, diff_weight=1.0 - LBFGS step 1/2, loss: 27990.1914 - LBFGS step 2/2, loss: 27924.8984 - Updated all component solvent masks. - R-factor (dark): Rwork=0.1646, Rfree=0.1841 - R-factor (light): Rwork=0.1729, Rfree=0.1913 -[9/10] cycle 9/10, diff_weight=1.0 - LBFGS step 1/2, loss: 27947.4336 - LBFGS step 2/2, loss: 27941.8203 - Updated all component solvent masks. - R-factor (dark): Rwork=0.1646, Rfree=0.1839 - R-factor (light): Rwork=0.1730, Rfree=0.1914 -[10/10] cycle 10/10, diff_weight=1.0 - LBFGS step 1/2, loss: 27923.9453 - LBFGS step 2/2, loss: 27961.4883 - Updated all component solvent masks. - R-factor (dark): Rwork=0.1646, Rfree=0.1838 - R-factor (light): Rwork=0.1730, Rfree=0.1907 - -======================================================================== -Refinement complete -======================================================================== - Final R-factor (dark vs dark data): R_work=0.1645 R_free=0.1833 - Final R-factor (mixed vs light data): R_work=0.1729 R_free=0.1908 - Final R-factor (dark vs light data): R_work=0.1765 R_free=0.1918 - Refined fractions: [0.78 0.22] - - Merged deposition CIF written to figure4_difference_refinement/refinement_output/fractions_78_22_merged.cif - Dark SF written to figure4_difference_refinement/refinement_output/fractions_78_22_dark-sf.mtz, figure4_difference_refinement/refinement_output/fractions_78_22_dark-sf.cif - Light SF written to figure4_difference_refinement/refinement_output/fractions_78_22_light-sf.mtz, figure4_difference_refinement/refinement_output/fractions_78_22_light-sf.cif -/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/datasets/reflection_data.py:1651: UserWarning: Found 3622 non-positive F values, masking them out. This really should not happen! - warnings.warn( -/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/.dev/lib/python3.11/site-packages/torch/masked/maskedtensor/core.py:247: UserWarning: It is not recommended to create a MaskedTensor with a tensor that requires_grad. To avoid this, you can use data.detach().clone() - return MaskedTensor(data, mask) -Phase-aware extrapolation rfactors: (0.32576924562454224, 0.3731415271759033) -Classic extrapolation rfactors: (0.3291517496109009, 0.37334439158439636) -Bayes extrapolation rfactors: (0.24041321873664856, 0.28061872720718384) - Bayes: tau^2 = 299.6659, mean w(h) = 0.548 - Results MTZ written to figure4_difference_refinement/refinement_output/fractions_78_22_difference_data.mtz - w_dark=0.780, w_light=0.220 -Output files: - - figure4_difference_refinement/refinement_output/fractions_78_22_dark.pdb - - figure4_difference_refinement/refinement_output/fractions_78_22_light.pdb - - figure4_difference_refinement/refinement_output/fractions_78_22_merged.cif - - figure4_difference_refinement/refinement_output/fractions_78_22_dark-sf.mtz / figure4_difference_refinement/refinement_output/fractions_78_22_dark-sf.cif - - figure4_difference_refinement/refinement_output/fractions_78_22_light-sf.mtz / figure4_difference_refinement/refinement_output/fractions_78_22_light-sf.cif - - figure4_difference_refinement/refinement_output/fractions_78_22_difference_data.mtz - - figure4_difference_refinement/refinement_output/fractions_78_22_summary.json - -Done. - -Timing: 47.1s wall, 43.1s CPU diff --git a/paper/figure4_difference_refinement/validation/output/extrapol_FULL.log b/paper/figure4_difference_refinement/validation/output/extrapol_FULL.log deleted file mode 100644 index 167d42d..0000000 --- a/paper/figure4_difference_refinement/validation/output/extrapol_FULL.log +++ /dev/null @@ -1,81 +0,0 @@ -/var/spool/slurmd/job890410/slurm_script:5: UserWarning: TorchRef auto-configured 16 threads. Set TORCHREF_NUM_THREADS to override. - from torchref.cli.validate_ded import main -====================================================================== -DED Validation: WDFo vs WDFcalc -====================================================================== - ---- Loading reflection data --- - Dark SF: /das/work/p17/p17490/Peter/Library/torchref/paper/figure4_difference_refinement/validation/../data/8QL2-sf.cif - Light SF: /das/work/p17/p17490/Peter/Library/torchref/paper/figure4_difference_refinement/validation/../data/7YYZ-light.mtz -/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/cif_readers.py:695: RuntimeWarning: invalid value encountered in cast - self.data["R-free-flags"] = rfree_characters.to_numpy().astype(np.int32) -Filtering: 57575/124404 reflections in range [inf - 2.2] Å (56814 valid after all masks) -Filtering: 56820/123649 reflections in range [inf - 2.2] Å (56796 valid after all masks) -Added dataset 'dark' (124404 reflections) -Added dataset 'light' (124404 reflections) -Scale parameters after optimization: - dark: log_scale=0.000000 (scale=1.000000) - light: log_scale=-0.035216 (scale=0.965397) -Matched reflections: 56790 -Cell: [74.53 92.58 83.99 90. 96.71 90. ], Spacegroup: P 1 21 1 -Resolution: 2.20 A, Grid: (108, 128, 120) -ASU: 56790, P1: 113563 - ---- Loading models --- - Dark model: 8523 atoms - Light model: 8228 atoms - Fraction: 0.22 - ---- Setting up joint scaler --- -Initialized ScalerBase with 20 bins. -Joint initial scale from 2 data-model pairs (20 bins). - Created 2 component solvent models. -/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/.dev/lib/python3.11/site-packages/torch/masked/maskedtensor/core.py:247: UserWarning: It is not recommended to create a MaskedTensor with a tensor that requires_grad. To avoid this, you can use data.detach().clone() - return MaskedTensor(data, mask) -Joint solvent screening: k_sol=0.3596, B_sol=55.0, NLL=53.5996 -Refining scales jointly with LBFGS... -Joint scale refinement complete. rwork: 0.1648, rfree: 0.1837 - R-factor (dark): R_work=0.1648 R_free=0.1837 - R-factor (mixed): R_work=0.1697 R_free=0.1831 - ---- Computing DED maps and correlations --- - |DFcalc| mean: 3.194 - |WDFcalc| mean: 2.936 - full_cell: CC = 0.3014 (n_vox = 1658880) - selection: CC = 0.3344 (n_vox = 444911) - Bin 0: 9.49- 5.59 A n= 2839 CC=0.1824 - Bin 1: 5.59- 4.60 A n= 2839 CC=0.2400 - Bin 2: 4.60- 4.06 A n= 2839 CC=0.3086 - Bin 3: 4.06- 3.71 A n= 2839 CC=0.3565 - Bin 4: 3.71- 3.46 A n= 2839 CC=0.3292 - Bin 5: 3.46- 3.26 A n= 2839 CC=0.3225 - Bin 6: 3.26- 3.10 A n= 2839 CC=0.2850 - Bin 7: 3.10- 2.97 A n= 2839 CC=0.3165 - Bin 8: 2.97- 2.86 A n= 2839 CC=0.2995 - Bin 9: 2.86- 2.76 A n= 2839 CC=0.3371 - Bin 10: 2.76- 2.68 A n= 2839 CC=0.3166 - Bin 11: 2.68- 2.60 A n= 2839 CC=0.3042 - Bin 12: 2.60- 2.54 A n= 2839 CC=0.3169 - Bin 13: 2.54- 2.48 A n= 2839 CC=0.3885 - Bin 14: 2.48- 2.42 A n= 2839 CC=0.3068 - Bin 15: 2.42- 2.37 A n= 2839 CC=0.3034 - Bin 16: 2.37- 2.32 A n= 2839 CC=0.3301 - Bin 17: 2.32- 2.28 A n= 2839 CC=0.3479 - Bin 18: 2.28- 2.24 A n= 2839 CC=0.3305 - Bin 19: 2.24- 2.20 A n= 2849 CC=0.3187 - Overall CC = 0.3097 - Work CC = 0.3110, Free CC = 0.2299 - -Results saved to /das/work/p17/p17490/Peter/Library/torchref/paper/figure4_difference_refinement/validation/output/extrapol_FULL/validate_ded_results.json - WDFo map: /das/work/p17/p17490/Peter/Library/torchref/paper/figure4_difference_refinement/validation/output/extrapol_FULL/validate_WDFo.ccp4 - WDFc map: /das/work/p17/p17490/Peter/Library/torchref/paper/figure4_difference_refinement/validation/output/extrapol_FULL/validate_WDFc.ccp4 - Plots saved to /das/work/p17/p17490/Peter/Library/torchref/paper/figure4_difference_refinement/validation/output/extrapol_FULL/validate_ded.png - -====================================================================== -Summary: - full_cell: CC = 0.3014 - selection: CC = 0.3344 - Reciprocal-space CC (overall): 0.3097 -====================================================================== - -Timing: 35.2s wall, 237.7s CPU diff --git a/paper/figure4_difference_refinement/validation/output/extrapol_IBL.log b/paper/figure4_difference_refinement/validation/output/extrapol_IBL.log deleted file mode 100644 index 048268f..0000000 --- a/paper/figure4_difference_refinement/validation/output/extrapol_IBL.log +++ /dev/null @@ -1,81 +0,0 @@ -/var/spool/slurmd/job890409/slurm_script:5: UserWarning: TorchRef auto-configured 16 threads. Set TORCHREF_NUM_THREADS to override. - from torchref.cli.validate_ded import main -====================================================================== -DED Validation: WDFo vs WDFcalc -====================================================================== - ---- Loading reflection data --- - Dark SF: /das/work/p17/p17490/Peter/Library/torchref/paper/figure4_difference_refinement/validation/../data/8QL2-sf.cif - Light SF: /das/work/p17/p17490/Peter/Library/torchref/paper/figure4_difference_refinement/validation/../data/7YYZ-light.mtz -/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/cif_readers.py:695: RuntimeWarning: invalid value encountered in cast - self.data["R-free-flags"] = rfree_characters.to_numpy().astype(np.int32) -Filtering: 57575/124404 reflections in range [inf - 2.2] Å (56814 valid after all masks) -Filtering: 56820/123649 reflections in range [inf - 2.2] Å (56796 valid after all masks) -Added dataset 'dark' (124404 reflections) -Added dataset 'light' (124404 reflections) -Scale parameters after optimization: - dark: log_scale=0.000000 (scale=1.000000) - light: log_scale=-0.035216 (scale=0.965397) -Matched reflections: 56790 -Cell: [74.53 92.58 83.99 90. 96.71 90. ], Spacegroup: P 1 21 1 -Resolution: 2.20 A, Grid: (108, 128, 120) -ASU: 56790, P1: 113563 - ---- Loading models --- - Dark model: 8523 atoms - Light model: 8228 atoms - Fraction: 0.22 - ---- Setting up joint scaler --- -Initialized ScalerBase with 20 bins. -Joint initial scale from 2 data-model pairs (20 bins). - Created 2 component solvent models. -/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/.dev/lib/python3.11/site-packages/torch/masked/maskedtensor/core.py:247: UserWarning: It is not recommended to create a MaskedTensor with a tensor that requires_grad. To avoid this, you can use data.detach().clone() - return MaskedTensor(data, mask) -Joint solvent screening: k_sol=0.3596, B_sol=55.0, NLL=53.1525 -Refining scales jointly with LBFGS... -Joint scale refinement complete. rwork: 0.1648, rfree: 0.1837 - R-factor (dark): R_work=0.1648 R_free=0.1837 - R-factor (mixed): R_work=0.1697 R_free=0.1831 - ---- Computing DED maps and correlations --- - |DFcalc| mean: 3.194 - |WDFcalc| mean: 2.936 - full_cell: CC = 0.3014 (n_vox = 1658880) - selection: CC = 0.7354 (n_vox = 1996) - Bin 0: 9.49- 5.59 A n= 2839 CC=0.1824 - Bin 1: 5.59- 4.60 A n= 2839 CC=0.2400 - Bin 2: 4.60- 4.06 A n= 2839 CC=0.3086 - Bin 3: 4.06- 3.71 A n= 2839 CC=0.3565 - Bin 4: 3.71- 3.46 A n= 2839 CC=0.3292 - Bin 5: 3.46- 3.26 A n= 2839 CC=0.3225 - Bin 6: 3.26- 3.10 A n= 2839 CC=0.2850 - Bin 7: 3.10- 2.97 A n= 2839 CC=0.3165 - Bin 8: 2.97- 2.86 A n= 2839 CC=0.2995 - Bin 9: 2.86- 2.76 A n= 2839 CC=0.3371 - Bin 10: 2.76- 2.68 A n= 2839 CC=0.3166 - Bin 11: 2.68- 2.60 A n= 2839 CC=0.3042 - Bin 12: 2.60- 2.54 A n= 2839 CC=0.3169 - Bin 13: 2.54- 2.48 A n= 2839 CC=0.3885 - Bin 14: 2.48- 2.42 A n= 2839 CC=0.3068 - Bin 15: 2.42- 2.37 A n= 2839 CC=0.3034 - Bin 16: 2.37- 2.32 A n= 2839 CC=0.3301 - Bin 17: 2.32- 2.28 A n= 2839 CC=0.3479 - Bin 18: 2.28- 2.24 A n= 2839 CC=0.3305 - Bin 19: 2.24- 2.20 A n= 2849 CC=0.3187 - Overall CC = 0.3097 - Work CC = 0.3110, Free CC = 0.2299 - -Results saved to /das/work/p17/p17490/Peter/Library/torchref/paper/figure4_difference_refinement/validation/output/extrapol_IBL/validate_ded_results.json - WDFo map: /das/work/p17/p17490/Peter/Library/torchref/paper/figure4_difference_refinement/validation/output/extrapol_IBL/validate_WDFo.ccp4 - WDFc map: /das/work/p17/p17490/Peter/Library/torchref/paper/figure4_difference_refinement/validation/output/extrapol_IBL/validate_WDFc.ccp4 - Plots saved to /das/work/p17/p17490/Peter/Library/torchref/paper/figure4_difference_refinement/validation/output/extrapol_IBL/validate_ded.png - -====================================================================== -Summary: - full_cell: CC = 0.3014 - selection: CC = 0.7354 - Reciprocal-space CC (overall): 0.3097 -====================================================================== - -Timing: 30.0s wall, 304.8s CPU diff --git a/paper/figure4_difference_refinement/validation/output/torchref_FULL.log b/paper/figure4_difference_refinement/validation/output/torchref_FULL.log deleted file mode 100644 index 6411923..0000000 --- a/paper/figure4_difference_refinement/validation/output/torchref_FULL.log +++ /dev/null @@ -1,81 +0,0 @@ -/var/spool/slurmd/job890408/slurm_script:5: UserWarning: TorchRef auto-configured 16 threads. Set TORCHREF_NUM_THREADS to override. - from torchref.cli.validate_ded import main -====================================================================== -DED Validation: WDFo vs WDFcalc -====================================================================== - ---- Loading reflection data --- - Dark SF: /das/work/p17/p17490/Peter/Library/torchref/paper/figure4_difference_refinement/validation/../data/8QL2-sf.cif - Light SF: /das/work/p17/p17490/Peter/Library/torchref/paper/figure4_difference_refinement/validation/../data/7YYZ-light.mtz -/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/cif_readers.py:695: RuntimeWarning: invalid value encountered in cast - self.data["R-free-flags"] = rfree_characters.to_numpy().astype(np.int32) -Filtering: 57575/124404 reflections in range [inf - 2.2] Å (56814 valid after all masks) -Filtering: 56820/123649 reflections in range [inf - 2.2] Å (56796 valid after all masks) -Added dataset 'dark' (124404 reflections) -Added dataset 'light' (124404 reflections) -Scale parameters after optimization: - dark: log_scale=0.000000 (scale=1.000000) - light: log_scale=-0.035216 (scale=0.965397) -Matched reflections: 56790 -Cell: [74.53 92.58 83.99 90. 96.71 90. ], Spacegroup: P 1 21 1 -Resolution: 2.20 A, Grid: (108, 128, 120) -ASU: 56790, P1: 113563 - ---- Loading models --- - Dark model: 8523 atoms - Light model: 8204 atoms - Fraction: 0.22 - ---- Setting up joint scaler --- -Initialized ScalerBase with 20 bins. -Joint initial scale from 2 data-model pairs (20 bins). - Created 2 component solvent models. -/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/.dev/lib/python3.11/site-packages/torch/masked/maskedtensor/core.py:247: UserWarning: It is not recommended to create a MaskedTensor with a tensor that requires_grad. To avoid this, you can use data.detach().clone() - return MaskedTensor(data, mask) -Joint solvent screening: k_sol=0.3596, B_sol=55.0, NLL=53.6882 -Refining scales jointly with LBFGS... -Joint scale refinement complete. rwork: 0.1644, rfree: 0.1835 - R-factor (dark): R_work=0.1644 R_free=0.1835 - R-factor (mixed): R_work=0.1729 R_free=0.1910 - ---- Computing DED maps and correlations --- - |DFcalc| mean: 2.462 - |WDFcalc| mean: 2.115 - full_cell: CC = 0.5162 (n_vox = 1658880) - selection: CC = 0.5622 (n_vox = 443198) - Bin 0: 9.49- 5.59 A n= 2839 CC=0.4597 - Bin 1: 5.59- 4.60 A n= 2839 CC=0.5731 - Bin 2: 4.60- 4.06 A n= 2839 CC=0.6316 - Bin 3: 4.06- 3.71 A n= 2839 CC=0.6186 - Bin 4: 3.71- 3.46 A n= 2839 CC=0.6130 - Bin 5: 3.46- 3.26 A n= 2839 CC=0.5679 - Bin 6: 3.26- 3.10 A n= 2839 CC=0.5243 - Bin 7: 3.10- 2.97 A n= 2839 CC=0.5240 - Bin 8: 2.97- 2.86 A n= 2839 CC=0.5498 - Bin 9: 2.86- 2.76 A n= 2839 CC=0.5423 - Bin 10: 2.76- 2.68 A n= 2839 CC=0.4952 - Bin 11: 2.68- 2.60 A n= 2839 CC=0.5152 - Bin 12: 2.60- 2.54 A n= 2839 CC=0.4784 - Bin 13: 2.54- 2.48 A n= 2839 CC=0.5077 - Bin 14: 2.48- 2.42 A n= 2839 CC=0.4028 - Bin 15: 2.42- 2.37 A n= 2839 CC=0.4354 - Bin 16: 2.37- 2.32 A n= 2839 CC=0.4580 - Bin 17: 2.32- 2.28 A n= 2839 CC=0.4328 - Bin 18: 2.28- 2.24 A n= 2839 CC=0.3759 - Bin 19: 2.24- 2.20 A n= 2849 CC=0.3597 - Overall CC = 0.5114 - Work CC = 0.5170, Free CC = 0.2251 - -Results saved to /das/work/p17/p17490/Peter/Library/torchref/paper/figure4_difference_refinement/validation/output/torchref_FULL/validate_ded_results.json - WDFo map: /das/work/p17/p17490/Peter/Library/torchref/paper/figure4_difference_refinement/validation/output/torchref_FULL/validate_WDFo.ccp4 - WDFc map: /das/work/p17/p17490/Peter/Library/torchref/paper/figure4_difference_refinement/validation/output/torchref_FULL/validate_WDFc.ccp4 - Plots saved to /das/work/p17/p17490/Peter/Library/torchref/paper/figure4_difference_refinement/validation/output/torchref_FULL/validate_ded.png - -====================================================================== -Summary: - full_cell: CC = 0.5162 - selection: CC = 0.5622 - Reciprocal-space CC (overall): 0.5114 -====================================================================== - -Timing: 34.1s wall, 331.9s CPU diff --git a/paper/figure4_difference_refinement/validation/output/torchref_IBL.log b/paper/figure4_difference_refinement/validation/output/torchref_IBL.log deleted file mode 100644 index f83c1c2..0000000 --- a/paper/figure4_difference_refinement/validation/output/torchref_IBL.log +++ /dev/null @@ -1,81 +0,0 @@ -/var/spool/slurmd/job890407/slurm_script:5: UserWarning: TorchRef auto-configured 16 threads. Set TORCHREF_NUM_THREADS to override. - from torchref.cli.validate_ded import main -====================================================================== -DED Validation: WDFo vs WDFcalc -====================================================================== - ---- Loading reflection data --- - Dark SF: /das/work/p17/p17490/Peter/Library/torchref/paper/figure4_difference_refinement/validation/../data/8QL2-sf.cif - Light SF: /das/work/p17/p17490/Peter/Library/torchref/paper/figure4_difference_refinement/validation/../data/7YYZ-light.mtz -/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/torchref/io/cif_readers.py:695: RuntimeWarning: invalid value encountered in cast - self.data["R-free-flags"] = rfree_characters.to_numpy().astype(np.int32) -Filtering: 57575/124404 reflections in range [inf - 2.2] Å (56814 valid after all masks) -Filtering: 56820/123649 reflections in range [inf - 2.2] Å (56796 valid after all masks) -Added dataset 'dark' (124404 reflections) -Added dataset 'light' (124404 reflections) -Scale parameters after optimization: - dark: log_scale=0.000000 (scale=1.000000) - light: log_scale=-0.035216 (scale=0.965397) -Matched reflections: 56790 -Cell: [74.53 92.58 83.99 90. 96.71 90. ], Spacegroup: P 1 21 1 -Resolution: 2.20 A, Grid: (108, 128, 120) -ASU: 56790, P1: 113563 - ---- Loading models --- - Dark model: 8523 atoms - Light model: 8204 atoms - Fraction: 0.22 - ---- Setting up joint scaler --- -Initialized ScalerBase with 20 bins. -Joint initial scale from 2 data-model pairs (20 bins). - Created 2 component solvent models. -/das/work/units/LBR-FEL/p17490/Peter/Library/torchref/.dev/lib/python3.11/site-packages/torch/masked/maskedtensor/core.py:247: UserWarning: It is not recommended to create a MaskedTensor with a tensor that requires_grad. To avoid this, you can use data.detach().clone() - return MaskedTensor(data, mask) -Joint solvent screening: k_sol=0.3596, B_sol=55.0, NLL=53.8316 -Refining scales jointly with LBFGS... -Joint scale refinement complete. rwork: 0.1644, rfree: 0.1835 - R-factor (dark): R_work=0.1644 R_free=0.1835 - R-factor (mixed): R_work=0.1729 R_free=0.1910 - ---- Computing DED maps and correlations --- - |DFcalc| mean: 2.462 - |WDFcalc| mean: 2.115 - full_cell: CC = 0.5162 (n_vox = 1658880) - selection: CC = 0.8489 (n_vox = 1955) - Bin 0: 9.49- 5.59 A n= 2839 CC=0.4596 - Bin 1: 5.59- 4.60 A n= 2839 CC=0.5731 - Bin 2: 4.60- 4.06 A n= 2839 CC=0.6316 - Bin 3: 4.06- 3.71 A n= 2839 CC=0.6186 - Bin 4: 3.71- 3.46 A n= 2839 CC=0.6130 - Bin 5: 3.46- 3.26 A n= 2839 CC=0.5679 - Bin 6: 3.26- 3.10 A n= 2839 CC=0.5243 - Bin 7: 3.10- 2.97 A n= 2839 CC=0.5240 - Bin 8: 2.97- 2.86 A n= 2839 CC=0.5498 - Bin 9: 2.86- 2.76 A n= 2839 CC=0.5423 - Bin 10: 2.76- 2.68 A n= 2839 CC=0.4952 - Bin 11: 2.68- 2.60 A n= 2839 CC=0.5152 - Bin 12: 2.60- 2.54 A n= 2839 CC=0.4784 - Bin 13: 2.54- 2.48 A n= 2839 CC=0.5077 - Bin 14: 2.48- 2.42 A n= 2839 CC=0.4028 - Bin 15: 2.42- 2.37 A n= 2839 CC=0.4354 - Bin 16: 2.37- 2.32 A n= 2839 CC=0.4580 - Bin 17: 2.32- 2.28 A n= 2839 CC=0.4328 - Bin 18: 2.28- 2.24 A n= 2839 CC=0.3759 - Bin 19: 2.24- 2.20 A n= 2849 CC=0.3597 - Overall CC = 0.5114 - Work CC = 0.5170, Free CC = 0.2251 - -Results saved to /das/work/p17/p17490/Peter/Library/torchref/paper/figure4_difference_refinement/validation/output/torchref_IBL/validate_ded_results.json - WDFo map: /das/work/p17/p17490/Peter/Library/torchref/paper/figure4_difference_refinement/validation/output/torchref_IBL/validate_WDFo.ccp4 - WDFc map: /das/work/p17/p17490/Peter/Library/torchref/paper/figure4_difference_refinement/validation/output/torchref_IBL/validate_WDFc.ccp4 - Plots saved to /das/work/p17/p17490/Peter/Library/torchref/paper/figure4_difference_refinement/validation/output/torchref_IBL/validate_ded.png - -====================================================================== -Summary: - full_cell: CC = 0.5162 - selection: CC = 0.8489 - Reciprocal-space CC (overall): 0.5114 -====================================================================== - -Timing: 26.6s wall, 227.4s CPU diff --git a/paper/slurm-886596.out b/paper/slurm-886596.out deleted file mode 100644 index 510c2f4..0000000 --- a/paper/slurm-886596.out +++ /dev/null @@ -1 +0,0 @@ -Submitted batch job 886597 diff --git a/paper/slurm-889402.out b/paper/slurm-889402.out deleted file mode 100644 index 138185e..0000000 --- a/paper/slurm-889402.out +++ /dev/null @@ -1 +0,0 @@ -Submitted batch job 889463 From 44abbf78ac2162c0916a2194acf31f2ed0c9e96e Mon Sep 17 00:00:00 2001 From: HatPdotS Date: Tue, 7 Jul 2026 11:04:28 +0200 Subject: [PATCH 3/4] Standardized rfactor reporting and switched to accessor based syntax --- tests/unit/refinement/test_ml_sigmaa.py | 1 - torchref/cli/collection_difference_refine.py | 36 ++- torchref/experimental/kinetic/refinement.py | 47 ++++ torchref/experimental/kinetic/targets.py | 1 - torchref/io/datasets/collection.py | 84 ++++++ .../refinement/targets/collection/__init__.py | 4 +- .../refinement/targets/collection/_util.py | 26 -- .../refinement/targets/collection/base.py | 226 ++++++++++++++++ .../refinement/targets/collection/xray.py | 241 ++++++++++-------- torchref/scaling/collection_scaler.py | 37 +-- 10 files changed, 537 insertions(+), 166 deletions(-) create mode 100644 torchref/refinement/targets/collection/base.py diff --git a/tests/unit/refinement/test_ml_sigmaa.py b/tests/unit/refinement/test_ml_sigmaa.py index 105ab50..a264d2d 100644 --- a/tests/unit/refinement/test_ml_sigmaa.py +++ b/tests/unit/refinement/test_ml_sigmaa.py @@ -94,7 +94,6 @@ def test_kinetic_backcompat_reexports(self): from torchref.experimental.kinetic.targets import ( # noqa: F401 KineticPriorTarget, _scale_fcalc, - _unpack_masked_data, ) from torchref.refinement.targets import CollectionMLTarget as RefCML diff --git a/torchref/cli/collection_difference_refine.py b/torchref/cli/collection_difference_refine.py index 9419e7f..c2e0b60 100644 --- a/torchref/cli/collection_difference_refine.py +++ b/torchref/cli/collection_difference_refine.py @@ -216,18 +216,19 @@ def setup_scaler(dataset_collection, model_collection, device, verbose=1): def compute_rfactors(model, data, scaler): - """Compute R-work/R-free using forward_mixed for proper solvent.""" - from torchref.base.metrics import get_rfactors + """Compute R-work/R-free using forward_mixed for proper solvent. + + Routes through ``rfactor_work_free`` — the shared source of truth used by the + refinement targets — so the validity mask is applied and the validation set is + excluded from both work and free, matching every other reported R-factor. + """ + from torchref.base.metrics.rfactor import rfactor_work_free with torch.no_grad(): hkl = data.hkl - fobs = data.get_corrected_data()[0] - rfree = data.rfree_flags fcalc = model(hkl) fcalc_scaled = scaler.forward_mixed(fcalc, model.fractions) - return get_rfactors( - torch.abs(fobs), torch.abs(fcalc_scaled), rfree - ) + return rfactor_work_free(data, torch.abs(fcalc_scaled)) def setup_loss_state(dataset_collection, model_collection, scaler, @@ -454,15 +455,10 @@ def write_results_mtz(dc, mc, scaler, filename): amp_2fofc_bayes = 2 * F_ext_bayes_amp - amp_calc_bayes amp_fofc_bayes = F_ext_bayes_amp - amp_calc_bayes - from torchref.base.metrics import get_rfactors + from torchref.base.metrics.rfactor import rfactor_work_free def _extrapolation_rfactors(data, fcalc_scaled): - valid = data.masks().to(torch.bool) - return get_rfactors( - torch.abs(data.get_corrected_data()[0][valid]), - torch.abs(fcalc_scaled[valid]), - data.rfree_flags[valid], - ) + return rfactor_work_free(data, torch.abs(fcalc_scaled)) print("Phase-aware extrapolation rfactors:", _extrapolation_rfactors(data_light_extra, F_calc_extra)) @@ -932,14 +928,12 @@ def _build_metadata(model, data, r_work, r_free): meta.resolution_high = float(res_valid.min()) meta.resolution_low = float(res_valid.max()) - # Reflection counts + # Reflection counts (standard work/free subset accessors: validity + # masked, validation carved out of both). with torch.no_grad(): - rfree_flags = data.rfree_flags - if rfree_flags is not None: - rfree_bool = rfree_flags.bool() - valid_mask = data.masks().to(torch.bool) - n_work = int((valid_mask & rfree_bool).sum().item()) - n_test = int((valid_mask & ~rfree_bool).sum().item()) + if data.rfree_flags is not None: + n_work = data.work.n + n_test = data.free.n n_all = n_work + n_test meta.n_reflections_work = n_work meta.n_reflections_test = n_test diff --git a/torchref/experimental/kinetic/refinement.py b/torchref/experimental/kinetic/refinement.py index 5d69de6..556ac17 100644 --- a/torchref/experimental/kinetic/refinement.py +++ b/torchref/experimental/kinetic/refinement.py @@ -265,6 +265,46 @@ def print_loss_summary(self): self.loss_state.aggregate(log_values=True) self.loss_state.summary() + def report_rfactors(self) -> Dict[str, dict]: + """Report per-target work/free R-factors across the collection. + + Routes through each collection X-ray target's shared ``get_rfactor`` + (``rfactor_work_free`` per dataset — validity masked, validation excluded + from both work and free), so the reported numbers use exactly the same + reflection partition the loss does. R-factors form a distribution over the + datasets; the median is the headline and the 10-90% spread is shown. + + Returns + ------- + dict + ``{target_name: get_rfactor() result}`` for each X-ray target. + """ + from torchref.refinement.targets.collection import CollectionXrayTarget + + out: Dict[str, dict] = {} + with torch.no_grad(): + for name, target in self.loss_state.targets.items(): + if not isinstance(target, CollectionXrayTarget): + continue + rf = target.get_rfactor() + out[name] = rf + pw, pf = rf["rwork_pct"], rf["rfree_pct"] + if not pw or self.verbose <= 0: + continue + print(f" [{name}] R-factors (median [10-90%]):") + print( + f" R_work = {pw['p50']:.4f} " + f"[{pw['p10']:.4f}-{pw['p90']:.4f}]" + ) + print( + f" R_free = {pf['p50']:.4f} " + f"[{pf['p10']:.4f}-{pf['p90']:.4f}]" + ) + if self.verbose > 1: + for k, (rw, rfr) in rf["per_dataset"].items(): + print(f" {k}: R_work={rw:.4f} R_free={rfr:.4f}") + return out + # ------------------------------------------------------------------ # Optimization loops # ------------------------------------------------------------------ @@ -342,6 +382,10 @@ def refine(self, macro_cycles: int = 5, niter: int = 10, max_iter: int = 50): if hasattr(self.scaler, "update_all_solvent"): self.scaler.update_all_solvent() + # Report R-factors through the shared source of truth. + if self.verbose > 0: + self.report_rfactors() + def refine_structures(self, niter: int = 10, max_iter: int = 50): """ Refine base model structures only (xyz, adp). @@ -433,6 +477,9 @@ def refine_alternating( frac_str = ", ".join(f"{v:.3f}" for v in f.detach().tolist()) print(f" {name}: [{frac_str}]") + # Report R-factors through the shared source of truth. + self.report_rfactors() + def refit_kinetic_prior(self, niter: int = 50, lr: float = 1e-2): """ Refit the kinetic model to match current free fractions. diff --git a/torchref/experimental/kinetic/targets.py b/torchref/experimental/kinetic/targets.py index 83d363a..5420123 100644 --- a/torchref/experimental/kinetic/targets.py +++ b/torchref/experimental/kinetic/targets.py @@ -37,7 +37,6 @@ from torchref.refinement.targets.collection._util import ( # noqa: F401 _LOG_2PI, _scale_fcalc, - _unpack_masked_data, ) if TYPE_CHECKING: diff --git a/torchref/io/datasets/collection.py b/torchref/io/datasets/collection.py index 27f54de..54722d0 100644 --- a/torchref/io/datasets/collection.py +++ b/torchref/io/datasets/collection.py @@ -235,6 +235,90 @@ def _calculate_resolution(self) -> None: resolution = 1.0 / torch.linalg.norm(s, axis=1) self._resolution = resolution + def harmonize_partition( + self, + val_fraction_of_free: Optional[float] = None, + seed: Optional[int] = None, + source: Optional[str] = None, + ) -> "DatasetCollection": + """Make the work/free (and validation) partition identical across members. + + In a collection every member is expanded onto the same common HKL grid, so + a single work/free/validation assignment can be shared by all datasets. + Sharing it is also *correct*: per-dataset free sets would let a reflection + that is free in one timepoint leak into the work set of another, biasing the + cross-dataset R-free. This copies one canonical ``rfree_flags`` and + ``validation_flags`` onto every member (row-aligned; cheap clone). + + The canonical partition is taken from a ``source`` member (default: the + reference dataset), whose ``rfree_flags`` define the common work/free split. + When ``val_fraction_of_free`` is given, a common validation set is carved out + of that member's free reflections (resolution-stratified via + :meth:`ReflectionData.generate_validation_set`) before broadcasting. + + Parameters + ---------- + val_fraction_of_free : float, optional + Fraction of the free reflections to reassign as a held-out validation + set, shared across all datasets. If None, no validation set is created + (existing ``validation_flags`` on the source, if any, are still + broadcast). + seed : int, optional + Seed for the validation split (reproducibility). + source : str, optional + Name of the member whose partition is canonical. Defaults to the + reference dataset (or the first added dataset). + + Returns + ------- + DatasetCollection + Self, for chaining. + """ + if not self._datasets: + raise RuntimeError("Cannot harmonize an empty collection.") + + src_name = source or self._reference_dataset or self._dataset_order[0] + if src_name not in self._datasets: + raise KeyError(f"Source dataset {src_name!r} not in collection.") + src = self._datasets[src_name] + + if src.rfree_flags is None: + raise ValueError( + f"Source dataset {src_name!r} has no rfree_flags to harmonize on." + ) + + # Optionally carve a common validation set out of the source's free set. + if val_fraction_of_free is not None: + src.generate_validation_set( + val_fraction_of_free=val_fraction_of_free, seed=seed + ) + + # Broadcast the canonical partition to every member (row-aligned clones). + canonical_rfree = src.rfree_flags + canonical_val = src.validation_flags + for name, ds in self._datasets.items(): + if name == src_name: + continue + ds.rfree_flags = canonical_rfree.clone().to(ds.device) + if canonical_val is not None: + ds.validation_flags = canonical_val.clone().to(ds.device) + + if self.verbose > 0: + n_work = int(canonical_rfree.to(torch.bool).sum().item()) + total = len(canonical_rfree) + n_val = ( + int(canonical_val.to(torch.bool).sum().item()) + if canonical_val is not None + else 0 + ) + n_free = total - n_work - n_val + print( + f"Harmonized partition from {src_name!r} across " + f"{self.n_datasets} datasets: work={n_work}, free={n_free}, " + f"val={n_val}." + ) + return self + def __call__(self, mask: bool = True) -> Dict[str, Tuple]: """ Return all datasets' data scaled if scale factors are set. diff --git a/torchref/refinement/targets/collection/__init__.py b/torchref/refinement/targets/collection/__init__.py index 9740215..f4302a7 100644 --- a/torchref/refinement/targets/collection/__init__.py +++ b/torchref/refinement/targets/collection/__init__.py @@ -7,7 +7,8 @@ :mod:`torchref.experimental.kinetic.targets`. """ -from ._util import _scale_fcalc, _unpack_masked_data +from ._util import _scale_fcalc +from .base import CollectionXrayTarget from .multimodel import MultiModelADPTarget, MultiModelGeometryTarget from .xray import ( CollectionDifferenceTarget, @@ -16,6 +17,7 @@ ) __all__ = [ + "CollectionXrayTarget", "CollectionDifferenceTarget", "CollectionRiceTarget", "CollectionMLTarget", diff --git a/torchref/refinement/targets/collection/_util.py b/torchref/refinement/targets/collection/_util.py index 75c133e..7611ccf 100644 --- a/torchref/refinement/targets/collection/_util.py +++ b/torchref/refinement/targets/collection/_util.py @@ -1,36 +1,10 @@ """Shared helpers for collection (multi-dataset) targets.""" -from typing import TYPE_CHECKING, Optional, Tuple - import numpy as np -import torch - -if TYPE_CHECKING: - from torchref.io.datasets.reflection_data import ReflectionData _LOG_2PI = np.log(2.0 * np.pi) -def _unpack_masked_data( - data: "ReflectionData", -) -> Tuple[ - torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, Optional[torch.Tensor] -]: - """Extract plain tensors + validity mask from a ReflectionData call. - - Returns - ------- - F_obs, sigma, rfree_bool, validity, centric - All as plain tensors (not MaskedTensors). *rfree_bool* has - True = work, False = free. *centric* may be None. - """ - F_obs, sigma = data.get_corrected_data() - rfree = data.rfree_flags - validity = data.masks().to(torch.bool) - centric = data.centric if hasattr(data, "centric") else None - return F_obs, sigma, rfree.bool(), validity, centric - - def _scale_fcalc(scaler, fcalc, model): """Apply scaler, using forward_mixed when available.""" if scaler is None: diff --git a/torchref/refinement/targets/collection/base.py b/torchref/refinement/targets/collection/base.py new file mode 100644 index 0000000..8cb2026 --- /dev/null +++ b/torchref/refinement/targets/collection/base.py @@ -0,0 +1,226 @@ +"""Shared base for collection (multi-dataset) X-ray targets. + +``CollectionXrayTarget`` gives the multi-dataset targets the same subset and +R-factor contract as the single-dataset :class:`~torchref.refinement.targets.xray.base.XrayTarget`: + +* a canonical 3-way subset selector ``use_set`` in ``{"work", "free", "val"}`` + that maps onto each member :class:`~torchref.io.datasets.reflection_data.ReflectionData`'s + ``data.work`` / ``data.free`` / ``data.validation`` accessors (validity mask + applied, validation carved out of both work and free); +* one shared R-factor source of truth + (:func:`~torchref.base.metrics.rfactor.rfactor_work_free`) computed per dataset + through the scaler's scaling, exactly what the loss sees; +* a standard ``stats()`` dict (``loss`` / ``n`` / ``rwork`` / ``rfree``) so the + refinement logger surfaces R-factors for the collection just like it does for a + single dataset. + +Because every member of a :class:`~torchref.io.datasets.collection.DatasetCollection` +is expanded onto one common HKL grid, per-dataset R-factors form a distribution; +the headline ``rwork`` / ``rfree`` is the median and the 10/25/75/90 percentiles are +reported at higher verbosity. +""" + +from typing import TYPE_CHECKING, Dict, List, Optional + +import torch + +from torchref.base.metrics.rfactor import rfactor_work_free +from torchref.refinement.targets.base import Target +from torchref.utils.stats import ( + VERBOSITY_DEBUG, + VERBOSITY_DETAILED, + VERBOSITY_STANDARD, + StatEntry, + stat, +) + +from ._util import _scale_fcalc + +if TYPE_CHECKING: + from torchref.io.datasets.collection import DatasetCollection + from torchref.model.model_collection import ModelCollection + from torchref.scaling.scaler_base import ScalerBase + + +# Percentiles reported for the per-dataset R-factor distribution. +_R_PERCENTILES = (0.10, 0.25, 0.50, 0.75, 0.90) +_R_PCT_LABELS = ("p10", "p25", "p50", "p75", "p90") + + +class CollectionXrayTarget(Target): + """Base class for multi-dataset X-ray targets. + + Parameters + ---------- + dataset_collection : DatasetCollection + Collection of reflection datasets keyed by timepoint name. + model_collection : ModelCollection + Collection of mixed models keyed by timepoint name. + scaler : ScalerBase, optional + Single scaler applied to every F_calc (``forward_mixed`` when available). + use_work_set : bool, optional + Legacy bool; superseded by ``use_set`` when the latter is given. Default + True (work set). + use_set : str, optional + Canonical 3-way subset selector ``"work"``/``"free"``/``"val"``. Takes + precedence over ``use_work_set``; derived from it if None. + verbose : int, optional + Verbosity level. + """ + + name: str = "collection_xray" + + def __init__( + self, + dataset_collection: "DatasetCollection", + model_collection: "ModelCollection", + scaler: "ScalerBase" = None, + use_work_set: bool = True, + use_set: str = None, + verbose: int = 0, + ): + super().__init__(verbose=verbose) + self._dataset_collection = dataset_collection + self._model_collection = model_collection + self.add_module("_scaler", scaler) + # Canonical 3-way subset selector, mirroring XrayTarget.__init__ so the + # loss and the reported subset never disagree. + if use_set is None: + use_set = "work" if use_work_set else "free" + self.use_set = use_set + self.use_work_set = use_set == "work" + + # ------------------------------------------------------------------ + # Dataset / model / subset plumbing + # ------------------------------------------------------------------ + + def _keys(self) -> List[str]: + """Matched dataset keys this target fits: dark + present timepoints. + + Overridden by targets that fit only a subset of the collection (e.g. + :class:`CollectionRiceTarget` drops the dark reference). + """ + dc = self._dataset_collection + mc = self._model_collection + keys = [mc.dark_key] if mc.dark_key in dc else [] + keys += [n for n in mc.timepoint_names if n in dc] + return keys + + def _subset(self, data): + """Return the ``_ReflectionSubset`` view selected by ``use_set``.""" + if self.use_set == "free": + return data.free + if self.use_set == "val": + return data.validation + return data.work + + def _reset_model_caches(self) -> None: + """Clear cached forwards on every base model. + + Called at the start of each ``forward`` so a preceding no-grad R-factor + evaluation (which may leave a detached, grad-less tensor in the cache) + cannot starve the loss backward. + """ + for bm in getattr(self._model_collection, "base_models", []): + if hasattr(bm, "reset_cache"): + bm.reset_cache() + + def _scaled_amp_full(self, data, model, recalc: bool = True) -> torch.Tensor: + """Full-size, scaled ``|F_calc|`` for one data-model pair. + + Uses ``hkl_for_sf()`` (signed indices) so anomalous mates get distinct + amplitudes. ``recalc=True`` (the default, used by the no-grad R-factor + path) forces a fresh compute so it cannot reuse — or leave behind — a + cached tensor that would break gradient flow; the loss path passes + ``recalc=False`` after :meth:`_reset_model_caches` has cleared the cache. + """ + fcalc = model(data.hkl_for_sf(), recalc=recalc) + return torch.abs(_scale_fcalc(self._scaler, fcalc, model)) + + # ------------------------------------------------------------------ + # R-factor reporting (shared source of truth) + # ------------------------------------------------------------------ + + def get_rfactor(self) -> Dict[str, object]: + """Per-dataset R-work / R-free plus percentile summaries. + + Each dataset's R-factor is computed with + :func:`~torchref.base.metrics.rfactor.rfactor_work_free` on the exact + scaled ``|F_calc|`` the loss sees, so the collection cannot disagree with + the single-dataset targets on convention. R-factors are unweighted (no + ``base_weight``) and scale-invariant within a dataset. + + Returns + ------- + dict + ``{"per_dataset": {key: (rwork, rfree)}, + "rwork_pct": {label: value}, "rfree_pct": {label: value}}``. + The percentile dicts are empty when no dataset contributed. + """ + dc = self._dataset_collection + mc = self._model_collection + per_dataset: Dict[str, tuple] = {} + rworks: List[float] = [] + rfrees: List[float] = [] + with torch.no_grad(): + for key in self._keys(): + data = dc[key] + model = mc[key] + amp = self._scaled_amp_full(data, model) + rwork, rfree = rfactor_work_free(data, amp) + per_dataset[key] = (rwork, rfree) + rworks.append(rwork) + rfrees.append(rfree) + rwork_pct = self._percentiles(rworks) + rfree_pct = self._percentiles(rfrees) + return { + "per_dataset": per_dataset, + "rwork_pct": rwork_pct, + "rfree_pct": rfree_pct, + } + + @staticmethod + def _percentiles(values: List[float]) -> Dict[str, float]: + """10/25/50/75/90 percentiles of a list of per-dataset R-factors.""" + if not values: + return {} + t = torch.tensor(values, dtype=torch.float64) + q = torch.quantile(t, torch.tensor(_R_PERCENTILES, dtype=torch.float64)) + return {lbl: q[i].item() for i, lbl in enumerate(_R_PCT_LABELS)} + + # ------------------------------------------------------------------ + # Stats + # ------------------------------------------------------------------ + + def _n_reflections(self) -> int: + """Total reflections in this target's subset across all datasets.""" + dc = self._dataset_collection + return int(sum(self._subset(dc[k]).n for k in self._keys())) + + def stats(self) -> Dict[str, StatEntry]: + """Standard collection X-ray stats: loss / n / rwork / rfree (+percentiles). + + Headline ``rwork`` / ``rfree`` are the medians of the per-dataset + distribution (``VERBOSITY_STANDARD``); the 10/25/75/90 percentiles and the + per-dataset values are reported at higher verbosity. Subclasses that own + extra diagnostics (e.g. shared beta) override this and merge on top. + """ + out: Dict[str, StatEntry] = {} + loss = self.forward() + out["loss"] = stat(loss.item(), VERBOSITY_STANDARD) + out["n"] = stat(self._n_reflections(), VERBOSITY_DEBUG) + + rf = self.get_rfactor() + rwork_pct, rfree_pct = rf["rwork_pct"], rf["rfree_pct"] + if rwork_pct: + out["rwork"] = stat(rwork_pct["p50"], VERBOSITY_STANDARD) + out["rfree"] = stat(rfree_pct["p50"], VERBOSITY_STANDARD) + for lbl in _R_PCT_LABELS: + if lbl == "p50": + continue + out[f"rwork_{lbl}"] = stat(rwork_pct[lbl], VERBOSITY_DETAILED) + out[f"rfree_{lbl}"] = stat(rfree_pct[lbl], VERBOSITY_DETAILED) + for key, (rw, rfr) in rf["per_dataset"].items(): + out[f"rwork_{key}"] = stat(rw, VERBOSITY_DEBUG) + out[f"rfree_{key}"] = stat(rfr, VERBOSITY_DEBUG) + return out diff --git a/torchref/refinement/targets/collection/xray.py b/torchref/refinement/targets/collection/xray.py index 7780042..78ee4bd 100644 --- a/torchref/refinement/targets/collection/xray.py +++ b/torchref/refinement/targets/collection/xray.py @@ -2,8 +2,14 @@ Apply a single X-ray likelihood across a paired ``DatasetCollection`` + ``ModelCollection``. Keys are matched automatically so each timepoint dataset -is paired with its corresponding mixed model. All computation is vectorized on -stacked ``(N, n_hkl)`` tensors. +is paired with its corresponding mixed model. + +All three targets subclass :class:`~torchref.refinement.targets.collection.base.CollectionXrayTarget`, +so they share the single-dataset subset/masking/R-factor contract: reflections +are selected through each member's ``data.work`` / ``data.free`` / ``data.validation`` +accessors (validity mask applied, validation carved out of both), and R-factors +are reported through ``stats()`` via the one shared +:func:`~torchref.base.metrics.rfactor.rfactor_work_free` source of truth. Targets ------- @@ -29,10 +35,10 @@ epsilon_from_hkl, ml_xray_loss_beta_math, ) -from torchref.refinement.targets.base import Target from torchref.utils.stats import VERBOSITY_STANDARD, StatEntry, stat -from ._util import _LOG_2PI, _scale_fcalc, _unpack_masked_data +from ._util import _LOG_2PI, _scale_fcalc +from .base import CollectionXrayTarget if TYPE_CHECKING: from torchref.io.datasets.collection import DatasetCollection @@ -45,7 +51,7 @@ # ========================================================================= -class CollectionDifferenceTarget(Target): +class CollectionDifferenceTarget(CollectionXrayTarget): """ Mean-based difference target using DatasetCollection + ModelCollection. @@ -63,8 +69,11 @@ class CollectionDifferenceTarget(Target): direct dark-reference subtraction. For N>2 the mean reference has lower noise. - All computation is vectorized on stacked (N, n_hkl) tensors — no - Python loops over datasets. + This is a cross-dataset-coupled target: the per-reflection mean couples all + datasets, so it works on aligned ``(N, n_hkl)`` stacks on the common HKL grid + (not the flat concatenate-then-mask form the independent targets use). The + combined mask requires a reflection to be in this target's subset in **every** + dataset. Parameters ---------- @@ -77,7 +86,9 @@ class CollectionDifferenceTarget(Target): Unused placeholder. ``forward`` always returns the unnormalised summed NLL regardless of this flag. use_work_set : bool - If True, compute loss only on the work set (rfree_flags=True). + Legacy bool; superseded by ``use_set``. If True, loss on the work set. + use_set : str, optional + Canonical 3-way subset selector ``"work"``/``"free"``/``"val"``. verbose : int Verbosity level. """ @@ -91,37 +102,44 @@ def __init__( scaler: "ScalerBase" = None, normalize: bool = True, use_work_set: bool = True, + use_set: str = None, verbose: int = 0, ): - super().__init__(verbose=verbose) - self._dataset_collection = dataset_collection - self._model_collection = model_collection - self.add_module("_scaler", scaler) + super().__init__( + dataset_collection, + model_collection, + scaler=scaler, + use_work_set=use_work_set, + use_set=use_set, + verbose=verbose, + ) self.normalize = normalize - self.use_work_set = use_work_set def forward(self) -> torch.Tensor: dc = self._dataset_collection mc = self._model_collection - hkl = dc.hkl - # Collect all matched dataset keys (dark + timepoints) - all_keys = [mc.dark_key] + [n for n in mc.timepoint_names if n in dc] + all_keys = self._keys() N = len(all_keys) if N < 2: - return torch.tensor(0.0, device=hkl.device) + return torch.tensor(0.0, device=dc.hkl.device) + + # Clear caches so a preceding no-grad stats()/get_rfactor() call cannot + # leave a detached tensor that breaks the loss backward. + self._reset_model_caches() - # --- Gather per-dataset tensors --- + # --- Gather per-dataset full-size tensors on the common HKL grid --- F_obs_list, sigma_list, mask_list, F_calc_list = [], [], [], [] for key in all_keys: data = dc[key] model = mc[key] - F_obs, sigma, rfree, validity, _ = _unpack_masked_data(data) - F_calc = torch.abs(_scale_fcalc(self._scaler, model(hkl), model)) - - mask = validity & rfree if self.use_work_set else validity + F_obs, sigma = data.get_corrected_data() + F_calc = self._scaled_amp_full(data, model, recalc=False) + # Subset boolean mask: validity + work/free/val selection, validation + # carved out of both work and free. + mask = self._subset(data).mask F_obs_list.append(F_obs) sigma_list.append(sigma) @@ -134,7 +152,7 @@ def forward(self) -> torch.Tensor: mask_stack = torch.stack(mask_list) # (N, n_hkl) F_calc_stack = torch.stack(F_calc_list) # (N, n_hkl) - # Combined mask: reflection must be valid + work-set in ALL datasets + # Combined mask: reflection must be in this subset in ALL datasets mask_all = mask_stack.all(dim=0) # (n_hkl,) # --- Mean across datasets --- @@ -184,13 +202,15 @@ def forward(self) -> torch.Tensor: # ========================================================================= -class CollectionRiceTarget(Target): +class CollectionRiceTarget(CollectionXrayTarget): """ Multi-timepoint Rice maximum-likelihood amplitude target. Computes Rice-distribution NLL (acentric) and the corresponding - centric NLL for each timepoint, with proper validity masking and - NaN/Inf protection. Vectorized on stacked (N_tp, n_hkl) tensors. + centric NLL for each timepoint, with proper subset masking and + NaN/Inf protection. The per-timepoint losses are independent, so the + reflections and boolean masks are **concatenated across timepoints and + masked once** (flat vectorization) rather than summed in a Python loop. Parameters ---------- @@ -202,7 +222,9 @@ class CollectionRiceTarget(Target): Unused placeholder. ``forward`` always returns the unnormalised summed NLL regardless of this flag. use_work_set : bool - Compute loss only on work set. + Legacy bool; superseded by ``use_set``. If True, loss on the work set. + use_set : str, optional + Canonical 3-way subset selector ``"work"``/``"free"``/``"val"``. verbose : int Verbosity level. """ @@ -216,58 +238,67 @@ def __init__( scaler: "ScalerBase" = None, normalize: bool = True, use_work_set: bool = True, + use_set: str = None, verbose: int = 0, ): - super().__init__(verbose=verbose) - self._dataset_collection = dataset_collection - self._model_collection = model_collection - self.add_module("_scaler", scaler) + super().__init__( + dataset_collection, + model_collection, + scaler=scaler, + use_work_set=use_work_set, + use_set=use_set, + verbose=verbose, + ) self.normalize = normalize - self.use_work_set = use_work_set + + def _keys(self): + """Rice fits the timepoints only — the dark reference is excluded.""" + mc = self._model_collection + dc = self._dataset_collection + return [n for n in mc.timepoint_names if n in dc] def forward(self) -> torch.Tensor: dc = self._dataset_collection mc = self._model_collection - tp_names = [n for n in mc.timepoint_names if n in dc] + tp_names = self._keys() if not tp_names: return torch.tensor(0.0, device=mc.device) - # --- Gather per-timepoint tensors --- - F_obs_list, F_calc_list, sigma_list = [], [], [] - mask_list, centric_list = [], [] + self._reset_model_caches() + # --- Gather per-timepoint full-size tensors, then concatenate + mask --- + fo_parts, fc_parts, sig_parts, cen_parts, mask_parts = [], [], [], [], [] for tp_name in tp_names: data = dc[tp_name] model = mc[tp_name] - hkl = data.hkl - - F_obs, sigma, rfree, validity, centric = _unpack_masked_data(data) - F_calc_amp = torch.abs(_scale_fcalc(self._scaler, model(hkl), model)) - mask = validity & rfree if self.use_work_set else validity + F_obs, sigma = data.get_corrected_data() + F_calc = self._scaled_amp_full(data, model, recalc=False) + centric = data.centric if centric is None: - centric = torch.zeros(len(hkl), dtype=torch.bool, device=hkl.device) + centric = torch.zeros( + len(F_obs), dtype=torch.bool, device=F_obs.device + ) - F_obs_list.append(F_obs) - F_calc_list.append(F_calc_amp) - sigma_list.append(sigma) - mask_list.append(mask) - centric_list.append(centric) + fo_parts.append(F_obs) + fc_parts.append(F_calc) + sig_parts.append(sigma) + cen_parts.append(centric) + mask_parts.append(self._subset(data).mask) - # --- Stack into (N_tp, n_hkl) --- - F_obs = torch.stack(F_obs_list) - F_calc = torch.stack(F_calc_list) - sigma = torch.stack(sigma_list) - mask = torch.stack(mask_list) - centric = torch.stack(centric_list) + # Concatenate the data and the boolean masks, then mask the flat arrays + # once (compact — no MaskedTensors, no per-dataset Python reduction). + mask = torch.cat(mask_parts) + F_obs = torch.cat(fo_parts)[mask] + F_calc = torch.cat(fc_parts)[mask] + sigma = torch.cat(sig_parts)[mask] + centric = torch.cat(cen_parts)[mask] - # --- Apply mask via torch.where --- - F_obs = torch.where(mask, F_obs, torch.zeros_like(F_obs)) - F_calc = torch.where(mask, F_calc, torch.zeros_like(F_calc)) - sigma = torch.where(mask, sigma, torch.ones_like(sigma)) + if F_obs.numel() == 0: + return torch.tensor(0.0, device=mc.device) - # ML parameters (defaults) + # ML parameters (defaults): plain Rice, beta = sigma^2 beta = sigma**2 eb = beta.clamp(min=1e-6) @@ -292,10 +323,8 @@ def forward(self) -> torch.Tensor: loss = torch.where(centric, loss_centric, loss_acentric) loss = torch.where(torch.isfinite(loss), loss, torch.full_like(loss, 1e6)) - # Sum over valid reflections across all timepoints (unnormalised NLL) - total_nll = (loss * mask).sum() - - return total_nll + # Sum over all masked reflections (unnormalised NLL) + return loss.sum() # ========================================================================= @@ -303,7 +332,7 @@ def forward(self) -> torch.Tensor: # ========================================================================= -class CollectionMLTarget(Target): +class CollectionMLTarget(CollectionXrayTarget): """ Multi-dataset maximum-likelihood σ_A (Read MLF) target. @@ -317,9 +346,10 @@ class CollectionMLTarget(Target): The estimate is owned by **this target** (a :class:`SigmaAEstimator`), not the scaler — the scaler owns scaling only. The per-dataset loss is the Read MLF form (``mean = |Fc|``, variance ``epsilon*beta``) from - :func:`torchref.base.targets.xray_ml_sigmaa.ml_xray_loss_beta_math`, summed - over datasets. ``beta`` is detached (a constant in autograd); gradients reach - the models only through ``F_calc``. + :func:`torchref.base.targets.xray_ml_sigmaa.ml_xray_loss_beta_math`. Because + the loss is an independent per-dataset sum, the datasets are concatenated and + masked once (flat vectorization). ``beta`` is detached (a constant in + autograd); gradients reach the models only through ``F_calc``. Parameters ---------- @@ -331,7 +361,9 @@ class CollectionMLTarget(Target): Unused placeholder (kept for signature parity with the other collection targets, where the flag is also non-functional). TODO: remove from all three. use_work_set : bool - Compute loss only on the work set. + Legacy bool; superseded by ``use_set``. If True, loss on the work set. + use_set : str, optional + Canonical 3-way subset selector ``"work"``/``"free"``/``"val"``. verbose : int Verbosity level. base_weight : float, optional @@ -355,15 +387,19 @@ def __init__( scaler: "ScalerBase" = None, normalize: bool = True, use_work_set: bool = True, + use_set: str = None, verbose: int = 0, base_weight: float = None, ): - super().__init__(verbose=verbose) - self._dataset_collection = dataset_collection - self._model_collection = model_collection - self.add_module("_scaler", scaler) + super().__init__( + dataset_collection, + model_collection, + scaler=scaler, + use_work_set=use_work_set, + use_set=use_set, + verbose=verbose, + ) self.normalize = normalize - self.use_work_set = use_work_set self.base_weight = ( self.DEFAULT_BASE_WEIGHT if base_weight is None else float(base_weight) ) @@ -392,9 +428,7 @@ def forward(self) -> torch.Tensor: dc = self._dataset_collection mc = self._model_collection - tp_names = [n for n in mc.timepoint_names if n in dc] - # Include the dark/reference dataset too — it is a real dataset to fit. - all_keys = [mc.dark_key] + tp_names if mc.dark_key in dc else tp_names + all_keys = self._keys() if not all_keys: return torch.tensor(0.0, device=mc.device) @@ -402,44 +436,42 @@ def forward(self) -> torch.Tensor: dtype = dss_common.dtype # Clear any structure-factor cache populated under no_grad (e.g. by the - # scaler's joint initialization) so the F_calc computed below carries a - # live graph — otherwise gradients never reach the models. - for bm in getattr(mc, "base_models", []): - if hasattr(bm, "reset_cache"): - bm.reset_cache() + # scaler's joint initialization or a preceding stats() call) so the + # F_calc computed below carries a live graph. + self._reset_model_caches() # Compute each pair's F_calc once (with grad — used by the loss), and # collect detached copies to pool the free reflections for ONE shared # beta estimate across all datasets. - per_ds = [] # (F_obs, F_calc[grad], centric, mask) - fo_parts, fc_parts, cen_parts = [], [], [] + fo_parts, fc_parts, cen_parts, msk_parts = [], [], [], [] eps_parts, dss_parts, free_parts = [], [], [] for key in all_keys: data = dc[key] model = mc[key] - hkl = data.hkl - F_obs, _sigma, rfree, validity, centric = _unpack_masked_data(data) - F_obs = F_obs.to(dtype) - F_calc = torch.abs(_scale_fcalc(self._scaler, model(hkl), model)).to(dtype) + F_obs = data.get_corrected_data()[0].to(dtype) + F_calc = self._scaled_amp_full(data, model, recalc=False).to(dtype) + centric = data.centric if centric is None: - centric = torch.zeros(len(hkl), dtype=torch.bool, device=hkl.device) - mask = (validity & rfree) if self.use_work_set else validity - per_ds.append((F_obs, F_calc, centric, mask)) + centric = torch.zeros( + len(F_obs), dtype=torch.bool, device=F_obs.device + ) - free = validity & (~rfree) fo_parts.append(F_obs) - fc_parts.append(F_calc.detach()) # beta needs no gradient + fc_parts.append(F_calc) cen_parts.append(centric) + msk_parts.append(self._subset(data).mask) + + # Beta is estimated on the free set (validation excluded by data.free). + free_parts.append(data.free.mask) eps_parts.append(eps_common.to(dtype)) dss_parts.append(dss_common.to(dtype)) - free_parts.append(free) # One shared (beta, epsilon) for all datasets, mapped onto the common # HKL via target_dss. Detached; cached until maintenance() resets it. beta, eps = self._sigma_a.get( torch.cat(fo_parts), - torch.cat(fc_parts), + torch.cat([fc.detach() for fc in fc_parts]), # beta needs no gradient torch.cat(cen_parts), torch.cat(eps_parts), torch.cat(dss_parts), @@ -448,11 +480,20 @@ def forward(self) -> torch.Tensor: target_dss=dss_common, ) - total = torch.tensor(0.0, device=mc.device) - for F_obs, F_calc, centric, mask in per_ds: - b = beta.to(F_obs.dtype) - e = eps.to(F_obs.dtype) if eps is not None else None - total = total + ml_xray_loss_beta_math(F_obs, F_calc, b, centric, mask, e) + # Concatenate data + boolean masks across datasets and evaluate the + # Read-MLF loss once. beta/eps are on the common HKL, so they are tiled + # once per dataset to align with the concatenation order. + n_ds = len(all_keys) + F_obs_cat = torch.cat(fo_parts) + F_calc_cat = torch.cat(fc_parts) + centric_cat = torch.cat(cen_parts) + mask_cat = torch.cat(msk_parts) + beta_cat = beta.to(F_obs_cat.dtype).repeat(n_ds) + eps_cat = eps.to(F_obs_cat.dtype).repeat(n_ds) if eps is not None else None + + total = ml_xray_loss_beta_math( + F_obs_cat, F_calc_cat, beta_cat, centric_cat, mask_cat, eps_cat + ) # Base weight drives refinement; applied on the work set only. if self.use_work_set: @@ -466,8 +507,8 @@ def maintenance(self) -> None: self._sigma_a.reset() def stats(self) -> Dict[str, StatEntry]: - """Report shared beta diagnostics (low/high-resolution shell values).""" - out: Dict[str, StatEntry] = {} + """Base collection X-ray stats plus shared-beta diagnostics.""" + out = super().stats() bb = self._sigma_a.beta_per_bin if bb is not None and bb.numel() > 0: out["beta_bin0"] = stat(bb[0].item(), VERBOSITY_STANDARD) diff --git a/torchref/scaling/collection_scaler.py b/torchref/scaling/collection_scaler.py index 0087718..2683ea4 100644 --- a/torchref/scaling/collection_scaler.py +++ b/torchref/scaling/collection_scaler.py @@ -19,7 +19,7 @@ import torch import torch.nn as nn -from torchref.base.metrics import get_rfactors, nll_xray +from torchref.base.metrics.rfactor import rfactor_work_free from torchref.base.reciprocal import get_scattering_vectors from torchref.base.targets.xray_ml_sigmaa import ( SigmaAEstimator, @@ -144,18 +144,22 @@ def _calc_initial_scale_joint(self): data = dc[name] model = mc[name] - hkl, fobs, sigma, rfree = data(mask=False) + hkl = data.hkl + fobs = data.get_corrected_data()[0] with torch.no_grad(): fcalc = model(hkl) fcalc_amp = torch.abs(fcalc).clamp(min=1e-3).to(fobs.dtype) fobs_clamped = fobs.clamp(min=1e-3) - # Mask: work set, positive intensities + # Mask: work subset (validity + work, validation carved out), and + # positive intensities. ``data.work.mask`` is the standard subset + # boolean mask, replacing the ad-hoc ``masks() & rfree``. + work_mask = data.work.mask if hasattr(data, "I") and data.I is not None: pos_mask = data.I > 0 else: pos_mask = torch.ones_like(fobs, dtype=torch.bool) - mask = (data.masks() & rfree & pos_mask).to(torch.bool) + mask = (work_mask & pos_mask).to(torch.bool) bins = self.bins[mask].to(torch.int64) log_ratios = ( @@ -358,7 +362,6 @@ def refine_lbfgs_joint( # scale down (model under-scaled, R blow-up). fcalc_cache = {} fractions_cache = {} - data_cache = {} beta_cache = {} eps_cache = {} work_cache = {} @@ -370,7 +373,6 @@ def refine_lbfgs_joint( model = mc[name] hkl = data.hkl fobs, sigma = data.get_corrected_data() - rfree = data.rfree_flags with torch.no_grad(): fc = model(hkl).detach() fracs = model.fractions.detach() @@ -390,7 +392,6 @@ def refine_lbfgs_joint( ) fcalc_cache[name] = fc fractions_cache[name] = fracs - data_cache[name] = (fobs, sigma, rfree) beta_cache[name] = beta0 eps_cache[name] = eps0 work_cache[name] = data.work @@ -482,19 +483,21 @@ def forward(self): context="collection_scaler.refine_lbfgs_joint", ) - # Evaluate metrics once on the dark dataset after refinement. + # Evaluate metrics once on the dark dataset after refinement, through the + # shared ``rfactor_work_free`` source of truth (validity-masked work/free + # subsets, validation excluded from both) — the same partition the + # refinement targets report. with torch.no_grad(): dark_name = mc.dark_key if dark_name in fcalc_cache: fc = fcalc_cache[dark_name] fracs = fractions_cache[dark_name] - fobs, sigma, rfree = data_cache[dark_name] f_sol_raw = self.get_mixed_solvent_raw(fracs) scaled = super(CollectionScaler, self).forward( fc, f_sol_override=f_sol_raw ) - rwork, rfree_val = get_rfactors( - torch.abs(fobs), torch.abs(scaled), rfree + rwork, rfree_val = rfactor_work_free( + dc[dark_name], torch.abs(scaled) ) metrics["steps"].append(nsteps) metrics["rwork"].append(rwork) @@ -539,10 +542,12 @@ def screen_solvent_params_joint(self, steps: int = 15): model = mc[name] hkl = data.hkl fobs, sigma = data.get_corrected_data() - rfree = data.rfree_flags + work_mask = data.work.mask with torch.no_grad(): fc = model(hkl) - pairs.append((fc.detach(), model.fractions.detach(), fobs, sigma, rfree)) + pairs.append( + (fc.detach(), model.fractions.detach(), fobs, sigma, work_mask) + ) sol = self.solvent best_log_k = sol.log_k_solvent.clone() @@ -560,13 +565,13 @@ def screen_solvent_params_joint(self, steps: int = 15): sol.b_solvent.data = b.to(dtype=sol.b_solvent.dtype) total = 0.0 - for fc, fracs, fobs, sigma, rfree in pairs: + for fc, fracs, fobs, sigma, work_mask in pairs: f_sol_raw = self.get_mixed_solvent_raw(fracs) scaled = super(CollectionScaler, self).forward( fc, f_sol_override=f_sol_raw ) - diff = fobs[rfree] - torch.abs(scaled[rfree]) - sigma_safe = sigma[rfree].clamp(min=1e-3) + diff = fobs[work_mask] - torch.abs(scaled[work_mask]) + sigma_safe = sigma[work_mask].clamp(min=1e-3) total += (0.5 * (diff**2) / sigma_safe**2).mean().item() if total < best_loss: From 199c422fcfeb9e38f33890a9e472041f802b6bbc Mon Sep 17 00:00:00 2001 From: HatPdotS Date: Tue, 7 Jul 2026 16:16:41 +0200 Subject: [PATCH 4/4] Added changelog messgae about rfactor reporting in collection scaler --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 78601f5..4e4d40f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Version 0.6.1 ------------- - Fixed bug in beta estimation that caused instability in GPU refinement +- Fixed reporting bug in collection scaler were rfactor reporting would ignore masks Version 0.6.0