From 02e832b421e9dbec668fe3ee4b58ad730e958c04 Mon Sep 17 00:00:00 2001
From: Gregor Weiss <52161555+gregorweiss@users.noreply.github.com>
Date: Tue, 11 Aug 2026 20:22:05 +0200
Subject: [PATCH 1/5] chore: open PR for issue 38


From 63d88b1c89fdde502191c0dda15827d83ccaf5dd Mon Sep 17 00:00:00 2001
From: Gregor Weiss <52161555+gregorweiss@users.noreply.github.com>
Date: Tue, 11 Aug 2026 20:38:03 +0200
Subject: [PATCH 2/5] feat: add --clean flag to simulate and fix stale-cpt
 detection

Add clean_simulation_outputs() that removes stage outputs (tpr, cpt, log,
edr, gro, xtc/trr, rescue MDPs, GROMACS backups, mdout.mdp) while
preserving build inputs. Respects --stages filter and integrates with
--dry-run for preview.

Fix _detect_stage_state: trajectory stages (Production) with cpt+tpr but
no trajectory file now return not_started instead of partial, preventing
the stale-cpt-append crash where GROMACS refuses -append without a
trajectory to append to.
---
 mdfactory/cli.py                              |   9 +
 mdfactory/orchestration/__init__.py           |   3 +-
 mdfactory/orchestration/simulate.py           | 102 ++++++++
 .../tests/test_orchestration_cli_simulate.py  |  55 +++++
 .../tests/test_orchestration_simulate.py      | 217 ++++++++++++++++++
 5 files changed, 385 insertions(+), 1 deletion(-)

diff --git a/mdfactory/cli.py b/mdfactory/cli.py
index ab65115..e6e3863 100644
--- a/mdfactory/cli.py
+++ b/mdfactory/cli.py
@@ -574,6 +574,10 @@ def simulate_systems(
         bool,
         Parameter(help="Preview plan without executing"),
     ] = False,
+    clean: Annotated[
+        bool,
+        Parameter(help="Remove simulation outputs before running"),
+    ] = False,
     max_rescue: Annotated[
         int,
         Parameter(
@@ -609,6 +613,10 @@ def simulate_systems(
 
         mdfactory simulate output_dir/ --slurm gpu.yaml --dry-run
 
+    Clean before running::
+
+        mdfactory simulate output_dir/ --clean --stages Production
+
     """
     source = source.resolve()
 
@@ -646,6 +654,7 @@ def simulate_systems(
             stages=stages,
             checkpoint_mode=checkpoint,
             dry_run=dry_run,
+            clean=clean,
             max_rescue=max_rescue,
         )
     except ValueError as exc:
diff --git a/mdfactory/orchestration/__init__.py b/mdfactory/orchestration/__init__.py
index 2b32ec2..c5d40e5 100644
--- a/mdfactory/orchestration/__init__.py
+++ b/mdfactory/orchestration/__init__.py
@@ -4,13 +4,14 @@
 
 from .build import build_systems
 from .config import ExecutorConfig, SlurmExecutorConfig
-from .simulate import find_structure_file, run_simulations
+from .simulate import clean_simulation_outputs, find_structure_file, run_simulations
 from .tui import configure_and_save_slurm, configure_slurm_interactive
 
 __all__ = [
     "ExecutorConfig",
     "SlurmExecutorConfig",
     "build_systems",
+    "clean_simulation_outputs",
     "find_structure_file",
     "run_simulations",
     "configure_and_save_slurm",
diff --git a/mdfactory/orchestration/simulate.py b/mdfactory/orchestration/simulate.py
index d14bdcd..6b4ea7c 100644
--- a/mdfactory/orchestration/simulate.py
+++ b/mdfactory/orchestration/simulate.py
@@ -70,6 +70,7 @@ def run_simulations(
     stages: list[str] | None = None,
     wait: bool = True,
     dry_run: bool = False,
+    clean: bool = False,
     checkpoint_mode: str = "auto",
     max_rescue: int = 3,
 ) -> list[dict]:
@@ -87,6 +88,11 @@ def run_simulations(
         Wait for completion (default: True).
     dry_run : bool
         Preview plan without executing (default: False).
+    clean : bool
+        Remove simulation outputs before running (default: False).
+        Respects ``stages`` filter — only files belonging to requested
+        stages are deleted.  Combined with ``dry_run``, previews what
+        would be deleted without acting.
     checkpoint_mode : str
         - "auto": Skip stages with valid outputs (default)
         - "skip": Never re-run completed stages
@@ -148,6 +154,20 @@ def run_simulations(
         return skipped_results
     sim_paths = ready_paths
 
+    # 1b. Clean outputs (if requested) before checkpoint detection.
+    if clean:
+        for sim_dir in sim_paths:
+            deleted = clean_simulation_outputs(sim_dir, stages, dry_run=dry_run)
+            if dry_run and deleted:
+                logger.info(
+                    f"Would delete {len(deleted)} file(s) from {sim_dir.name}: "
+                    + ", ".join(p.name for p in deleted)
+                )
+        if dry_run:
+            # After previewing deletions, still show the dry-run execution plan
+            # (which will report all stages as needed since nothing was deleted).
+            pass
+
     # 2. Checkpoint detection (includes restart info for -cpi -append support)
     work_plan = []
     for sim_dir in sim_paths:
@@ -279,6 +299,83 @@ def _missing_build_files(sim_dir: Path, stages: list[str]) -> list[str]:
     return [f for f in required if not (sim_dir / f).exists()]
 
 
+def clean_simulation_outputs(
+    sim_dir: Path,
+    stages: list[str],
+    *,
+    dry_run: bool = False,
+) -> list[Path]:
+    """Remove simulation outputs for the given stages, preserving build inputs.
+
+    Derives deletable files from :data:`STAGE_BY_NAME` (tpr, cpt, log, edr,
+    gro, trr, xtc) plus ``mdout.mdp``, rescue-tier MDPs, and GROMACS backup
+    files (``#*#``).
+
+    Parameters
+    ----------
+    sim_dir : Path
+        Simulation directory.
+    stages : list[str]
+        Stage names whose outputs should be removed.
+    dry_run : bool
+        If ``True``, collect and return the list of files that *would* be
+        deleted without actually removing them.
+
+    Returns
+    -------
+    list[Path]
+        Paths that were deleted (or would be deleted in dry-run mode).
+
+    """
+    to_delete: list[Path] = []
+
+    for stage_name in stages:
+        spec = STAGE_BY_NAME[stage_name]
+        deffnm = spec.deffnm
+
+        # Named output files: tpr, cpt, log, edr
+        for ext in ("tpr", "cpt", "log", "edr"):
+            to_delete.append(sim_dir / f"{deffnm}.{ext}")
+
+        # Structure output (EM/NVT/NPT)
+        if spec.gro_out:
+            to_delete.append(sim_dir / spec.gro_out)
+
+        # Trajectory outputs (Production)
+        for traj in spec.traj_files:
+            to_delete.append(sim_dir / traj)
+
+        # Rescue-tier MDPs (e.g. em_rescue_t1.mdp, em_rescue_t2.mdp)
+        mdp_stem = spec.mdp_file.rsplit(".", 1)[0]
+        to_delete.extend(sim_dir.glob(f"{mdp_stem}_rescue_t*.mdp"))
+
+        # GROMACS backup files (#deffnm.*#)
+        to_delete.extend(sim_dir.glob(f"#{deffnm}.*#"))
+
+    # mdout.mdp — grompp output, always regenerated
+    to_delete.append(sim_dir / "mdout.mdp")
+
+    # Deduplicate (glob results may overlap with named files) and filter to
+    # files that actually exist.
+    seen: set[Path] = set()
+    existing: list[Path] = []
+    for p in to_delete:
+        if p not in seen and p.exists():
+            seen.add(p)
+            existing.append(p)
+
+    if not dry_run:
+        for p in existing:
+            p.unlink()
+            logger.debug(f"Deleted {p}")
+        if existing:
+            logger.info(
+                f"Cleaned {len(existing)} file(s) from {sim_dir.name}"
+            )
+
+    return existing
+
+
 def _validate_simulation_dir(sim_dir: Path, stages: list[str]) -> None:
     """Raise if any required build output files are missing.
 
@@ -496,6 +593,11 @@ def _detect_stage_state(sim_dir: Path, stage: str, mode: str = "auto") -> dict:
 
     # Check partial progress (checkpoint exists, output doesn't).
     if cpt_file.exists() and tpr_file.exists():
+        if spec.traj_files:
+            # Trajectory stage (Production): stale checkpoint without a
+            # trajectory file cannot use -append — GROMACS would crash.
+            # Treat as not_started so grompp+mdrun run from scratch.
+            return {"status": "not_started", "cpt_file": None, "restart": False}
         return {"status": "partial", "cpt_file": cpt_file, "restart": True}
 
     return {"status": "not_started", "cpt_file": None, "restart": False}
diff --git a/mdfactory/tests/test_orchestration_cli_simulate.py b/mdfactory/tests/test_orchestration_cli_simulate.py
index de2afda..f1197ec 100644
--- a/mdfactory/tests/test_orchestration_cli_simulate.py
+++ b/mdfactory/tests/test_orchestration_cli_simulate.py
@@ -275,3 +275,58 @@ def test_simulate_systems_max_rescue_default_is_3(mock_slurm, mock_config, mock_
     mock_run.assert_called_once()
     _, kwargs = mock_run.call_args
     assert kwargs["max_rescue"] == 3
+
+
+@patch("mdfactory.orchestration.run_simulations", return_value=_SUCCESS_RESULTS)
+@patch("mdfactory.cli._load_executor_config", return_value=MagicMock(provider="local"))
+@patch("mdfactory.cli._resolve_slurm_flag", return_value=None)
+def test_simulate_systems_forwards_clean_flag(mock_slurm, mock_config, mock_run, tmp_path):
+    """--clean is forwarded from CLI to run_simulations."""
+    from mdfactory.cli import simulate_systems
+
+    sim_dir = tmp_path / "abc123"
+    sim_dir.mkdir()
+    (sim_dir / "system.pdb").touch()
+
+    simulate_systems(source=tmp_path, clean=True)
+
+    mock_run.assert_called_once()
+    _, kwargs = mock_run.call_args
+    assert kwargs["clean"] is True
+
+
+@patch("mdfactory.orchestration.run_simulations", return_value=_SUCCESS_RESULTS)
+@patch("mdfactory.cli._load_executor_config", return_value=MagicMock(provider="local"))
+@patch("mdfactory.cli._resolve_slurm_flag", return_value=None)
+def test_simulate_systems_clean_default_is_false(mock_slurm, mock_config, mock_run, tmp_path):
+    """--clean defaults to False when not specified."""
+    from mdfactory.cli import simulate_systems
+
+    sim_dir = tmp_path / "abc123"
+    sim_dir.mkdir()
+    (sim_dir / "system.pdb").touch()
+
+    simulate_systems(source=tmp_path)
+
+    mock_run.assert_called_once()
+    _, kwargs = mock_run.call_args
+    assert kwargs["clean"] is False
+
+
+@patch("mdfactory.orchestration.run_simulations", return_value=[])
+@patch("mdfactory.cli._load_executor_config", return_value=MagicMock(provider="local"))
+@patch("mdfactory.cli._resolve_slurm_flag", return_value=None)
+def test_simulate_systems_clean_with_dry_run(mock_slurm, mock_config, mock_run, tmp_path):
+    """--clean combined with --dry-run passes both flags through."""
+    from mdfactory.cli import simulate_systems
+
+    sim_dir = tmp_path / "abc123"
+    sim_dir.mkdir()
+    (sim_dir / "system.pdb").touch()
+
+    simulate_systems(source=tmp_path, clean=True, dry_run=True)
+
+    mock_run.assert_called_once()
+    _, kwargs = mock_run.call_args
+    assert kwargs["clean"] is True
+    assert kwargs["dry_run"] is True
diff --git a/mdfactory/tests/test_orchestration_simulate.py b/mdfactory/tests/test_orchestration_simulate.py
index bc2f3ac..cbbc2d4 100644
--- a/mdfactory/tests/test_orchestration_simulate.py
+++ b/mdfactory/tests/test_orchestration_simulate.py
@@ -1849,3 +1849,220 @@ def test_extract_resource_hints_divides_by_max_workers():
     hints = _extract_resource_hints(cfg)
 
     assert hints.ntasks == 6  # 12 // 2
+
+
+# --- clean_simulation_outputs tests ---
+
+
+def _populate_sim_dir(sim_dir):
+    """Create a fake simulation directory with build inputs and all stage outputs."""
+    # Build inputs (must be preserved)
+    (sim_dir / "system.pdb").write_bytes(b"pdb")
+    (sim_dir / "topology.top").write_bytes(b"top")
+    (sim_dir / "em.mdp").write_bytes(b"mdp")
+    (sim_dir / "nvt.mdp").write_bytes(b"mdp")
+    (sim_dir / "npt.mdp").write_bytes(b"mdp")
+    (sim_dir / "md.mdp").write_bytes(b"mdp")
+    (sim_dir / "posres.itp").write_bytes(b"itp")
+    (sim_dir / "config.yaml").write_bytes(b"yaml")
+
+    # Simulation outputs per stage
+    for deffnm in ("min", "nvt", "npt", "prod"):
+        for ext in ("tpr", "cpt", "log", "edr"):
+            (sim_dir / f"{deffnm}.{ext}").write_bytes(b"x")
+
+    # Structure outputs (EM/NVT/NPT)
+    for deffnm in ("min", "nvt", "npt"):
+        (sim_dir / f"{deffnm}.gro").write_bytes(b"x")
+
+    # Trajectory outputs (Production)
+    (sim_dir / "prod.xtc").write_bytes(b"x")
+    (sim_dir / "prod.trr").write_bytes(b"x")
+
+    # Rescue MDPs
+    (sim_dir / "em_rescue_t1.mdp").write_bytes(b"x")
+    (sim_dir / "em_rescue_t2.mdp").write_bytes(b"x")
+    (sim_dir / "nvt_rescue_t1.mdp").write_bytes(b"x")
+
+    # GROMACS backup files
+    (sim_dir / "#min.tpr.1#").write_bytes(b"x")
+    (sim_dir / "#prod.xtc.1#").write_bytes(b"x")
+
+    # grompp output
+    (sim_dir / "mdout.mdp").write_bytes(b"x")
+
+
+def test_clean_simulation_outputs_all_stages(tmp_path):
+    """All stage outputs removed; build inputs preserved."""
+    from mdfactory.orchestration.simulate import clean_simulation_outputs
+
+    sim_dir = tmp_path / "sim"
+    sim_dir.mkdir()
+    _populate_sim_dir(sim_dir)
+
+    all_stages = ["EM", "NVT", "NPT", "Production"]
+    deleted = clean_simulation_outputs(sim_dir, all_stages)
+
+    # Build inputs must survive
+    assert (sim_dir / "system.pdb").exists()
+    assert (sim_dir / "topology.top").exists()
+    assert (sim_dir / "em.mdp").exists()
+    assert (sim_dir / "nvt.mdp").exists()
+    assert (sim_dir / "npt.mdp").exists()
+    assert (sim_dir / "md.mdp").exists()
+    assert (sim_dir / "posres.itp").exists()
+    assert (sim_dir / "config.yaml").exists()
+
+    # Simulation outputs must be gone
+    for deffnm in ("min", "nvt", "npt", "prod"):
+        for ext in ("tpr", "cpt", "log", "edr"):
+            assert not (sim_dir / f"{deffnm}.{ext}").exists()
+
+    for deffnm in ("min", "nvt", "npt"):
+        assert not (sim_dir / f"{deffnm}.gro").exists()
+
+    assert not (sim_dir / "prod.xtc").exists()
+    assert not (sim_dir / "prod.trr").exists()
+
+    # Rescue MDPs and backups must be gone
+    assert not (sim_dir / "em_rescue_t1.mdp").exists()
+    assert not (sim_dir / "em_rescue_t2.mdp").exists()
+    assert not (sim_dir / "nvt_rescue_t1.mdp").exists()
+    assert not (sim_dir / "#min.tpr.1#").exists()
+    assert not (sim_dir / "#prod.xtc.1#").exists()
+    assert not (sim_dir / "mdout.mdp").exists()
+
+    assert len(deleted) > 0
+
+
+def test_clean_simulation_outputs_filtered_stages(tmp_path):
+    """Clean only Production: prod.* removed; min/nvt/npt files preserved."""
+    from mdfactory.orchestration.simulate import clean_simulation_outputs
+
+    sim_dir = tmp_path / "sim"
+    sim_dir.mkdir()
+    _populate_sim_dir(sim_dir)
+
+    deleted = clean_simulation_outputs(sim_dir, ["Production"])
+
+    # Production outputs must be gone
+    assert not (sim_dir / "prod.tpr").exists()
+    assert not (sim_dir / "prod.cpt").exists()
+    assert not (sim_dir / "prod.log").exists()
+    assert not (sim_dir / "prod.edr").exists()
+    assert not (sim_dir / "prod.xtc").exists()
+    assert not (sim_dir / "prod.trr").exists()
+    assert not (sim_dir / "#prod.xtc.1#").exists()
+
+    # Other stages must survive
+    assert (sim_dir / "min.tpr").exists()
+    assert (sim_dir / "min.gro").exists()
+    assert (sim_dir / "nvt.cpt").exists()
+    assert (sim_dir / "npt.gro").exists()
+
+    # EM rescue MDPs must survive (not in requested stages)
+    assert (sim_dir / "em_rescue_t1.mdp").exists()
+    assert (sim_dir / "em_rescue_t2.mdp").exists()
+
+    # mdout.mdp is always deleted
+    assert not (sim_dir / "mdout.mdp").exists()
+
+    deleted_names = {p.name for p in deleted}
+    assert "prod.tpr" in deleted_names
+    assert "min.tpr" not in deleted_names
+
+
+def test_clean_simulation_outputs_missing_files(tmp_path):
+    """Partial outputs (some absent) — no errors, remaining files cleaned."""
+    from mdfactory.orchestration.simulate import clean_simulation_outputs
+
+    sim_dir = tmp_path / "sim"
+    sim_dir.mkdir()
+    # Only create a subset of files
+    (sim_dir / "system.pdb").write_bytes(b"pdb")
+    (sim_dir / "topology.top").write_bytes(b"top")
+    (sim_dir / "min.tpr").write_bytes(b"x")
+    # min.cpt, min.gro, etc. intentionally absent
+
+    deleted = clean_simulation_outputs(sim_dir, ["EM"])
+
+    assert not (sim_dir / "min.tpr").exists()
+    assert (sim_dir / "system.pdb").exists()
+    assert len(deleted) == 1
+
+
+def test_clean_simulation_outputs_preserves_build_inputs(tmp_path):
+    """Explicitly verify all build input types survive cleaning."""
+    from mdfactory.orchestration.simulate import clean_simulation_outputs
+
+    sim_dir = tmp_path / "sim"
+    sim_dir.mkdir()
+    _populate_sim_dir(sim_dir)
+
+    clean_simulation_outputs(sim_dir, ["EM", "NVT", "NPT", "Production"])
+
+    # Every build input must still exist
+    for name in ("system.pdb", "topology.top", "em.mdp", "nvt.mdp", "npt.mdp",
+                 "md.mdp", "posres.itp", "config.yaml"):
+        assert (sim_dir / name).exists(), f"{name} should be preserved"
+
+
+def test_clean_simulation_outputs_dry_run(tmp_path):
+    """Dry-run mode returns files that would be deleted but doesn't remove them."""
+    from mdfactory.orchestration.simulate import clean_simulation_outputs
+
+    sim_dir = tmp_path / "sim"
+    sim_dir.mkdir()
+    _populate_sim_dir(sim_dir)
+
+    would_delete = clean_simulation_outputs(
+        sim_dir, ["EM", "NVT", "NPT", "Production"], dry_run=True
+    )
+
+    # Files must still exist
+    assert (sim_dir / "min.tpr").exists()
+    assert (sim_dir / "prod.xtc").exists()
+    assert (sim_dir / "mdout.mdp").exists()
+
+    # But the return value should list them
+    assert len(would_delete) > 0
+    deleted_names = {p.name for p in would_delete}
+    assert "min.tpr" in deleted_names
+    assert "prod.xtc" in deleted_names
+
+
+# --- _detect_stage_state trajectory stale-cpt fix tests ---
+
+
+def test_detect_stage_state_trajectory_stale_cpt(tmp_path):
+    """Production with cpt+tpr but no trajectory → not_started (stale-cpt fix)."""
+    from mdfactory.orchestration.simulate import _detect_stage_state
+
+    sim_dir = tmp_path / "sim"
+    sim_dir.mkdir()
+    (sim_dir / "prod.cpt").write_bytes(b"x")
+    (sim_dir / "prod.tpr").write_bytes(b"x")
+    # prod.xtc / prod.trr intentionally absent
+
+    state = _detect_stage_state(sim_dir, "Production", "auto")
+
+    assert state["status"] == "not_started"
+    assert state["restart"] is False
+    assert state["cpt_file"] is None
+
+
+def test_detect_stage_state_structure_partial_cpt_unchanged(tmp_path):
+    """EM with cpt+tpr but no gro → still partial (structure stages unchanged)."""
+    from mdfactory.orchestration.simulate import _detect_stage_state
+
+    sim_dir = tmp_path / "sim"
+    sim_dir.mkdir()
+    (sim_dir / "min.cpt").write_bytes(b"x")
+    (sim_dir / "min.tpr").write_bytes(b"x")
+    # min.gro intentionally absent
+
+    state = _detect_stage_state(sim_dir, "EM", "auto")
+
+    assert state["status"] == "partial"
+    assert state["restart"] is True
+    assert state["cpt_file"] == sim_dir / "min.cpt"

From dc847d43868ef4dd3b51b2e48b16c3e8b64f1b30 Mon Sep 17 00:00:00 2001
From: Gregor Weiss <52161555+gregorweiss@users.noreply.github.com>
Date: Tue, 11 Aug 2026 20:43:57 +0200
Subject: [PATCH 3/5] docs: add --clean flag to CLI reference, troubleshooting,
 and HPC guide

---
 .../content/docs/user-guide/cli-reference.mdx | 10 ++++++++
 .../docs/user-guide/running-on-hpc.mdx        |  8 +++++++
 .../docs/user-guide/troubleshooting.mdx       | 23 +++++++++++++++++++
 3 files changed, 41 insertions(+)

diff --git a/docs/content/docs/user-guide/cli-reference.mdx b/docs/content/docs/user-guide/cli-reference.mdx
index b351691..b16a7d6 100644
--- a/docs/content/docs/user-guide/cli-reference.mdx
+++ b/docs/content/docs/user-guide/cli-reference.mdx
@@ -58,6 +58,7 @@ Options:
 - `--checkpoint` — checkpoint mode: `auto` (default), `skip`, `force`
 - `--hash` — filter by hash prefix (can specify multiple)
 - `--dry-run` — preview plan without executing
+- `--clean` — remove simulation outputs before running (respects `--stages` filter)
 - `--max-rescue` — max rescue tiers for physics failures (default: 3; set to 0 to disable)
 
 Usage examples:
@@ -80,6 +81,15 @@ mdfactory simulate output_dir/ --hash abc123 def456
 
 # Dry-run preview
 mdfactory simulate output_dir/ --slurm gpu.yaml --dry-run
+
+# Clean outputs and re-run (e.g. after a killed job left stale checkpoints)
+mdfactory simulate output_dir/ --clean --slurm gpu.yaml
+
+# Clean only Production stage before re-running it
+mdfactory simulate output_dir/ --clean --stages Production --slurm gpu.yaml
+
+# Preview what --clean would delete without acting
+mdfactory simulate output_dir/ --clean --dry-run
 ```
 
 ## Configuration commands
diff --git a/docs/content/docs/user-guide/running-on-hpc.mdx b/docs/content/docs/user-guide/running-on-hpc.mdx
index 0eec38b..88f8fac 100644
--- a/docs/content/docs/user-guide/running-on-hpc.mdx
+++ b/docs/content/docs/user-guide/running-on-hpc.mdx
@@ -205,6 +205,14 @@ Simulations automatically resume from checkpoints. If a job is interrupted, re-r
 mdfactory simulate results/ --slurm slurm_executor.yaml --checkpoint auto
 ```
 
+If a job was killed mid-flight and left stale checkpoints (e.g. `.cpt` files without matching trajectory files), use `--clean` to reset before re-running:
+
+```bash
+mdfactory simulate results/ --clean --slurm slurm_executor.yaml
+```
+
+This removes simulation outputs while preserving build inputs. Use `--clean --dry-run` to preview what would be deleted. The `--stages` filter is respected — `--clean --stages Production` only cleans Production files.
+
 ## Submit analyses with submitit
 
 For a full walkthrough of running analyses, see the
diff --git a/docs/content/docs/user-guide/troubleshooting.mdx b/docs/content/docs/user-guide/troubleshooting.mdx
index 7034668..7a03d33 100644
--- a/docs/content/docs/user-guide/troubleshooting.mdx
+++ b/docs/content/docs/user-guide/troubleshooting.mdx
@@ -108,6 +108,29 @@ To skip validation and rely only on file existence:
 mdfactory simulate output_dir/ --slurm gpu.yaml --checkpoint skip
 ```
 
+</Accordion>
+<Accordion title="Stale checkpoints after a killed job">
+
+When a SLURM job is cancelled mid-flight, checkpoint files (`.cpt`) may be left behind
+without corresponding trajectory files. Use `--clean` to reset simulation directories to
+post-build state before re-running:
+
+```bash
+# Clean all stages and re-run
+mdfactory simulate output_dir/ --clean --slurm gpu.yaml
+
+# Clean only Production and re-run it
+mdfactory simulate output_dir/ --clean --stages Production --slurm gpu.yaml
+
+# Preview what would be deleted
+mdfactory simulate output_dir/ --clean --dry-run
+```
+
+The `--clean` flag removes simulation outputs (`.tpr`, `.cpt`, `.log`, `.edr`, `.gro`,
+`.xtc`, `.trr`, rescue MDPs, GROMACS backup files) while preserving build inputs
+(`system.pdb`, `topology.top`, template `.mdp` files, `.itp`, `.yaml`). It respects
+the `--stages` filter — only files belonging to requested stages are deleted.
+
 </Accordion>
 <Accordion title="Simulation fails with LINCS or constraint errors">
 

From a2ec675864e1cb55fd34881a6c1a16e83d8d1f49 Mon Sep 17 00:00:00 2001
From: Gregor Weiss <52161555+gregorweiss@users.noreply.github.com>
Date: Tue, 11 Aug 2026 21:04:23 +0200
Subject: [PATCH 4/5] test: add integration tests for clean=True wiring in
 run_simulations

---
 .../tests/test_orchestration_simulate.py      | 55 +++++++++++++++++++
 1 file changed, 55 insertions(+)

diff --git a/mdfactory/tests/test_orchestration_simulate.py b/mdfactory/tests/test_orchestration_simulate.py
index cbbc2d4..5bb5b1e 100644
--- a/mdfactory/tests/test_orchestration_simulate.py
+++ b/mdfactory/tests/test_orchestration_simulate.py
@@ -2066,3 +2066,58 @@ def test_detect_stage_state_structure_partial_cpt_unchanged(tmp_path):
     assert state["status"] == "partial"
     assert state["restart"] is True
     assert state["cpt_file"] == sim_dir / "min.cpt"
+
+
+# --- run_simulations clean integration tests ---
+
+
+@patch("mdfactory.orchestration.simulate.parsl_session")
+@patch("mdfactory.orchestration.simulate._execute_stage_list")
+def test_run_simulations_clean_deletes_before_checkpoint_detection(
+    mock_execute, mock_session, tmp_path
+):
+    """run_simulations(clean=True) deletes outputs then detects all stages needed."""
+    sim_dir = tmp_path / "sim"
+    sim_dir.mkdir()
+    _populate_sim_dir(sim_dir)
+
+    mock_future = MagicMock()
+    mock_future.done.return_value = True
+    mock_future.result.return_value = {"status": "success"}
+    mock_execute.return_value = mock_future
+    mock_session.return_value.__enter__.return_value = MagicMock()
+
+    config = ExecutorConfig()
+    run_simulations([sim_dir], config, clean=True)
+
+    # Verify files were actually deleted before execution
+    assert not (sim_dir / "min.tpr").exists()
+    assert not (sim_dir / "prod.xtc").exists()
+    assert not (sim_dir / "mdout.mdp").exists()
+    # Build inputs preserved
+    assert (sim_dir / "system.pdb").exists()
+    assert (sim_dir / "topology.top").exists()
+
+    # All stages should have been submitted (clean removed all outputs)
+    mock_execute.assert_called_once()
+    stages_arg = mock_execute.call_args[0][1]
+    assert set(stages_arg) == {"EM", "NVT", "NPT", "Production"}
+
+
+def test_run_simulations_clean_dry_run_preserves_files(tmp_path):
+    """run_simulations(clean=True, dry_run=True) previews without removing files."""
+    sim_dir = tmp_path / "sim"
+    sim_dir.mkdir()
+    _populate_sim_dir(sim_dir)
+
+    config = ExecutorConfig()
+    results = run_simulations([sim_dir], config, clean=True, dry_run=True)
+
+    # dry_run=True propagates to clean_simulation_outputs — files NOT deleted
+    assert (sim_dir / "min.gro").exists()
+    assert (sim_dir / "min.tpr").exists()
+    assert (sim_dir / "prod.xtc").exists()
+
+    # Still returns a work plan (dry-run shows what would run)
+    plan_items = [r for r in results if "sim_dir" in r]
+    assert len(plan_items) == 1

From d702b672162aa0b5d1c83cd28160a0cbce905529 Mon Sep 17 00:00:00 2001
From: Gregor Weiss <52161555+gregorweiss@users.noreply.github.com>
Date: Tue, 11 Aug 2026 21:06:19 +0200
Subject: [PATCH 5/5] style: apply ruff formatting

---
 mdfactory/orchestration/simulate.py           | 17 +++------
 .../tests/test_orchestration_simulate.py      | 36 ++++++++++---------
 2 files changed, 24 insertions(+), 29 deletions(-)

diff --git a/mdfactory/orchestration/simulate.py b/mdfactory/orchestration/simulate.py
index 6b4ea7c..a72e109 100644
--- a/mdfactory/orchestration/simulate.py
+++ b/mdfactory/orchestration/simulate.py
@@ -236,8 +236,7 @@ def _run_pipeline(item):
 
             with ThreadPoolExecutor() as pool:
                 thread_futs = [
-                    (item["hash"], pool.submit(_run_pipeline, item))
-                    for item in active_items
+                    (item["hash"], pool.submit(_run_pipeline, item)) for item in active_items
                 ]
                 for h, tf in thread_futs:
                     try:
@@ -369,9 +368,7 @@ def clean_simulation_outputs(
             p.unlink()
             logger.debug(f"Deleted {p}")
         if existing:
-            logger.info(
-                f"Cleaned {len(existing)} file(s) from {sim_dir.name}"
-            )
+            logger.info(f"Cleaned {len(existing)} file(s) from {sim_dir.name}")
 
     return existing
 
@@ -771,11 +768,7 @@ def _execute_stage_list(
         cpt_file = restarts.get(stage, "")
 
         # Use rescue loop for eligible stages without checkpoint restart
-        if (
-            max_rescue > 0
-            and stage in RESCUE_ELIGIBLE_STAGES
-            and not cpt_file
-        ):
+        if max_rescue > 0 and stage in RESCUE_ELIGIBLE_STAGES and not cpt_file:
             prev_future = execute_stage_with_rescue(
                 sim_dir,
                 stage,
@@ -788,9 +781,7 @@ def _execute_stage_list(
         else:
             # Standard execution: chain futures without waiting
             stage_cfg = (
-                config.get_stage_config(stage)
-                if hasattr(config, "get_stage_config")
-                else None
+                config.get_stage_config(stage) if hasattr(config, "get_stage_config") else None
             )
             cfg_kwarg = {"stage_config": stage_cfg} if stage_cfg is not None else {}
             prev_future = run_stage(
diff --git a/mdfactory/tests/test_orchestration_simulate.py b/mdfactory/tests/test_orchestration_simulate.py
index 5bb5b1e..ca458df 100644
--- a/mdfactory/tests/test_orchestration_simulate.py
+++ b/mdfactory/tests/test_orchestration_simulate.py
@@ -797,17 +797,13 @@ def test_execute_stage_list_partial_pipeline_from_checkpoint(mock_run_stage):
 
 @patch("mdfactory.orchestration.rescue.execute_stage_with_rescue")
 @patch("mdfactory.orchestration.simulate.run_stage")
-def test_execute_stage_list_routes_to_rescue_when_enabled(
-    mock_run_stage, mock_rescue
-):
+def test_execute_stage_list_routes_to_rescue_when_enabled(mock_run_stage, mock_rescue):
     """Rescue-eligible stages dispatch to execute_stage_with_rescue when max_rescue > 0."""
     rescue_future = MagicMock()
     mock_rescue.return_value = rescue_future
     sim_dir = Path("/tmp/test")
 
-    result = _execute_stage_list(
-        sim_dir, ["EM"], MagicMock(), MagicMock(), max_rescue=3
-    )
+    result = _execute_stage_list(sim_dir, ["EM"], MagicMock(), MagicMock(), max_rescue=3)
 
     mock_rescue.assert_called_once()
     mock_run_stage.assert_not_called()
@@ -816,17 +812,13 @@ def test_execute_stage_list_routes_to_rescue_when_enabled(
 
 @patch("mdfactory.orchestration.rescue.execute_stage_with_rescue")
 @patch("mdfactory.orchestration.simulate.run_stage")
-def test_execute_stage_list_production_bypasses_rescue(
-    mock_run_stage, mock_rescue
-):
+def test_execute_stage_list_production_bypasses_rescue(mock_run_stage, mock_rescue):
     """Production stage uses run_stage even when max_rescue > 0."""
     prod_future = MagicMock()
     mock_run_stage.return_value = prod_future
     sim_dir = Path("/tmp/test")
 
-    result = _execute_stage_list(
-        sim_dir, ["Production"], MagicMock(), MagicMock(), max_rescue=3
-    )
+    result = _execute_stage_list(sim_dir, ["Production"], MagicMock(), MagicMock(), max_rescue=3)
 
     mock_run_stage.assert_called_once()
     mock_rescue.assert_not_called()
@@ -1416,8 +1408,12 @@ def test_execute_stage_list_no_stage_config_for_local(mock_run_stage):
     mock_run_stage.return_value = MagicMock()
 
     _execute_stage_list(
-        Path("/tmp/test"), ["EM"], MagicMock(), MagicMock(),
-        config=ExecutorConfig(), max_rescue=0,
+        Path("/tmp/test"),
+        ["EM"],
+        MagicMock(),
+        MagicMock(),
+        config=ExecutorConfig(),
+        max_rescue=0,
     )
 
     _, call_kwargs = mock_run_stage.call_args
@@ -2002,8 +1998,16 @@ def test_clean_simulation_outputs_preserves_build_inputs(tmp_path):
     clean_simulation_outputs(sim_dir, ["EM", "NVT", "NPT", "Production"])
 
     # Every build input must still exist
-    for name in ("system.pdb", "topology.top", "em.mdp", "nvt.mdp", "npt.mdp",
-                 "md.mdp", "posres.itp", "config.yaml"):
+    for name in (
+        "system.pdb",
+        "topology.top",
+        "em.mdp",
+        "nvt.mdp",
+        "npt.mdp",
+        "md.mdp",
+        "posres.itp",
+        "config.yaml",
+    ):
         assert (sim_dir / name).exists(), f"{name} should be preserved"