feat: add --clean flag to simulate for resetting output directories

docs/content/docs/user-guide/cli-reference.mdx

@@ -58,6 +58,7 @@ Options:
 - `--checkpoint` — checkpoint mode: `auto` (default), `skip`, `force`
 - `--hash` — filter by hash prefix (can specify multiple)
 - `--dry-run` — preview plan without executing
+- `--clean` — remove simulation outputs before running (respects `--stages` filter)
 - `--max-rescue` — max rescue tiers for physics failures (default: 3; set to 0 to disable)
 
 Usage examples:
@@ -80,6 +81,15 @@ mdfactory simulate output_dir/ --hash abc123 def456
 
 # Dry-run preview
 mdfactory simulate output_dir/ --slurm gpu.yaml --dry-run
+
+# Clean outputs and re-run (e.g. after a killed job left stale checkpoints)
+mdfactory simulate output_dir/ --clean --slurm gpu.yaml
+
+# Clean only Production stage before re-running it
+mdfactory simulate output_dir/ --clean --stages Production --slurm gpu.yaml
+
+# Preview what --clean would delete without acting
+mdfactory simulate output_dir/ --clean --dry-run
 ```
 
 ## Configuration commands

docs/content/docs/user-guide/running-on-hpc.mdx

@@ -205,6 +205,14 @@ Simulations automatically resume from checkpoints. If a job is interrupted, re-r
 mdfactory simulate results/ --slurm slurm_executor.yaml --checkpoint auto
 ```
 
+If a job was killed mid-flight and left stale checkpoints (e.g. `.cpt` files without matching trajectory files), use `--clean` to reset before re-running:
+
+```bash
+mdfactory simulate results/ --clean --slurm slurm_executor.yaml
+```
+
+This removes simulation outputs while preserving build inputs. Use `--clean --dry-run` to preview what would be deleted. The `--stages` filter is respected — `--clean --stages Production` only cleans Production files.
+
 ## Submit analyses with submitit
 
 For a full walkthrough of running analyses, see the

docs/content/docs/user-guide/troubleshooting.mdx

@@ -108,6 +108,29 @@ To skip validation and rely only on file existence:
 mdfactory simulate output_dir/ --slurm gpu.yaml --checkpoint skip
 ```
 
+</Accordion>
+<Accordion title="Stale checkpoints after a killed job">
+
+When a SLURM job is cancelled mid-flight, checkpoint files (`.cpt`) may be left behind
+without corresponding trajectory files. Use `--clean` to reset simulation directories to
+post-build state before re-running:
+
+```bash
+# Clean all stages and re-run
+mdfactory simulate output_dir/ --clean --slurm gpu.yaml
+
+# Clean only Production and re-run it
+mdfactory simulate output_dir/ --clean --stages Production --slurm gpu.yaml
+
+# Preview what would be deleted
+mdfactory simulate output_dir/ --clean --dry-run
+```
+
+The `--clean` flag removes simulation outputs (`.tpr`, `.cpt`, `.log`, `.edr`, `.gro`,
+`.xtc`, `.trr`, rescue MDPs, GROMACS backup files) while preserving build inputs
+(`system.pdb`, `topology.top`, template `.mdp` files, `.itp`, `.yaml`). It respects
+the `--stages` filter — only files belonging to requested stages are deleted.
+
 </Accordion>
 <Accordion title="Simulation fails with LINCS or constraint errors">
 

mdfactory/cli.py

@@ -574,6 +574,10 @@ def simulate_systems(
         bool,
         Parameter(help="Preview plan without executing"),
     ] = False,
+    clean: Annotated[
+        bool,
+        Parameter(help="Remove simulation outputs before running"),
+    ] = False,
     max_rescue: Annotated[
         int,
         Parameter(
@@ -609,6 +613,10 @@ def simulate_systems(
 
         mdfactory simulate output_dir/ --slurm gpu.yaml --dry-run
 
+    Clean before running::
+
+        mdfactory simulate output_dir/ --clean --stages Production
+
     """
     source = source.resolve()
 
@@ -646,6 +654,7 @@ def simulate_systems(
             stages=stages,
             checkpoint_mode=checkpoint,
             dry_run=dry_run,
+            clean=clean,
             max_rescue=max_rescue,
         )
     except ValueError as exc:

mdfactory/orchestration/__init__.py

@@ -4,13 +4,14 @@
 
 from .build import build_systems
 from .config import ExecutorConfig, SlurmExecutorConfig
-from .simulate import find_structure_file, run_simulations
+from .simulate import clean_simulation_outputs, find_structure_file, run_simulations
 from .tui import configure_and_save_slurm, configure_slurm_interactive
 
 __all__ = [
     "ExecutorConfig",
     "SlurmExecutorConfig",
     "build_systems",
+    "clean_simulation_outputs",
     "find_structure_file",
     "run_simulations",
     "configure_and_save_slurm",

mdfactory/orchestration/simulate.py

@@ -70,6 +70,7 @@ def run_simulations(
     stages: list[str] | None = None,
     wait: bool = True,
     dry_run: bool = False,
+    clean: bool = False,
     checkpoint_mode: str = "auto",
     max_rescue: int = 3,
 ) -> list[dict]:
@@ -87,6 +88,11 @@ def run_simulations(
         Wait for completion (default: True).
     dry_run : bool
         Preview plan without executing (default: False).
+    clean : bool
+        Remove simulation outputs before running (default: False).
+        Respects ``stages`` filter — only files belonging to requested
+        stages are deleted.  Combined with ``dry_run``, previews what
+        would be deleted without acting.
     checkpoint_mode : str
         - "auto": Skip stages with valid outputs (default)
         - "skip": Never re-run completed stages
@@ -148,6 +154,20 @@ def run_simulations(
         return skipped_results
     sim_paths = ready_paths
 
+    # 1b. Clean outputs (if requested) before checkpoint detection.
+    if clean:
+        for sim_dir in sim_paths:
+            deleted = clean_simulation_outputs(sim_dir, stages, dry_run=dry_run)
+            if dry_run and deleted:
+                logger.info(
+                    f"Would delete {len(deleted)} file(s) from {sim_dir.name}: "
+                    + ", ".join(p.name for p in deleted)
+                )
+        if dry_run:
+            # After previewing deletions, still show the dry-run execution plan
+            # (which will report all stages as needed since nothing was deleted).
+            pass
+
     # 2. Checkpoint detection (includes restart info for -cpi -append support)
     work_plan = []
     for sim_dir in sim_paths:
@@ -216,8 +236,7 @@ def _run_pipeline(item):
 
             with ThreadPoolExecutor() as pool:
                 thread_futs = [
-                    (item["hash"], pool.submit(_run_pipeline, item))
-                    for item in active_items
+                    (item["hash"], pool.submit(_run_pipeline, item)) for item in active_items
                 ]
                 for h, tf in thread_futs:
                     try:
@@ -279,6 +298,81 @@ def _missing_build_files(sim_dir: Path, stages: list[str]) -> list[str]:
     return [f for f in required if not (sim_dir / f).exists()]
 
 
+def clean_simulation_outputs(
+    sim_dir: Path,
+    stages: list[str],
+    *,
+    dry_run: bool = False,
+) -> list[Path]:
+    """Remove simulation outputs for the given stages, preserving build inputs.
+
+    Derives deletable files from :data:`STAGE_BY_NAME` (tpr, cpt, log, edr,
+    gro, trr, xtc) plus ``mdout.mdp``, rescue-tier MDPs, and GROMACS backup
+    files (``#*#``).
+
+    Parameters
+    ----------
+    sim_dir : Path
+        Simulation directory.
+    stages : list[str]
+        Stage names whose outputs should be removed.
+    dry_run : bool
+        If ``True``, collect and return the list of files that *would* be
+        deleted without actually removing them.
+
+    Returns
+    -------
+    list[Path]
+        Paths that were deleted (or would be deleted in dry-run mode).
+
+    """
+    to_delete: list[Path] = []
+
+    for stage_name in stages:
+        spec = STAGE_BY_NAME[stage_name]
+        deffnm = spec.deffnm
+
+        # Named output files: tpr, cpt, log, edr
+        for ext in ("tpr", "cpt", "log", "edr"):
+            to_delete.append(sim_dir / f"{deffnm}.{ext}")
+
+        # Structure output (EM/NVT/NPT)
+        if spec.gro_out:
+            to_delete.append(sim_dir / spec.gro_out)
+
+        # Trajectory outputs (Production)
+        for traj in spec.traj_files:
+            to_delete.append(sim_dir / traj)
+
+        # Rescue-tier MDPs (e.g. em_rescue_t1.mdp, em_rescue_t2.mdp)
+        mdp_stem = spec.mdp_file.rsplit(".", 1)[0]
+        to_delete.extend(sim_dir.glob(f"{mdp_stem}_rescue_t*.mdp"))
+
+        # GROMACS backup files (#deffnm.*#)
+        to_delete.extend(sim_dir.glob(f"#{deffnm}.*#"))
+
+    # mdout.mdp — grompp output, always regenerated
+    to_delete.append(sim_dir / "mdout.mdp")
+
+    # Deduplicate (glob results may overlap with named files) and filter to
+    # files that actually exist.
+    seen: set[Path] = set()
+    existing: list[Path] = []
+    for p in to_delete:
+        if p not in seen and p.exists():
+            seen.add(p)
+            existing.append(p)
+
+    if not dry_run:
+        for p in existing:
+            p.unlink()
+            logger.debug(f"Deleted {p}")
+        if existing:
+            logger.info(f"Cleaned {len(existing)} file(s) from {sim_dir.name}")
+
+    return existing
+
+
 def _validate_simulation_dir(sim_dir: Path, stages: list[str]) -> None:
     """Raise if any required build output files are missing.
 
@@ -496,6 +590,11 @@ def _detect_stage_state(sim_dir: Path, stage: str, mode: str = "auto") -> dict:
 
     # Check partial progress (checkpoint exists, output doesn't).
     if cpt_file.exists() and tpr_file.exists():
+        if spec.traj_files:
+            # Trajectory stage (Production): stale checkpoint without a
+            # trajectory file cannot use -append — GROMACS would crash.
+            # Treat as not_started so grompp+mdrun run from scratch.
+            return {"status": "not_started", "cpt_file": None, "restart": False}
         return {"status": "partial", "cpt_file": cpt_file, "restart": True}
 
     return {"status": "not_started", "cpt_file": None, "restart": False}
@@ -669,11 +768,7 @@ def _execute_stage_list(
         cpt_file = restarts.get(stage, "")
 
         # Use rescue loop for eligible stages without checkpoint restart
-        if (
-            max_rescue > 0
-            and stage in RESCUE_ELIGIBLE_STAGES
-            and not cpt_file
-        ):
+        if max_rescue > 0 and stage in RESCUE_ELIGIBLE_STAGES and not cpt_file:
             prev_future = execute_stage_with_rescue(
                 sim_dir,
                 stage,
@@ -686,9 +781,7 @@ def _execute_stage_list(
         else:
             # Standard execution: chain futures without waiting
             stage_cfg = (
-                config.get_stage_config(stage)
-                if hasattr(config, "get_stage_config")
-                else None
+                config.get_stage_config(stage) if hasattr(config, "get_stage_config") else None
             )
             cfg_kwarg = {"stage_config": stage_cfg} if stage_cfg is not None else {}
             prev_future = run_stage(

mdfactory/tests/test_orchestration_cli_simulate.py

@@ -275,3 +275,58 @@ def test_simulate_systems_max_rescue_default_is_3(mock_slurm, mock_config, mock_
     mock_run.assert_called_once()
     _, kwargs = mock_run.call_args
     assert kwargs["max_rescue"] == 3
+
+
+@patch("mdfactory.orchestration.run_simulations", return_value=_SUCCESS_RESULTS)
+@patch("mdfactory.cli._load_executor_config", return_value=MagicMock(provider="local"))
+@patch("mdfactory.cli._resolve_slurm_flag", return_value=None)
+def test_simulate_systems_forwards_clean_flag(mock_slurm, mock_config, mock_run, tmp_path):
+    """--clean is forwarded from CLI to run_simulations."""
+    from mdfactory.cli import simulate_systems
+
+    sim_dir = tmp_path / "abc123"
+    sim_dir.mkdir()
+    (sim_dir / "system.pdb").touch()
+
+    simulate_systems(source=tmp_path, clean=True)
+
+    mock_run.assert_called_once()
+    _, kwargs = mock_run.call_args
+    assert kwargs["clean"] is True
+
+
+@patch("mdfactory.orchestration.run_simulations", return_value=_SUCCESS_RESULTS)
+@patch("mdfactory.cli._load_executor_config", return_value=MagicMock(provider="local"))
+@patch("mdfactory.cli._resolve_slurm_flag", return_value=None)
+def test_simulate_systems_clean_default_is_false(mock_slurm, mock_config, mock_run, tmp_path):
+    """--clean defaults to False when not specified."""
+    from mdfactory.cli import simulate_systems
+
+    sim_dir = tmp_path / "abc123"
+    sim_dir.mkdir()
+    (sim_dir / "system.pdb").touch()
+
+    simulate_systems(source=tmp_path)
+
+    mock_run.assert_called_once()
+    _, kwargs = mock_run.call_args
+    assert kwargs["clean"] is False
+
+
+@patch("mdfactory.orchestration.run_simulations", return_value=[])
+@patch("mdfactory.cli._load_executor_config", return_value=MagicMock(provider="local"))
+@patch("mdfactory.cli._resolve_slurm_flag", return_value=None)
+def test_simulate_systems_clean_with_dry_run(mock_slurm, mock_config, mock_run, tmp_path):
+    """--clean combined with --dry-run passes both flags through."""
+    from mdfactory.cli import simulate_systems
+
+    sim_dir = tmp_path / "abc123"
+    sim_dir.mkdir()
+    (sim_dir / "system.pdb").touch()
+
+    simulate_systems(source=tmp_path, clean=True, dry_run=True)
+
+    mock_run.assert_called_once()
+    _, kwargs = mock_run.call_args
+    assert kwargs["clean"] is True
+    assert kwargs["dry_run"] is True