Pipedream trigger minor fix

Author: rowanwalker96

src/dlstbx/services/trigger_xchem.py

@@ -238,7 +238,7 @@ def trigger_pandda_xchem(
         proposal = query.first()
 
         # 0. Check that this is an XChem expt & locate .SQLite database
-        if proposal.proposalCode not in {"lb"}:  # need to handle industrial 'sw' also
+        if proposal.proposalCode not in {"lb"}:
             self.log.debug(
                 f"Not triggering PanDDA2 pipeline for dcid={dcid} with proposal_code={proposal.proposalCode}"
             )
@@ -338,12 +338,6 @@ def trigger_pandda_xchem(
         db = processing_dir / "database" / "soakDBDataFile.sqlite"
         processed_dir = xchem_visit_dir / "processed"
 
-        # Make a copy of the most recent sqlite for reading
-        # db_copy = xchem_visit_dir / "processing/database" / "auto_soakDBDataFile.sqlite"
-        # if not db_copy.exists() or (db.stat().st_mtime != db_copy.stat().st_mtime):
-        #     shutil.copy2(str(db), str(db_copy))
-        #     self.log.info(f"Made a copy of {db}, auto_soakDBDataFile.sqlite")
-
         # 1. Trigger when all upstream pipelines & related dimple jobs have finished
 
         program_list = [
@@ -600,11 +594,10 @@ def trigger_pandda_xchem(
             self.log.info(f"Chosen mtz for dcid {dcid} is {upstream_mtz}")
         else:
             self.log.info(
-                "Exiting PanDDA2/Pipedream trigger: no environment information"
+                f"Exiting PanDDA2/Pipedream trigger: no environment information for dcid {dcid}"
             )
             return {"success": True}
 
-        # upstream_proc = df[df['autoProcScalingId']==scaling_id]['processingPrograms'].item() # fails
         pdb = chosen_dataset_path + "/final.pdb"
         mtz = chosen_dataset_path + "/final.mtz"
 
@@ -634,7 +627,7 @@ def trigger_pandda_xchem(
 
         except Exception as e:
             self.log.info(
-                f"Exiting PanDDA2/Pipedream trigger: Exception whilst reading ligand information from {db} for dtag {dtag}: {e}"
+                f"Exiting PanDDA2/Pipedream trigger: Exception whilst reading ligand information from {db} for dtag {dtag}, dcid {dcid}: {e}"
             )
             return {"success": True}
 
@@ -644,7 +637,7 @@ def trigger_pandda_xchem(
 
         if len(df) != 1:
             self.log.info(
-                f"Exiting PanDDA2/Pipedream trigger: Unique row in .sqlite for dtag {dtag}, puck {code}, puck position {location} cannot be found in {db}, skipping..."
+                f"Exiting PanDDA2/Pipedream trigger: Unique row in .sqlite for dtag {dtag}, puck {code}, puck position {location} cannot be found in {db}, skipping dcid {dcid}"
             )
             return {"success": True}
 
@@ -683,13 +676,33 @@ def trigger_pandda_xchem(
         # Copy the dimple files of the selected dataset
         shutil.copy(pdb, str(dataset_dir / "dimple.pdb"))
         shutil.copy(mtz, str(dataset_dir / "dimple.mtz"))
-        shutil.copy(
-            upstream_mtz, str(dataset_dir / pathlib.Path(upstream_mtz).parts[-1])
-        )
+        shutil.copy(upstream_mtz, str(dataset_dir / f"{dtag}.free.mtz"))
 
         with open(compound_dir / f"{CompoundCode}.smiles", "w") as smi_file:
             smi_file.write(CompoundSMILES)
 
+        # Check if Pipedream job was recently launched
+        min_start_time = datetime.now() - timedelta(hours=3)
+
+        query = (
+            (
+                session.query(AutoProcProgram, ProcessingJob.dataCollectionId).join(
+                    ProcessingJob,
+                    ProcessingJob.processingJobId == AutoProcProgram.processingJobId,
+                )
+            )
+            .filter(ProcessingJob.dataCollectionId == dcid)
+            .filter(ProcessingJob.automatic == True)  # noqa E711
+            .filter(AutoProcProgram.processingPrograms.in_(["Pipedream"]))
+            .filter(AutoProcProgram.recordTimeStamp > min_start_time)
+        )
+
+        if triggered_processing_job := query.first():
+            self.log.info(
+                f"Pipedream job recently launched for dcid {dcid}, skipping pipedream trigger"
+            )
+            pipedream = False
+
         # Create seperate pipedream directory
         if pipedream:
             pipedream_dir = analysis_dir / "pipedream"
@@ -700,9 +713,7 @@ def trigger_pandda_xchem(
             pathlib.Path(compound_dir_pd).mkdir(parents=True, exist_ok=True)
             shutil.copy(pdb, str(dataset_dir_pd / "dimple.pdb"))
             shutil.copy(mtz, str(dataset_dir_pd / "dimple.mtz"))
-            shutil.copy(
-                upstream_mtz, str(dataset_dir_pd / pathlib.Path(upstream_mtz).parts[-1])
-            )
+            shutil.copy(upstream_mtz, str(dataset_dir_pd / f"{dtag}.free.mtz"))
 
             with open(compound_dir_pd / f"{CompoundCode}.smiles", "w") as smi_file:
                 smi_file.write(CompoundSMILES)

src/dlstbx/wrapper/pandda_xchem.py

@@ -65,9 +65,7 @@ def run(self):
 
         # -------------------------------------------------------
         # acedrg_command = f"module load ccp4; acedrg -i {smiles_file} -o {CompoundCode}"
-        restraints_command = f"module load buster; module load graphviz; \
-                               export CSDHOME=/dls_sw/apps/CSDS/2024.1.0/; export BDG_TOOL_MOGUL=/dls_sw/apps/CSDS/2024.1.0/ccdc-software/mogul/bin/mogul; \
-                               grade2 --in {smiles_file} --itype smi --out {CompoundCode} -f"
+        restraints_command = f"grade2 --in {smiles_file} --itype smi --out {CompoundCode} -f"
 
         try:
             result = subprocess.run(
@@ -97,7 +95,7 @@ def run(self):
         with open(dataset_dir / "restraints.log", "w") as log_file:
             log_file.write(result.stdout)
 
-        self.log.info(f"Restraints generated succesfully for dtag {dtag}")
+        self.log.info(f"Restraints generated succesfully for dtag {dtag}, launching PanDDA2")
 
         pandda2_command = f"source /dls_sw/i04-1/software/PanDDA2/venv/bin/activate; \
         python -u /dls_sw/i04-1/software/PanDDA2/scripts/process_dataset.py --data_dirs={model_dir} --out_dir={auto_panddas_dir} --dtag={dtag} --use_ligand_data=False --local_cpus=1"
@@ -122,10 +120,14 @@ def run(self):
         dataset_pdir = auto_panddas_dir / "processed_datasets" / dtag
         ligand_dir = dataset_pdir / "ligand_files"
 
+        # pandda2 not moving files into ligand_dir, fix
         for file in compound_dir.rglob("*"):
             if file.is_file() and file.suffix.lower() in {".pdb", ".cif", ".smiles"}:
-                target = ligand_dir / file.name
-                shutil.copy2(file, target)
+                targets = [ligand_dir / file.name, dataset_dir / file.name]
+                for target in targets:
+                    if target.exists() or target.is_symlink():
+                        target.unlink()
+                    target.symlink_to(file)
 
         pandda_log = dataset_pdir / "pandda2.log"
         with open(pandda_log, "w") as log_file: