change shell scripts to universal filesystem calls

Author: yqzhishen

tasks/base_task.py

@@ -1,5 +1,6 @@
 import glob
 import re
+import shutil
 import subprocess
 from datetime import datetime
 
@@ -249,10 +250,12 @@ def start(cls):
         if not hparams['infer']:  # train
             t = datetime.now().strftime('%Y%m%d%H%M%S')
             code_dir = f'{work_dir}/codes/{t}'
-            subprocess.check_call(f'mkdir "{code_dir}"', shell=True)
+            # TODO: test filesystem calls
+            os.mkdir(code_dir)
+            # subprocess.check_call(f'mkdir "{code_dir}"', shell=True)
             for c in hparams['save_codes']:
-                # TODO
-                subprocess.check_call(f'xcopy "{c}" "{code_dir}/" /s /e /y', shell=True)
+                shutil.copytree(c, code_dir, dirs_exist_ok=True)
+                # subprocess.check_call(f'xcopy "{c}" "{code_dir}/" /s /e /y', shell=True)
             print(f"| Copied codes to {code_dir}.")
             trainer.checkpoint_callback.task = task
             trainer.fit(task)

utils/pl_utils.py

@@ -335,8 +335,9 @@ def on_epoch_end(self, epoch, logs=None):
                 logging.info(f'Epoch {epoch:05d}@{self.task.global_step}: saving model to {filepath}')
             self._save_model(filepath)
             for old_ckpt in self.get_all_ckpts()[self.num_ckpt_keep:]:
-                # TODO
-                subprocess.check_call(f'del "{old_ckpt}"', shell=True)
+                # TODO: test filesystem calls
+                os.remove(old_ckpt)
+                # subprocess.check_call(f'del "{old_ckpt}"', shell=True)
                 if self.verbose > 0:
                     logging.info(f'Delete ckpt: {os.path.basename(old_ckpt)}')
             current = logs.get(self.monitor)