diff --git a/avaframe/com4FlowPy/com4FlowPy.py b/avaframe/com4FlowPy/com4FlowPy.py index 9930fe626..972f10da2 100755 --- a/avaframe/com4FlowPy/com4FlowPy.py +++ b/avaframe/com4FlowPy/com4FlowPy.py @@ -421,8 +421,10 @@ def checkInputParameterValues(modelParameters, modelPaths): rasterValues = data["rasterData"] rasterValues[rasterValues < 0] = np.nan # handle different noData values if np.any(rasterValues > 90, where=~np.isnan(rasterValues)): - log.error("Error: Not all Alpha-raster values are within a physically sensible range ([0,90]),\ - in respective startcells the general alpha angle is used.") + log.error( + "Error: Not all Alpha-raster values are within a physically sensible range ([0,90]),\ + in respective startcells the general alpha angle is used." + ) _checkVarParams = False if modelParameters["varUmaxBool"]: @@ -774,10 +776,10 @@ def mergeAndWriteResults(modelPaths, modelOptions): if "relIdPolygon" in _outputs: pathPolygons = SPAM.mergeDictToPolygon(modelPaths["tempDir"], "res_startCellIdDict", outputHeader) pathPolygons.to_file( - modelPaths["resDir"] / "com4_{}_{}_pathPolygons.geojson".format(_uid, _ts), driver="GeoJSON" + modelPaths["resDir"] / "com4_{}_{}_relIdPolygon.geojson".format(_uid, _ts), driver="GeoJSON" ) del pathPolygons - log.info("com4_{}_{}_pathPolygons is written".format(_uid, _ts)) + log.info("com4_{}_{}_relIdPolygon is written".format(_uid, _ts)) if "relIdCount" in _outputs: countRelId = SPAM.mergeDictToRaster(modelPaths["tempDir"], "res_startCellIdDict") @@ -785,13 +787,13 @@ def mergeAndWriteResults(modelPaths, modelOptions): output = IOf.writeResultToRaster( outputHeader, countRelId, - modelPaths["resDir"] / "com4_{}_{}_countRelId".format(_uid, _ts), + modelPaths["resDir"] / "com4_{}_{}_relIdCount".format(_uid, _ts), flip=True, useCompression=useCompression, ) del countRelId del output - log.info("com4_{}_{}_countRelId is written".format(_uid, _ts)) + log.info("com4_{}_{}_relIdCount is written".format(_uid, _ts)) # NOTE: # if not modelOptions["infraBool"]: # if no infra diff --git a/avaframe/com4FlowPy/com4FlowPyCfg.ini b/avaframe/com4FlowPy/com4FlowPyCfg.ini index 97e248553..763164e3b 100644 --- a/avaframe/com4FlowPy/com4FlowPyCfg.ini +++ b/avaframe/com4FlowPy/com4FlowPyCfg.ini @@ -235,6 +235,16 @@ outputNoDataValue = -9999 # if relIdCount or relIdPolygon is in outputFiles, the ids of the PRAs should be provided in the raster file in the RELID folder outputFiles = zDelta|cellCounts|travelLengthMax|fpTravelAngleMax +# whether simulation results with the same simHash of the running simulation already exists +# AND the existing resultsFolder already contains valid com4FlowPy outputs. +# 1) overwriteResults = default ... does not re-run a simulation if results folder (res_) and .json +# -- if there are remnants from a previously attempted but not succesfully finished simulation (e.g. existing res_ and/or .json) +# --> delete existing results folder (res_) and .json and run Simulation +# 2) overwriteResults = reRunAndOverwrite ... deletes existing results folder (res_) and .json and runs Simulation +# 3) overwriteResults = reRunAndBackup ... moves existing result folder (res_) and .json to a dedicated backup folder (e.g. in /BACKUP) + +overwriteResults = default + #++++++++++++ Custom paths True/False # default: False # if set to 'False': @@ -263,7 +273,7 @@ releasePath = relIdPath = infraPath = forestPath = -varUmaxPath = +varUmaxPath = varAlphaPath = varExponentPath = diff --git a/avaframe/in3Utils/fileHandlerUtils.py b/avaframe/in3Utils/fileHandlerUtils.py index d15a35d8a..ea4abfc1b 100644 --- a/avaframe/in3Utils/fileHandlerUtils.py +++ b/avaframe/in3Utils/fileHandlerUtils.py @@ -811,3 +811,165 @@ def findAvaDirsBasedOnInputsDir(Dir): log.info(f"'{avaDir.name}'") return avaDirs + + +def checkResultFolderFilesExist(path, outputnames=""): + """ + check whether a (result) directory exists + whether it contains files whose names include the given output names. + + Parameters + -------- + path: pathlib.Path or str + Path to the directory to search in + outputnames: list + names of variables that are checked to appear within the file names + + Returns + ------- + folderExist: bool + True if the directory exists + filesExist: bool + True if the directory exists and contains files matching all output names, False otherwise. + """ + + folderExist = os.path.isdir(path) + if not folderExist: + filesExist = False + return folderExist, filesExist + + fileNames = os.listdir(path) + if outputnames != "": + for name in outputnames: + if not any(name.lower() in fileName.lower() for fileName in fileNames): + filesExist = False + return folderExist, filesExist + else: + filesExist = True + else: + filesExist = False + return folderExist, filesExist + + +def deleteCom4Results(outputPath, simHash): + """ + Deletes com4FlowPy results folder res_ and the .json file + + Parameters + ---------- + outputPath: pathlib.Path + Path to the outputs directory + simHash: string + simhash of simulation + """ + outputPath = pathlib.Path(outputPath) + jsonFile = outputPath / f"{simHash}.json" + + if os.path.isfile(jsonFile): + os.remove(jsonFile) + log.info(f"{jsonFile} is deleted.") + + resFolder = searchCom4ResDir(outputPath, simHash) + + if resFolder is not None: + shutil.rmtree(resFolder) + log.info(f"{resFolder} is deleted.") + + +def backupCom4Results(outputPath, simHash): + """ + move com4FlowPy results folder res_ and .json file to a backup folder + Parameters + ---------- + outputPath: pathlib.Path + Path to the outputs directory + simHash: string + simhash of simulation + """ + # create backup folder + outputPath = pathlib.Path(outputPath) + backupPath = outputPath / "backup" + makeADir(backupPath) + + jsonFile = outputPath / f"{simHash}.json" + + resFolder = searchCom4ResDir(outputPath, simHash) + + # Move results folder + if resFolder is not None and os.path.exists(resFolder): + dst = _getUniqueDst(resFolder, backupPath) + shutil.move(resFolder, dst) + log.info(f"{resFolder} is backed up to {dst}.") + + # Move JSON file + if os.path.isfile(jsonFile): + dst = _getUniqueDst(jsonFile, backupPath) + shutil.move(jsonFile, dst) + log.info(f"{jsonFile} is backed up to {dst}.") + + +def _getUniqueDst(srcPath, backupPath): + """ + Helper function to get a unique destination path in case + the backUp Files/Folder already exist in a given backup directory. + + If a file or directory with the same name already exists in the + backup location, an integer suffix in the form ``(n)`` is appended + to the stem of the source name until an unused path is found. + + e.g. if the folder "results_simHash" or the file "simHash.json" already + exist, the function will return "results_simHash(1)", "simHash(1).json", + etc. + + Parameters + ---------- + srcPath : str or pathlib.Path + Path to the source file or directory being backed up. + Only the name component is used when constructing the + destination path. + + backupPath : str or pathlib.Path + Directory where the backup item will be stored. + + Returns + ------- + target: pathlib.Path + A unique destination path that does not currently exist + within ``backupPath``. + + """ + srcPath = pathlib.Path(srcPath) + backupPath = pathlib.Path(backupPath) + + target = backupPath / srcPath.name + counter = 1 + while target.exists(): + # Formats as 'folder(1)', 'file(1).json', 'file(2).json', etc. + target = backupPath / f"{srcPath.stem}({counter}){srcPath.suffix}" + counter += 1 + return target + +def searchCom4ResDir(outputPath, simHash): + """ + search for the result folder with simhash in the output path + + Parameters + ---------- + outputPath: pathlib.Path + Path to the outputs directory + simHash: string + simhash of simulation + + Returns + ----------- + resFolder: pathlib.Path + path to the result folder + """ + if os.path.isdir(outputPath / f"res_{simHash}"): + resFolder = outputPath / f"res_{simHash}" + elif os.path.isdir(outputPath / "peakFiles" / f"res_{simHash}"): + resFolder = outputPath / "peakFiles" / f"res_{simHash}" + else: + resFolder = None + + return resFolder diff --git a/avaframe/runCom4FlowPy.py b/avaframe/runCom4FlowPy.py index 5a3f2b338..bfb000b5a 100644 --- a/avaframe/runCom4FlowPy.py +++ b/avaframe/runCom4FlowPy.py @@ -5,10 +5,10 @@ # Load modules import pathlib import os -import sys from datetime import datetime import logging import json +import shutil # Local imports import avaframe.in3Utils.initializeProject as initProj @@ -26,7 +26,7 @@ import avaframe.in3Utils.geoTrans as gT -def main(avalancheDir=""): +def main(avalancheDir="", cfg=None): """this is a wrapper around com4FlowPy.py that handles the following tasks: * reading inputs from (local_)avaframeCfg.ini and (local_)com4FlowPyCfg.ini * constructing cfgPath and cfgSetup dictionaries for passing to com4FlowPy.com4FlowPyMain() @@ -34,18 +34,34 @@ def main(avalancheDir=""): NOTE-TODO: * This function needs clean-up! + + Parameters + ------------ + avalancheDir: str + path to avalanche directory that is simulated (if "", the path of the (local_)avaframeCfg.ini is used) + cfg: configparser object + settings for the simulation (if None, (local_)com4FlowPyCfg.ini is used) + + Returns + ------------- + outputDict: dict + information about simulation: + "uid": id of simulation + "simulationPerformed": boolean - 'True' if the simulation terminated succesfully + "resultOverwritten": boolean - 'True' if results are overwritten + "message": explanation to simulation status """ # log file name; leave empty to use default runLog.log logName = "runcom4FlowPy" # Read main Config and com4FlowPy config from files cfgMain = cfgUtils.getGeneralConfig() - cfg = cfgUtils.getModuleConfig(com4FlowPy) + if cfg is None: + cfg = cfgUtils.getModuleConfig(com4FlowPy) # check and handle outputFiles list provided in (local_)com4FlowPyCfg.ini cfg["PATHS"]["outputFiles"] = checkOutputFilesFormat(cfg["PATHS"]["outputFiles"]) - + cfg["PATHS"]["overwriteResults"] = checkOverwriteRes(cfg["PATHS"]["overwriteResults"]) cfgSetup = cfg["GENERAL"] - cfgFlags = cfg["FLAGS"] cfgCustomPaths = cfg["PATHS"] # if customPaths == False --> use AvaFrame Folder structure @@ -88,18 +104,49 @@ def main(avalancheDir=""): cfgPath["resDir"] = cfgPath["outDir"] / "peakFiles" / "res_{}".format(uid) # (timeString) # check if simulation with same uid already has results folder - if os.path.isdir(cfgPath["resDir"]): + + resFolderExist, resFilesExist = fU.checkResultFolderFilesExist( + cfgPath["resDir"], cfgCustomPaths["outputFiles"].split("|") + ) + + if cfgCustomPaths["overwriteResults"] == "default" and resFilesExist: log.info("folder with same name already exists - aborting") log.info( "simulation results folder with same .ini parameters already exists: simulation {}".format( uid ) ) - sys.exit(1) + return { + "uid": uid, + "simulationPerformed": False, + "resultOverwritten": False, + "message": "Simulation not performed! - results for exact same configuration already exist." + } + elif cfgCustomPaths["overwriteResults"] == "default" and resFolderExist: + # delete existing result folder + fU.deleteCom4Results(cfgPath["outDir"], uid) + resultOverwritten = True + message = f"simulation completed, Leftover files from aborted run with same {uid} overwritten." + + elif cfgCustomPaths["overwriteResults"] == "reRunAndOverwrite" and resFolderExist: + # it does not matter if the files exist, the existing results folder is deleted + fU.deleteCom4Results(cfgPath["outDir"], uid) + resultOverwritten = True + message = "simulation completed, existing results overwritten." + + elif cfgCustomPaths["overwriteResults"] == "reRunAndBackup" and resFolderExist: + # move results folder and json file to backup folder + fU.backupCom4Results(cfgPath["outDir"], uid) + resultOverwritten = True + message = "simulation completed, existing results backed up." + else: - fU.makeADir(cfgPath["resDir"]) - cfgPath["tempDir"] = cfgPath["workDir"] / "temp" - fU.makeADir(cfgPath["tempDir"]) + resultOverwritten = False + message = "simulation completed, no previous results found." + + fU.makeADir(cfgPath["resDir"]) + cfgPath["tempDir"] = cfgPath["workDir"] / "temp" + fU.makeADir(cfgPath["tempDir"]) # writing config to .json file successToJSON = writeCfgJSON(cfg, uid, cfgPath["outDir"]) @@ -119,6 +166,12 @@ def main(avalancheDir=""): cfgPath["useCompression"] = cfgCustomPaths.getboolean("useCompression") com4FlowPy.com4FlowPyMain(cfgPath, cfgSetup) + return { + "uid": uid, + "simulationPerformed": True, + "resultOverwritten": resultOverwritten, + "message": message, + } # if customPaths == True --> check elif cfgCustomPaths["useCustomPaths"] == "True": @@ -139,22 +192,63 @@ def main(avalancheDir=""): log = logUtils.initiateLogger(workDir, logName + "_" + uid) timeString = datetime.now().strftime("%Y%m%d_%H%M%S") - try: - os.makedirs(workDir / "res_{}".format(uid)) # (time_string)) - res_dir = workDir / "res_{}".format(uid) # (time_string) - except FileExistsError: + + res_dir = workDir / "res_{}".format(uid) # (time_string) + + resFolderExist, resFilesExist = fU.checkResultFolderFilesExist( + res_dir, cfgCustomPaths["outputFiles"].split("|") + ) + + if cfgCustomPaths["overwriteResults"] == "default" and resFilesExist: log.info( "simulation results folder with same .ini parameters already exists: simulation {}".format( uid ) ) - sys.exit(1) - try: - os.makedirs(workDir / res_dir / "temp") - temp_dir = workDir / res_dir / "temp" - except FileExistsError: - log.info("temp folder for simualtion {} already exists - aborting".format(uid)) - sys.exit(1) + return { + "uid": uid, + "simulationPerformed": False, + "resultOverwritten": False, + "message": "Simulation not performed! - results for exact same configuration already exist." + } + elif cfgCustomPaths["overwriteResults"] == "default" and resFolderExist: + # delete existing result folder + fU.deleteCom4Results(workDir, uid) + resultOverwritten = True + message = f"simulation completed, Leftover files from aborted run with same {uid} overwritten." + + elif cfgCustomPaths["overwriteResults"] == "reRunAndOverwrite" and resFolderExist: + # it does not matter if the files exist, the existing results folder is deleted + fU.deleteCom4Results(workDir, uid) + resultOverwritten = True + message = "simulation completed, existing results overwritten." + + elif cfgCustomPaths["overwriteResults"] == "reRunAndBackup" and resFolderExist: + # move results folder and json file to backup folder + fU.backupCom4Results(workDir, uid) + resultOverwritten = True + message = "simulation completed, existing results backed up." + + else: + resultOverwritten = False + message = "simulation completed, no previous results found." + + fU.makeADir(res_dir) + + # TODO: how should we deal with the temp folder? + temp_dir = workDir / res_dir / "temp" + tempFolderExist, _ = fU.checkResultFolderFilesExist(temp_dir) + if cfgCustomPaths["overwriteResults"] == "default" and tempFolderExist: + log.info("temp folder for simulation {} already exists - aborting".format(uid)) + return { + "uid": uid, + "simulationPerformed": False, + "resultOverwritten": False, + "message": "temp folder already exists.", + } + elif tempFolderExist: + shutil.rmtree(temp_dir) + fU.makeADir(temp_dir) # writing config to .json file successToJSON = writeCfgJSON(cfg, uid, workDir) @@ -190,12 +284,23 @@ def main(avalancheDir=""): cfgPath["timeString"] = timeString com4FlowPy.com4FlowPyMain(cfgPath, cfgSetup) + return { + "uid": uid, + "simulationPerformed": True, + "resultOverwritten": resultOverwritten, + "message": message, + } else: print( "INPUT SETTINGS incorrect - please check (local_)avaframeCfg.ini and (local_)com4FlowPyCfg.ini" ) - sys.exit(1) + return { + "uid": None, + "simulationPerformed": False, + "resultOverwritten": False, + "message": "input settings incorrect.", + } def readFlowPyinputs(avalancheDir, cfgFlowPy, log): @@ -417,5 +522,36 @@ def writeCfgJSON(cfg, uid, workDir): return e +def checkOverwriteRes(overwriteResults): + """ + check if overwriteResults option is provided in proper format, else return default + + Parameters + ------------ + overwriteResults: str + overwriteResults option + + Returns + ---------- + overwriteResults: str + allowed overwriteResults option + """ + if overwriteResults not in ["reRunAndBackup", "reRunAndOverwrite", "default"]: + message = "'overwriteResults' setting not valid - please re-check settings and use any of <'default', 'reRunAndBackup', 'reRunAndOverwrite'>" + raise ValueError(message) + return overwriteResults + + if __name__ == "__main__": - main() + """ + main entry point for runCom4FlowPy.py + """ + resDict = main() + + if resDict["simulationPerformed"]: + print("simulation {} completed!".format(resDict["uid"])) + print("{}".format(resDict["message"])) + + else: + print("simulation {} NOT completed".format(resDict["uid"])) + print("{}".format(resDict["message"])) diff --git a/avaframe/tests/test_com4FlowPy.py b/avaframe/tests/test_com4FlowPy.py index f88a863b0..50525624c 100644 --- a/avaframe/tests/test_com4FlowPy.py +++ b/avaframe/tests/test_com4FlowPy.py @@ -10,11 +10,15 @@ import os import rasterio import geopandas as gpd +import configparser +import copy +import shutil from avaframe.com4FlowPy import flowClass import avaframe.com4FlowPy.flowCore as flowCore import avaframe.com4FlowPy.splitAndMerge as SPAM import avaframe.in2Trans.rasterUtils as IOf +import avaframe.runCom4FlowPy as runCom4FlowPy def test_add_os(): @@ -529,6 +533,225 @@ def test_mergeDictToPolygon(tmp_path): assert np.all(gdfPathPolygons.geometry.geom_equals(refPolygons.geometry)) +def test_runCom4FlowPy(tmp_path): + + _avaframeDir = pathlib.Path(__file__).parents[1] + avaFlowPyDir = str(_avaframeDir / "data" / "avaFlowPy" / "Inputs") + avaTestDir = pathlib.Path(tmp_path, "avaFlowPyTest") + avaTestDirInput = avaTestDir / "Inputs" + shutil.copytree(avaFlowPyDir, avaTestDirInput) + avalancheDir = str(avaTestDir) + + cfg = configparser.ConfigParser() + cfg["GENERAL"] = { + "infra": "False", + "variableUmaxLim": "False", + "variableAlpha": "False", + "variableExponent": "False", + "forest": "False", + "alpha": "40", + "exp": "8", + "flux_threshold": "3.0e-4", + "max_z": "200", + "previewMode": "False", + "fluxDistOldVersion": "False", + "procPerCPUCore": "1", + "chunkSize": "50", + "maxChunks": "500", + "cpuCount": "1", + "tileSize": "15000", + "tileOverlap": "5000", + } + cfg["PATHS"] = { + "outputFiles": "zDelta", + "useCustomPaths": "False", + "useCustomPathDEM": "False", + "outputFileFormat": ".tif", + "overwriteResults": "default", + } + + resDictTest1 = { + "simulationPerformed": True, + "resultOverwritten": False, + "message": "simulation completed, no previous results found." + } + resDict = runCom4FlowPy.main(avalancheDir=avalancheDir, cfg=copy.deepcopy(cfg)) + + for key in resDictTest1: + assert resDictTest1[key] == resDict[key] + + # second run + resDictTest2 = { + "uid": resDict["uid"], + "simulationPerformed": False, + "resultOverwritten": False, + "message": "Simulation not performed! - results for exact same configuration already exist.", + } + resDict = runCom4FlowPy.main(avalancheDir=avalancheDir, cfg=copy.deepcopy(cfg)) + + for key in resDictTest2: + assert resDictTest2[key] == resDict[key] + + # search for one result file and remove it + resFolder = ( + pathlib.Path(avalancheDir) / "Outputs" / "com4FlowPy" / "peakFiles" / "res_{}".format(resDict["uid"]) + ) + fileNames = os.listdir(resFolder) + os.remove(resFolder / fileNames[0]) + # second run + resDictTest21 = { + "uid": resDict["uid"], + "simulationPerformed": True, + "resultOverwritten": True, + "message": f"simulation completed, Leftover files from aborted run with same {resDict['uid']} overwritten.", + } + + resDict = runCom4FlowPy.main(avalancheDir=avalancheDir, cfg=copy.deepcopy(cfg)) + + for key in resDictTest21: + assert resDictTest21[key] == resDict[key] + + # third run with changing cfg: + cfg["PATHS"]["overwriteResults"] = "reRunAndOverwrite" + resDict = runCom4FlowPy.main(avalancheDir=avalancheDir, cfg=copy.deepcopy(cfg)) + + for key in resDictTest1: + assert resDictTest1[key] == resDict[key] + + # fourth run with overwriting results + resDictTest4 = { + "uid": resDict["uid"], + "simulationPerformed": True, + "resultOverwritten": True, + "message": "simulation completed, existing results overwritten.", + } + resDict = runCom4FlowPy.main(avalancheDir=avalancheDir, cfg=copy.deepcopy(cfg)) + + for key in resDictTest4: + assert resDictTest4[key] == resDict[key] + + # fifth run with changing cfg: + cfg["PATHS"]["overwriteResults"] = "reRunAndBackup" + resDict = runCom4FlowPy.main(avalancheDir=avalancheDir, cfg=copy.deepcopy(cfg)) + + for key in resDictTest1: + assert resDictTest1[key] == resDict[key] + + # sixth run with backuping results + resDictTest6 = { + "uid": resDict["uid"], + "simulationPerformed": True, + "resultOverwritten": True, + "message": "simulation completed, existing results backed up.", + } + resDict = runCom4FlowPy.main(avalancheDir=avalancheDir, cfg=copy.deepcopy(cfg)) + + for key in resDictTest6: + assert resDictTest6[key] == resDict[key] + + # make same for custom paths + _wDir = avalancheDir + "/Outputs/com4FlowPy" + _demPath = avalancheDir + "/Inputs/dem.tif" + _releasePath = avalancheDir + "/Inputs/REL/rel.shp" + + cfg["PATHS"] = { + "outputFiles": "zDelta", + "useCustomPaths": "True", + "useCustomPathDEM": "False", + "outputFileFormat": ".tif", + "overwriteResults": "default", + "workDir": _wDir, + "demPath": _demPath, + "releasePath": _releasePath, + "relIdPath": "", + "infraPath": "", + "forestPath": "", + "varUmaxPath": "", + "varAlphaPath": "", + "varExponentPath": "", + "deleteTempFolder": "True", + "outputNoDataValue": "-9999", + "useCompression": "False", + } + + resDictTest1 = { + "simulationPerformed": True, + "resultOverwritten": False, + "message": "simulation completed, no previous results found.", + } + resDict = runCom4FlowPy.main(cfg=copy.deepcopy(cfg)) + + for key in resDictTest1: + assert resDictTest1[key] == resDict[key] + + # second run + resDictTest2 = { + "uid": resDict["uid"], + "simulationPerformed": False, + "resultOverwritten": False, + "message": "Simulation not performed! - results for exact same configuration already exist.", + } + resDict = runCom4FlowPy.main(cfg=copy.deepcopy(cfg)) + + for key in resDictTest2: + assert resDictTest2[key] == resDict[key] + + # search for one result file and remove it + resFolder = pathlib.Path(avalancheDir) / "Outputs" / "com4FlowPy" / "res_{}".format(resDict["uid"]) + fileNames = os.listdir(resFolder) + os.remove(resFolder / fileNames[0]) + # second run + resDictTest21 = { + "uid": resDict["uid"], + "simulationPerformed": True, + "resultOverwritten": True, + "message": f"simulation completed, Leftover files from aborted run with same {resDict['uid']} overwritten.", + } + + resDict = runCom4FlowPy.main(avalancheDir=avalancheDir, cfg=copy.deepcopy(cfg)) + + for key in resDictTest21: + assert resDictTest21[key] == resDict[key] + + # second run with changing cfg: + cfg["PATHS"]["overwriteResults"] = "reRunAndOverwrite" + resDict = runCom4FlowPy.main(cfg=copy.deepcopy(cfg)) + + for key in resDictTest1: + assert resDictTest1[key] == resDict[key] + + # fourth run with overwriting results + resDictTest4 = { + "uid": resDict["uid"], + "simulationPerformed": True, + "resultOverwritten": True, + "message": "simulation completed, existing results overwritten.", + } + resDict = runCom4FlowPy.main(cfg=copy.deepcopy(cfg)) + + for key in resDictTest4: + assert resDictTest4[key] == resDict[key] + + # fifth run with changing cfg: + cfg["PATHS"]["overwriteResults"] = "reRunAndBackup" + resDict = runCom4FlowPy.main(avalancheDir=avalancheDir, cfg=copy.deepcopy(cfg)) + + for key in resDictTest1: + assert resDictTest1[key] == resDict[key] + + # sixth run with backuping results + resDictTest6 = { + "uid": resDict["uid"], + "simulationPerformed": True, + "resultOverwritten": True, + "message": "simulation completed, existing results backed up.", + } + resDict = runCom4FlowPy.main(avalancheDir=avalancheDir, cfg=copy.deepcopy(cfg)) + + for key in resDictTest6: + assert resDictTest6[key] == resDict[key] + + if __name__ == "__main__": test_add_os() test_reverseTopology() @@ -539,3 +762,4 @@ def test_mergeDictToPolygon(tmp_path): test_mergeDict(tmpDir) test_mergeDictToRaster(tmpDir) test_mergeDictToPolygon(tmpDir) + test_runCom4FlowPy() diff --git a/avaframe/tests/test_fileHandlerUtils.py b/avaframe/tests/test_fileHandlerUtils.py index 210df49cf..87efbce0a 100644 --- a/avaframe/tests/test_fileHandlerUtils.py +++ b/avaframe/tests/test_fileHandlerUtils.py @@ -9,10 +9,13 @@ import numpy as np import os from avaframe.in3Utils import fileHandlerUtils as fU +import avaframe.in2Trans.rasterUtils as rasterUtils import pytest import shutil import pathlib import configparser +import rasterio +import json def test_makeADir(tmp_path): @@ -529,3 +532,209 @@ def test_makeSimDF_layerColumn_singleLayer(tmp_path): assert "layer" in dataDF.columns assert all(v == "" for v in dataDF["layer"].tolist()) + + +def test_checkResultFolderFilesExist(tmp_path): + """ test make directory """ + + avaName = 'testRes' + resDir = pathlib.Path(tmp_path) / avaName + folderExist, filesExist = fU.checkResultFolderFilesExist(resDir) + + assert folderExist is False + assert filesExist is False + + fU.makeADir(resDir) + folderExist, filesExist = fU.checkResultFolderFilesExist(resDir) + assert folderExist + assert filesExist is False + + fU.makeADir(resDir) + folderExist, filesExist = fU.checkResultFolderFilesExist(resDir, outputnames=["zdelta"]) + assert folderExist + assert filesExist is False + + # create a result file + # first create test raster and save in test folder + rasterName = "resultTest_123_zdelta" + testRaster = np.zeros((10, 10)) + + cellsize = 10 + nrows, ncols = testRaster.shape + + header = { + "cellsize": cellsize, + "nrows": nrows, + "ncols": ncols, + "xllcenter": 0, + "yllcenter": 0, + "nodata_value": -9999, + "driver": "GTiff", + "crs": "EPSG:4326", + } + # convert lower-left center to upper-left corner + x_ul = header["xllcenter"] - cellsize / 2 + y_ul = header["yllcenter"] + nrows * cellsize - cellsize / 2 + + transform = rasterio.transform.from_origin(x_ul, y_ul, cellsize, cellsize) + header["transform"] = transform + + rasterUtils.writeResultToRaster(header, testRaster, resDir / rasterName, useCompression=True, flip=True) + + folderExist, filesExist = fU.checkResultFolderFilesExist(resDir, outputnames=["zdelta"]) + assert folderExist + assert filesExist + + folderExist, filesExist = fU.checkResultFolderFilesExist(resDir, outputnames=["zdelta", "flux"]) + assert folderExist + assert filesExist is False + + rasterName = "resultTest_456_flux" + rasterUtils.writeResultToRaster(header, testRaster, resDir / rasterName, useCompression=True, flip=True) + + folderExist, filesExist = fU.checkResultFolderFilesExist(resDir, outputnames=["zdelta", "flux"]) + assert folderExist + assert filesExist + + +def test_searchCom4ResDir(tmp_path): + """ test searchCom4ResDir function for locating com4FlowPy Result Directories""" + simHash = "abc123" + outputPath = pathlib.Path(tmp_path) / "Outputs" + + # no folder exists yet + resFolder = fU.searchCom4ResDir(outputPath, simHash) + assert resFolder is None + + # create folder directly in outputPath + directResFolder = outputPath / f"res_{simHash}" + fU.makeADir(directResFolder) + resFolder = fU.searchCom4ResDir(outputPath, simHash) + assert resFolder == directResFolder + + # remove it and create in peakFiles subfolder instead + shutil.rmtree(directResFolder) + peakResFolder = outputPath / "peakFiles" / f"res_{simHash}" + fU.makeADir(peakResFolder) + resFolder = fU.searchCom4ResDir(outputPath, simHash) + assert resFolder == peakResFolder + + # different simHash should not be found + resFolder = fU.searchCom4ResDir(outputPath, "otherHash") + assert resFolder is None + + +def test_deleteCom4Results(tmp_path): + """ test deleting com4FlowPy results folder and json file """ + outputPath = pathlib.Path(tmp_path) / "Outputs" + simHash = "abc123" + + jsonFile = outputPath / f"{simHash}.json" + resFolder = outputPath / f"res_{simHash}" + fU.makeADir(resFolder) + with open(jsonFile, "w") as f: + json.dump({"simHash": simHash}, f) + (resFolder / "dummy.txt").write_text("dummy") + + assert jsonFile.is_file() + assert resFolder.is_dir() + + # deleting a non-existing simHash should not raise and not touch existing files + fU.deleteCom4Results(outputPath, "otherHash") + assert jsonFile.is_file() + assert resFolder.is_dir() + + # deleting existing simHash removes both json and folder + fU.deleteCom4Results(outputPath, simHash) + assert not jsonFile.exists() + assert not resFolder.exists() + + # calling again should not raise even though nothing is left to delete + fU.deleteCom4Results(outputPath, simHash) + assert not jsonFile.exists() + assert not resFolder.exists() + + """ test deleting com4FlowPy results folder located in peakFiles subfolder """ + outputPath = pathlib.Path(tmp_path) / "Outputs" + simHash = "peak123" + + jsonFile = outputPath / f"{simHash}.json" + resFolder = outputPath / "peakFiles" / f"res_{simHash}" + fU.makeADir(resFolder) + with open(jsonFile, "w") as f: + json.dump({"simHash": simHash}, f) + + assert jsonFile.is_file() + assert resFolder.is_dir() + + fU.deleteCom4Results(outputPath, "otherHash") + assert jsonFile.is_file() + assert resFolder.is_dir() + + fU.deleteCom4Results(outputPath, simHash) + assert not jsonFile.exists() + assert not resFolder.exists() + + fU.deleteCom4Results(outputPath, simHash) + assert not jsonFile.exists() + assert not resFolder.exists() + + +def test_backupCom4Results(tmp_path): + """ test backing up com4FlowPy results folder and json file """ + outputPath = pathlib.Path(tmp_path) / "Outputs" + backupPath = outputPath / "backup" + simHash = "abc123" + + jsonFile = outputPath / f"{simHash}.json" + resFolder = outputPath / f"res_{simHash}" + fU.makeADir(resFolder) + with open(jsonFile, "w") as f: + json.dump({"simHash": simHash}, f) + (resFolder / "dummy.txt").write_text("dummy") + + assert jsonFile.exists() + assert resFolder.exists() + + fU.backupCom4Results(outputPath, simHash) + + # originals should be gone + assert not jsonFile.exists() + assert not resFolder.exists() + + # backups should exist + backupJson = backupPath / f"{simHash}.json" + backupFolder = backupPath / f"res_{simHash}" + assert backupJson.is_file() + assert backupFolder.is_dir() + assert (backupFolder / "dummy.txt").is_file() + + """ test that backing up twice does not overwrite previous backup """ + + jsonFile = outputPath / f"{simHash}.json" + resFolder = outputPath / f"res_{simHash}" + fU.makeADir(resFolder) + with open(jsonFile, "w") as f: + json.dump({"simHash": simHash}, f) + (resFolder / "dummy.txt").write_text("dummy") + + fU.backupCom4Results(outputPath, simHash) + assert not (outputPath / f"{simHash}.json").exists() + assert not (outputPath / f"res_{simHash}").exists() + + # both original and "(1)" suffixed backups should exist + assert (backupPath / f"{simHash}.json").is_file() + assert (backupPath / f"res_{simHash}").is_dir() + assert (backupPath / f"{simHash}(1).json").is_file() + assert (backupPath / f"res_{simHash}(1)").is_dir() + + """ test backing up when no results exist yet does not raise """ + outputPath = pathlib.Path(tmp_path) + simHash = "doesNotExist" + + fU.backupCom4Results(outputPath, simHash) + + backupPath = outputPath / "backup" + assert backupPath.is_dir() + assert not (backupPath / f"{simHash}.json").exists() + assert not (backupPath / f"res_{simHash}").exists() diff --git a/docs/moduleCom4FlowPy.rst b/docs/moduleCom4FlowPy.rst index 002b3ac7c..7efdc1ffb 100644 --- a/docs/moduleCom4FlowPy.rst +++ b/docs/moduleCom4FlowPy.rst @@ -34,9 +34,41 @@ Running the code ---------------- Generate an environment as described in :ref:`developinstall:Script Installation (Linux)` or -:ref:`developinstallwin:Script Installation (Windows)`. Run the model via:: +:ref:`developinstallwin:Script Installation (Windows)`. +Either modify the ``avaframe/com4FlowPy/(local_)com4FlowPyCfg.ini`` according to your requirements and run the model directly from the command-line via:: pixi run python runCom4FlowPy.py + +or setup model parameters/config, import and run ``runCom4FlowPy`` from an external script similar to this example:: + + from avaframe import runCom4FlowPy + import configparser + + cfg = configparser.ConfigParser() + + cfg["GENERAL"] = { + "alpha": "40", + "exp": "8", + ... + } + + cfg["PATHS"] = { + "useCustomPaths" = "True", + "outPputFileFormat" = ".tif", + ... + } + + results = runCom4FlowPy.main(cfg=cfg) # actual model execution + + # example for printing values of returned dictionary containing + # information on the return status of the model run + print("simulation {} completed?: {}".format(results["uid"], results["simulationPerformed"]) + print("return status: {}".format(results["message"])) + +**Note:** +Setup of the ConfigParser object needs to reflect the structure of the ``avaframe/com4FlowPy/(local_)com4FlowPyCfg.ini``. +In this context the configuration file can be overwritten, for more information see :ref:`complexUsage:Override configuration`. + Configuration @@ -256,6 +288,15 @@ If ``infra = True`` this layer will be written automatically (no need to separat - ``backcalculation``: Parts of modeled process paths upslope of infrastructure cells that are ''hit'' by (a) modeled process(es). +.. Note:: + + Behavior of the model in case of already existing model results for the exact same configuration can be controlled by the ``overwrite`` parameter in the cfg + + - ``default``: if model results with same configuration already exist from a previous run, then the simulation is not performed and existing results are kept + - ``reRunAndOverwrite``: model is run regardless of pre-existing results with same configuration -- existing results are deleted/overwritten + - ``reRunAndBackup``: model is run regardless of pre-existing results with same configuration -- existing results are moved to a backup Folder + + .. Model Parameterisation .. ------------------------ ..