diff --git a/.github/workflows/httomo_docs.yml b/.github/workflows/httomo_docs.yml index d0a908cad..375284420 100644 --- a/.github/workflows/httomo_docs.yml +++ b/.github/workflows/httomo_docs.yml @@ -39,6 +39,8 @@ jobs: - name: Generate full yaml pipelines using pipeline directives run: | python ./docs/source/scripts/execute_pipelines_build.py -o ./docs/source/pipelines_full/ + python ./docs/source/scripts/modify_pipeline.py -i ./docs/source/pipelines_full/tomopy_tomobank.yaml + python ./docs/source/scripts/modify_pipeline.py -i ./docs/source/pipelines_full/FBP3d_tomobar_tomobank.yaml - name: Create ZIP archive for pipelines run: | diff --git a/docs/source/_static/real_data/recon_tomo088.jpg b/docs/source/_static/real_data/recon_tomo088.jpg new file mode 100644 index 000000000..9d8ff3fbd Binary files /dev/null and b/docs/source/_static/real_data/recon_tomo088.jpg differ diff --git a/docs/source/_static/real_data/sino_tomo088.jpg b/docs/source/_static/real_data/sino_tomo088.jpg new file mode 100644 index 000000000..d2fde7c10 Binary files /dev/null and b/docs/source/_static/real_data/sino_tomo088.jpg differ diff --git a/docs/source/howto/how_to_run/real_data_example.rst b/docs/source/howto/how_to_run/real_data_example.rst new file mode 100644 index 000000000..ec81314e3 --- /dev/null +++ b/docs/source/howto/how_to_run/real_data_example.rst @@ -0,0 +1,80 @@ +.. _real-data-example: + +Real data processing +==================== + +This section presents an example of processing real experimental data using HTTomo from the `TomoBank`_ data archive. + +.. list-table:: + + + * - .. figure:: ../../_static/real_data/sino_tomo088.jpg + + Dark/Flat field corrected sinogram of the `Lorentz data set`_. + + - .. figure:: ../../_static/real_data/recon_tomo088.jpg + + Reconstructed slice using FBP method + + +Before starting, we assume that HTTomo has been successfully installed. If you have not installed HTTomo yet, please follow the +:ref:`installation_main`. + +We also recommend running the :ref:`run_tests` to verify that all required dependencies are installed and that the framework is +functioning correctly. + +For this example, we will use raw data from the `TomoBank`_ data archive. Please download the `Lorentz data set`_. The dataset is +hosted using the Globus file management system, which requires authentication. You can sign in using your GitHub credentials. + +.. _TomoBank: https://tomobank.readthedocs.io/en/latest/ + +.. _Lorentz data set: https://tomobank.readthedocs.io/en/latest/source/data/docs.data.lorentz.html + +Once the dataset has been downloaded, you should have the file :code:`tomo_00088.h5` on your disk. You can then proceed with running a simple HTTomo pipeline. + +TomoPy (CPU) pipeline ++++++++++++++++++++++ + +This pipeline uses the CPU implementation of the TomoPy library. TomoPy must be installed before running the pipeline. +See :ref:`backends_list`. + +Running this pipeline requires TomoPy package to be installed, see :ref:`backends_list`. Copy the following pipeline into a YAML file, +for example: :code:`tomopy_tomo_00088.yaml`. + +.. dropdown:: Standard 180 degrees pipeline using TomoPy (CPU) for tomo_00088.h5 dataset + + .. literalinclude:: ../../pipelines_full/tomopy_tomobank.yaml + :language: yaml + + +Then run HTTomo according to :ref:`howto_run_outside_diamond` documentation. In particular, provide the path to the input dataset, the pipeline YAML file, and the output directory. + +.. code-block:: console + + $ python -m httomo run path/to/tomo_00088.h5 tomopy_tomo_00088.yaml /path/to/output_folder + +GPU pipeline +++++++++++++ + +If a CUDA-enabled GPU is available, the same dataset can be processed using GPU-accelerated libraries. +This can significantly reduce the processing time for suitable pipelines. Run the pipeline bellow in a similar way as explained above. + +.. dropdown:: GPU-enabled processing for tomo_00088.h5 dataset + + .. literalinclude:: ../../pipelines_full/FBP3d_tomobar_tomobank.yaml + :language: yaml + +Output results +++++++++++++++ + +In the output folder you will find: + +1. Copied YAML file with the executed pipeline. +2. Debug log and the user log, see more :ref:`info_logger`. +3. The result of the reconstruction saved as HDF5 file. This file can be open, for instance with `Dawn`_ software. +4. Saved tiff files of the reconstructed image. You can use `ImageJ`_ or `ImageJ.JS`_ to visualise. + + +.. _Dawn: https://dawnsci.org/ +.. _ImageJ: https://imagej.net/ij/ +.. _ImageJ.JS: https://ij.imjoy.io/ \ No newline at end of file diff --git a/docs/source/howto/how_to_run/run_in_depth.rst b/docs/source/howto/how_to_run/run_in_depth.rst index 27122824b..fc9f8677b 100644 --- a/docs/source/howto/how_to_run/run_in_depth.rst +++ b/docs/source/howto/how_to_run/run_in_depth.rst @@ -13,7 +13,7 @@ As mentioned earlier, the preliminary step to accessing installed HTTomo softwar depends on if you are using a Diamond machine or not: - not on a Diamond machine: activate the conda environment that HTTomo was - installed into (please refer to :doc:`installation` for instructions on how to + installed into (please refer to :ref:`installation_main` for instructions on how to install HTTomo) - on a Diamond machine: run the command :code:`module load httomo` diff --git a/docs/source/howto/run_httomo.rst b/docs/source/howto/run_httomo.rst index 7532baae5..6b77d71d1 100644 --- a/docs/source/howto/run_httomo.rst +++ b/docs/source/howto/run_httomo.rst @@ -39,4 +39,5 @@ split into these two subsections where relevant. how_to_run/at_diamond how_to_run/outside_diamond how_to_run/run_in_depth + how_to_run/real_data_example diff --git a/docs/source/scripts/modify_pipeline.py b/docs/source/scripts/modify_pipeline.py new file mode 100644 index 000000000..237650588 --- /dev/null +++ b/docs/source/scripts/modify_pipeline.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# --------------------------------------------------------------------------- +# Copyright 2022 Diamond Light Source Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- +# Created By : Tomography Team +# Created Date: 11/August/2026 +# version ='0.1' +# --------------------------------------------------------------------------- +"""This script modifies a given pipeline by changing parameters in it.""" + +import argparse +import yaml +from typing import Union + + +def get_args(): + parser = argparse.ArgumentParser( + description="Script that modifies parameters in a YAML pipeline for HTTomo " + ) + parser.add_argument( + "-i", + "--input", + type=str, + default="./", + help="Full path to a specific pipeline that needs modification.", + ) + return parser.parse_args() + + +def change_value_parameters_method_pipeline( + yaml_path: str, + method: list, + key: list, + value: list, + save_result: Union[None, bool] = None, +): + # changes methods parameters in the given pipeline and re-save the pipeline + with open(yaml_path, "r") as f: + conf = list(yaml.load_all(f, Loader=yaml.FullLoader)) + opened_yaml = conf[0] + methods_no = len(opened_yaml) + methods_no_correct = len(method) + for i in range(methods_no): + method_content = opened_yaml[i] + method_name = method_content["method"] + for j in range(methods_no_correct): + if method[j] == method_name: + # change something in parameters here + opened_yaml[i]["parameters"][key[j]] = value[j] + if save_result is not None: + # add save_result to the list of keys + opened_yaml[i]["save_result"] = save_result + + with open(yaml_path, "w") as file_descriptor: + yaml.dump( + opened_yaml, file_descriptor, default_flow_style=False, sort_keys=False + ) + + +if __name__ == "__main__": + args = get_args() + path_to_pipeline = args.input + + change_value_parameters_method_pipeline( + path_to_pipeline, + method=[ + "standard_tomo", + "standard_tomo", + "standard_tomo", + "standard_tomo", + "standard_tomo", + "standard_tomo", + "find_center_vo", + ], + key=[ + "preview", + "data_path", + "image_key_path", + "rotation_angles", + "darks", + "flats", + "ind", + ], + value=[ + {"detector_y": {"start": 500, "stop": 510}}, + "/exchange/data", + None, + { + "user_defined": { + "start_angle": 0, + "stop_angle": 179.876, + "angles_total": 1500, + } + }, + { + "file": "input_data", + "image_key_path": None, + "data_path": "/exchange/data_dark", + }, + { + "file": "input_data", + "image_key_path": None, + "data_path": "/exchange/data_white", + }, + "mid", + ], + ) + + message_str = f"Pipeline {path_to_pipeline} has been modified." + print(message_str) diff --git a/tests/conftest.py b/tests/conftest.py index 7be2cab22..4d6a97d66 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,6 +6,7 @@ from typing import Any, Callable, Dict, List, TypeAlias, Union, Tuple import numpy as np from PIL import Image +import h5py import pytest import yaml @@ -234,6 +235,16 @@ def angles_averaging(): return "docs/source/pipelines_full/angles_averaging.yaml" +@pytest.fixture +def tomopy_tomobank(): + return "docs/source/pipelines_full/tomopy_tomobank.yaml" + + +@pytest.fixture +def FBP3d_tomobar_tomobank(): + return "docs/source/pipelines_full/FBP3d_tomobar_tomobank.yaml" + + @pytest.fixture def FISTA3d_tomobar(): return "docs/source/pipelines_full/FISTA3d_tomobar.yaml" @@ -317,6 +328,12 @@ def i12_119647(): return "tests/test_data/raw_data/i12/119647.nxs" +@pytest.fixture +def tomobank_00088(): + # TomoBank data + return "tests/test_data/raw_data/tomobank/tomo_00088.h5" + + ############## --Ground Truth references-- ################# @@ -386,6 +403,18 @@ def FISTA3d_tomobar_k11_38731_npz(): return np.load("tests/test_data/raw_data/diad/FISTA3d_tomobar_k11_38731.npz") +@pytest.fixture +def tomobank00088_tomopy_npz(): + # 10 slices numpy array + return np.load("tests/test_data/raw_data/tomobank/tomobank00088_tomopy.npz") + + +@pytest.fixture +def tomobank00088_FBP3d_tomobar(): + # 10 slices numpy array + return np.load("tests/test_data/raw_data/tomobank/tomobank00088_FBP3d_tomobar.npz") + + @pytest.fixture def pipeline_sweep_FBP3d_tomobar_i13_177906_tiffs(): # 8 tiff files of 32bit @@ -539,6 +568,44 @@ def compare_tif(files_list_to_compare: list, file_path_to_references: list): assert res_norm < 1e-3 +def calculate_gt_residual( + path_to_data: str, + h5_file_name: str, + h5_files: list, + data_gt: np.ndarray, + axis_slice: int, +) -> float: + + slices, sizeX, sizeY = np.shape(data_gt) + step = axis_slice // (slices + 2) + data_result = np.zeros( + (slices, sizeX, sizeY), + dtype=np.float32, + ) + for file_to_open in h5_files: + if h5_file_name in file_to_open: + with h5py.File(file_to_open, "r") as h5f: + dataset = h5f[path_to_data] + + if not isinstance(dataset, h5py.Dataset): + raise TypeError(f"{path_to_data!r} is not an HDF5 dataset.") + + index_prog = step + + for i in range(slices): + data_result[i, :, :] = dataset[:, index_prog, :] + index_prog += step + + break + else: + raise FileNotFoundError( + f"File name containing '{h5_file_name}' cannot be found." + ) + residual_im = data_gt - data_result + res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") + return res_norm + + def change_value_parameters_method_pipeline( yaml_path: str, method: list, diff --git a/tests/test_parallel_pipeline_big.py b/tests/test_parallel_pipeline_big.py index e64eed2b7..18347b7f6 100644 --- a/tests/test_parallel_pipeline_big.py +++ b/tests/test_parallel_pipeline_big.py @@ -1,17 +1,16 @@ -import subprocess -from typing import Callable, List, Tuple, Union +# NOTE: those tests have path integrated that are compatible with running jobs in Jenkins at DLS infrastructure. +from typing import Callable from subprocess import Popen, PIPE import os - -import h5py -import numpy as np import pytest -from plumbum import local -from .conftest import change_value_parameters_method_pipeline, check_tif, compare_tif - -# NOTE: those tests have path integrated that are compatible with running jobs in Jenkins at DLS infrastructure. +from .conftest import ( + change_value_parameters_method_pipeline, + check_tif, + compare_tif, + calculate_gt_residual, +) ######################################################################## @@ -72,28 +71,14 @@ def test_pipe_parallel_FBP3d_tomobar_k11_38730_in_disk_preview( # load the pre-saved numpy array for comparison bellow data_gt = FBP3d_tomobar_k11_38730_npz["data"] axis_slice = FBP3d_tomobar_k11_38730_npz["axis_slice"] - slices, sizeX, sizeY = np.shape(data_gt) - - step = axis_slice // (slices + 2) - # store for the result - data_result = np.zeros((slices, sizeX, sizeY), dtype=np.float32) - - path_to_data = "data/" - h5_file_name = "FBP3d_tomobar" - for file_to_open in h5_files: - if h5_file_name in file_to_open: - h5f = h5py.File(file_to_open, "r") - index_prog = step - for i in range(slices): - data_result[i, :, :] = h5f[path_to_data][:, index_prog, :] - index_prog += step - h5f.close() - else: - message_str = f"File name with {h5_file_name} string cannot be found." - raise FileNotFoundError(message_str) - - residual_im = data_gt - data_result - res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") + + res_norm = calculate_gt_residual( + path_to_data="data/", + h5_file_name="FBP3d_tomobar", + h5_files=h5_files, + data_gt=data_gt, + axis_slice=axis_slice, + ) assert res_norm < 1e-6 @@ -151,28 +136,13 @@ def test_pipe_parallel_FBP3d_tomobar_k11_38730_in_memory_preview( # load the pre-saved numpy array for comparison bellow data_gt = FBP3d_tomobar_k11_38730_npz["data"] axis_slice = FBP3d_tomobar_k11_38730_npz["axis_slice"] - slices, sizeX, sizeY = np.shape(data_gt) - - step = axis_slice // (slices + 2) - # store for the result - data_result = np.zeros((slices, sizeX, sizeY), dtype=np.float32) - - path_to_data = "data/" - h5_file_name = "FBP3d_tomobar" - for file_to_open in h5_files: - if h5_file_name in file_to_open: - h5f = h5py.File(file_to_open, "r") - index_prog = step - for i in range(slices): - data_result[i, :, :] = h5f[path_to_data][:, index_prog, :] - index_prog += step - h5f.close() - else: - message_str = f"File name with {h5_file_name} string cannot be found." - raise FileNotFoundError(message_str) - - residual_im = data_gt - data_result - res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") + res_norm = calculate_gt_residual( + path_to_data="data/", + h5_file_name="FBP3d_tomobar", + h5_files=h5_files, + data_gt=data_gt, + axis_slice=axis_slice, + ) assert res_norm < 1e-6 @@ -222,28 +192,14 @@ def test_angles_averaging_LPRec_i12_119647_preview( # load the pre-saved numpy array for comparison bellow data_gt = angle_average_LPrec_i12_119647_npz["data"] axis_slice = angle_average_LPrec_i12_119647_npz["axis_slice"] - slices, sizeX, sizeY = np.shape(data_gt) - - step = axis_slice // (slices + 2) - # store for the result - data_result = np.zeros((slices, sizeX, sizeY), dtype=np.float32) - - path_to_data = "data/" - h5_file_name = "LPRec3d_tomobar" - for file_to_open in h5_files: - if h5_file_name in file_to_open: - h5f = h5py.File(file_to_open, "r") - index_prog = step - for i in range(slices): - data_result[i, :, :] = h5f[path_to_data][:, index_prog, :] - index_prog += step - h5f.close() - else: - message_str = f"File name with {h5_file_name} string cannot be found." - raise FileNotFoundError(message_str) - - residual_im = data_gt - data_result - res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") + + res_norm = calculate_gt_residual( + path_to_data="data/", + h5_file_name="LPRec3d_tomobar", + h5_files=h5_files, + data_gt=data_gt, + axis_slice=axis_slice, + ) assert res_norm < 0.1 @@ -295,28 +251,13 @@ def test_parallel_pipe_LPRec3d_tomobar_i12_119647_preview( # load the pre-saved numpy array for comparison bellow data_gt = LPRec3d_tomobar_i12_119647_npz["data"] axis_slice = LPRec3d_tomobar_i12_119647_npz["axis_slice"] - slices, sizeX, sizeY = np.shape(data_gt) - - step = axis_slice // (slices + 2) - # store for the result - data_result = np.zeros((slices, sizeX, sizeY), dtype=np.float32) - - path_to_data = "data/" - h5_file_name = "LPRec3d_tomobar" - for file_to_open in h5_files: - if h5_file_name in file_to_open: - h5f = h5py.File(file_to_open, "r") - index_prog = step - for i in range(slices): - data_result[i, :, :] = h5f[path_to_data][:, index_prog, :] - index_prog += step - h5f.close() - else: - message_str = f"File name with {h5_file_name} string cannot be found." - raise FileNotFoundError(message_str) - - residual_im = data_gt - data_result - res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") + res_norm = calculate_gt_residual( + path_to_data="data/", + h5_file_name="LPRec3d_tomobar", + h5_files=h5_files, + data_gt=data_gt, + axis_slice=axis_slice, + ) assert ( res_norm < 0.2 ) # TODO: known issue with the Log-Polar, the tolerance will be reduced when fixed @@ -379,28 +320,13 @@ def test_parallel_pipe_360deg_distortion_FBP3d_tomobar_i13_179623_preview( # load the pre-saved numpy array for comparison bellow data_gt = FBP3d_tomobar_distortion_i13_179623_npz["data"] axis_slice = FBP3d_tomobar_distortion_i13_179623_npz["axis_slice"] - slices, sizeX, sizeY = np.shape(data_gt) - - step = axis_slice // (slices + 2) - # store for the result - data_result = np.zeros((slices, sizeX, sizeY), dtype=np.float32) - - path_to_data = "data/" - h5_file_name = "FBP3d_tomobar" - for file_to_open in h5_files: - if h5_file_name in file_to_open: - h5f = h5py.File(file_to_open, "r") - index_prog = step - for i in range(slices): - data_result[i, :, :] = h5f[path_to_data][:, index_prog, :] - index_prog += step - h5f.close() - else: - message_str = f"File name with {h5_file_name} string cannot be found." - raise FileNotFoundError(message_str) - - residual_im = data_gt - data_result - res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") + res_norm = calculate_gt_residual( + path_to_data="data/", + h5_file_name="FBP3d_tomobar", + h5_files=h5_files, + data_gt=data_gt, + axis_slice=axis_slice, + ) assert res_norm < 1e-4 @@ -519,28 +445,14 @@ def test_parallel_pipe_FISTA3d_tomobar_k11_38731( # load the pre-saved numpy array for comparison bellow data_gt = FISTA3d_tomobar_k11_38731_npz["data"] axis_slice = FISTA3d_tomobar_k11_38731_npz["axis_slice"] - slices, sizeX, sizeY = np.shape(data_gt) - - step = axis_slice // (slices + 2) - # store for the result - data_result = np.zeros((slices, sizeX, sizeY), dtype=np.float32) - - path_to_data = "data/" - h5_file_name = "FISTA3d_tomobar" - for file_to_open in h5_files: - if h5_file_name in file_to_open: - h5f = h5py.File(file_to_open, "r") - index_prog = step - for i in range(slices): - data_result[i, :, :] = h5f[path_to_data][:, index_prog, :] - index_prog += step - h5f.close() - else: - message_str = f"File name with {h5_file_name} string cannot be found." - raise FileNotFoundError(message_str) - - residual_im = data_gt - data_result - res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") + res_norm = calculate_gt_residual( + path_to_data="data/", + h5_file_name="FISTA3d_tomobar", + h5_files=h5_files, + data_gt=data_gt, + axis_slice=axis_slice, + ) + assert res_norm < 1e-2 diff --git a/tests/test_pipeline_big.py b/tests/test_pipeline_big.py index ba6a74adf..8273e8f23 100644 --- a/tests/test_pipeline_big.py +++ b/tests/test_pipeline_big.py @@ -1,13 +1,178 @@ +# NOTE: those tests have path integrated that are compatible with running jobs in Jenkins at DLS infrastructure. import subprocess -from typing import Callable, List, Tuple, Union -import h5py -import numpy as np +from typing import Callable import pytest import os -from plumbum import local -from .conftest import change_value_parameters_method_pipeline, check_tif, compare_tif +from .conftest import ( + change_value_parameters_method_pipeline, + check_tif, + compare_tif, + calculate_gt_residual, +) -# NOTE: those tests have path integrated that are compatible with running jobs in Jenkins at DLS infrastructure. + +@pytest.mark.full_data +def test_pipe_tomopy_tomobank_preview( + get_files: Callable, + cmd, + tomobank_00088, + tomopy_tomobank, + tomobank00088_tomopy_npz, + output_folder, +): + change_value_parameters_method_pipeline( + tomopy_tomobank, + method=[ + "standard_tomo", + "standard_tomo", + "standard_tomo", + "standard_tomo", + "standard_tomo", + "standard_tomo", + "find_center_vo", + ], + key=[ + "preview", + "data_path", + "image_key_path", + "rotation_angles", + "darks", + "flats", + "ind", + ], + value=[ + {"detector_y": {"start": 500, "stop": 510}}, + "/exchange/data", + None, + { + "user_defined": { + "start_angle": 0, + "stop_angle": 179.876, + "angles_total": 1500, + } + }, + { + "file": "input_data", + "image_key_path": None, + "data_path": "/exchange/data_dark", + }, + { + "file": "input_data", + "image_key_path": None, + "data_path": "/exchange/data_white", + }, + "mid", + ], + ) + + cmd.pop(4) #: don't save all + cmd.insert(5, tomobank_00088) + cmd.insert(7, tomopy_tomobank) + cmd.insert(8, output_folder) + + subprocess.check_output(cmd) + + files = get_files(output_folder) + + #: check the generated reconstruction (hdf5 file) + h5_files = list(filter(lambda x: ".h5" in x, files)) + assert len(h5_files) == 1 + + # load the pre-saved numpy array for comparison bellow + data_gt = tomobank00088_tomopy_npz["data"] + axis_slice = tomobank00088_tomopy_npz["axis_slice"] + + res_norm = calculate_gt_residual( + path_to_data="data/", + h5_file_name="tomopy", + h5_files=h5_files, + data_gt=data_gt, + axis_slice=axis_slice, + ) + + assert res_norm < 0.1 + + +@pytest.mark.full_data +def test_pipe_FBP3d_tomobar_tomobank_preview( + get_files: Callable, + cmd, + tomobank_00088, + FBP3d_tomobar_tomobank, + tomobank00088_FBP3d_tomobar, + output_folder, +): + change_value_parameters_method_pipeline( + FBP3d_tomobar_tomobank, + method=[ + "standard_tomo", + "standard_tomo", + "standard_tomo", + "standard_tomo", + "standard_tomo", + "standard_tomo", + "find_center_vo", + ], + key=[ + "preview", + "data_path", + "image_key_path", + "rotation_angles", + "darks", + "flats", + "ind", + ], + value=[ + {"detector_y": {"start": 500, "stop": 510}}, + "/exchange/data", + None, + { + "user_defined": { + "start_angle": 0, + "stop_angle": 179.876, + "angles_total": 1500, + } + }, + { + "file": "input_data", + "image_key_path": None, + "data_path": "/exchange/data_dark", + }, + { + "file": "input_data", + "image_key_path": None, + "data_path": "/exchange/data_white", + }, + "mid", + ], + ) + + cmd.pop(4) #: don't save all + cmd.insert(5, tomobank_00088) + cmd.insert(7, FBP3d_tomobar_tomobank) + cmd.insert(8, output_folder) + + subprocess.check_output(cmd) + + files = get_files(output_folder) + + #: check the generated reconstruction (hdf5 file) + h5_files = list(filter(lambda x: ".h5" in x, files)) + assert len(h5_files) == 1 + + # load the pre-saved numpy array for comparison bellow + data_gt = tomobank00088_FBP3d_tomobar["data"] + axis_slice = tomobank00088_FBP3d_tomobar["axis_slice"] + + res_norm = calculate_gt_residual( + path_to_data="data/", + h5_file_name="FBP3d_tomobar", + h5_files=h5_files, + data_gt=data_gt, + axis_slice=axis_slice, + ) + + assert res_norm < 1e-4 @pytest.mark.full_data @@ -59,28 +224,13 @@ def test_pipe_FBP3d_tomobar_k11_38731_in_disk( # load the pre-saved numpy array for comparison bellow data_gt = FBP3d_tomobar_k11_38731_npz["data"] axis_slice = FBP3d_tomobar_k11_38731_npz["axis_slice"] - slices, sizeX, sizeY = np.shape(data_gt) - - step = axis_slice // (slices + 2) - # store for the result - data_result = np.zeros((slices, sizeX, sizeY), dtype=np.float32) - - path_to_data = "data/" - h5_file_name = "FBP3d_tomobar" - for file_to_open in h5_files: - if h5_file_name in file_to_open: - h5f = h5py.File(file_to_open, "r") - index_prog = step - for i in range(slices): - data_result[i, :, :] = h5f[path_to_data][:, index_prog, :] - index_prog += step - h5f.close() - else: - message_str = f"File name with {h5_file_name} string cannot be found." - raise FileNotFoundError(message_str) - - residual_im = data_gt - data_result - res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") + res_norm = calculate_gt_residual( + path_to_data="data/", + h5_file_name="FBP3d_tomobar", + h5_files=h5_files, + data_gt=data_gt, + axis_slice=axis_slice, + ) assert res_norm < 1e-6 @@ -127,28 +277,13 @@ def test_pipe_FBP3d_tomobar_i12_119647_preview( # load the pre-saved numpy array for comparison bellow data_gt = FBP3d_tomobar_i12_119647_npz["data"] axis_slice = FBP3d_tomobar_i12_119647_npz["axis_slice"] - slices, sizeX, sizeY = np.shape(data_gt) - - step = axis_slice // (slices + 2) - # store for the result - data_result = np.zeros((slices, sizeX, sizeY), dtype=np.float32) - - path_to_data = "data/" - h5_file_name = "FBP3d_tomobar" - for file_to_open in h5_files: - if h5_file_name in file_to_open: - h5f = h5py.File(file_to_open, "r") - index_prog = step - for i in range(slices): - data_result[i, :, :] = h5f[path_to_data][:, index_prog, :] - index_prog += step - h5f.close() - else: - message_str = f"File name with {h5_file_name} string cannot be found." - raise FileNotFoundError(message_str) - - residual_im = data_gt - data_result - res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") + res_norm = calculate_gt_residual( + path_to_data="data/", + h5_file_name="FBP3d_tomobar", + h5_files=h5_files, + data_gt=data_gt, + axis_slice=axis_slice, + ) assert res_norm < 1e-4 @@ -197,28 +332,14 @@ def test_pipe_LPRec3d_tomobar_i12_119647_preview( # load the pre-saved numpy array for comparison bellow data_gt = LPRec3d_tomobar_i12_119647_npz["data"] axis_slice = LPRec3d_tomobar_i12_119647_npz["axis_slice"] - slices, sizeX, sizeY = np.shape(data_gt) - - step = axis_slice // (slices + 2) - # store for the result - data_result = np.zeros((slices, sizeX, sizeY), dtype=np.float32) - - path_to_data = "data/" - h5_file_name = "LPRec3d_tomobar" - for file_to_open in h5_files: - if h5_file_name in file_to_open: - h5f = h5py.File(file_to_open, "r") - index_prog = step - for i in range(slices): - data_result[i, :, :] = h5f[path_to_data][:, index_prog, :] - index_prog += step - h5f.close() - else: - message_str = f"File name with {h5_file_name} string cannot be found." - raise FileNotFoundError(message_str) - - residual_im = data_gt - data_result - res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") + res_norm = calculate_gt_residual( + path_to_data="data/", + h5_file_name="LPRec3d_tomobar", + h5_files=h5_files, + data_gt=data_gt, + axis_slice=axis_slice, + ) + assert res_norm < 1e-4 @@ -264,28 +385,14 @@ def test_pipe_FBP2d_astra_i12_119647_preview( # load the pre-saved numpy array for comparison bellow data_gt = FBP2d_astra_i12_119647_npz["data"] axis_slice = FBP2d_astra_i12_119647_npz["axis_slice"] - slices, sizeX, sizeY = np.shape(data_gt) - - step = axis_slice // (slices + 2) - # store for the result - data_result = np.zeros((slices, sizeX, sizeY), dtype=np.float32) - - path_to_data = "data/" - h5_file_name = "FBP2d_astra" - for file_to_open in h5_files: - if h5_file_name in file_to_open: - h5f = h5py.File(file_to_open, "r") - index_prog = step - for i in range(slices): - data_result[i, :, :] = h5f[path_to_data][:, index_prog, :] - index_prog += step - h5f.close() - else: - message_str = f"File name with {h5_file_name} string cannot be found." - raise FileNotFoundError(message_str) - - residual_im = data_gt - data_result - res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") + + res_norm = calculate_gt_residual( + path_to_data="data/", + h5_file_name="FBP2d_astra", + h5_files=h5_files, + data_gt=data_gt, + axis_slice=axis_slice, + ) assert res_norm < 1e-6 @@ -377,31 +484,17 @@ def test_pipe_FBP3d_tomobar_denoising_i13_177906_preview( assert len(h5_files) == 1 # load the pre-saved numpy array for comparison bellow - data_gt_tv = FBP3d_tomobar_TVdenoising_i13_177906_npz["data"] + data_gt = FBP3d_tomobar_TVdenoising_i13_177906_npz["data"] axis_slice = FBP3d_tomobar_TVdenoising_i13_177906_npz["axis_slice"] - slices, sizeX, sizeY = np.shape(data_gt_tv) - - step = axis_slice // (slices + 2) - # store for the result - data_result = np.zeros((slices, sizeX, sizeY), dtype=np.float32) - - path_to_data = "data/" - h5_file_name = "total_variation_PD" - for file_to_open in h5_files: - if h5_file_name in file_to_open: - h5f = h5py.File(file_to_open, "r") - index_prog = step - for i in range(slices): - data_result[i, :, :] = h5f[path_to_data][:, index_prog, :] - index_prog += step - h5f.close() - else: - message_str = f"File name with {h5_file_name} string cannot be found." - raise FileNotFoundError(message_str) - - residual_im = data_gt_tv - data_result - res_norm_tv_res = np.linalg.norm(residual_im.flatten()).astype("float32") - assert res_norm_tv_res < 0.1 + res_norm = calculate_gt_residual( + path_to_data="data/", + h5_file_name="total_variation_PD", + h5_files=h5_files, + data_gt=data_gt, + axis_slice=axis_slice, + ) + + assert res_norm < 0.1 # ######################################################################## @@ -454,28 +547,14 @@ def test_pipe_360deg_paganin_FBP3d_tomobar_i13_179623_preview( # load the pre-saved numpy array for comparison bellow data_gt = FBP3d_tomobar_paganin_i13_179623_npz["data"] axis_slice = FBP3d_tomobar_paganin_i13_179623_npz["axis_slice"] - slices, sizeX, sizeY = np.shape(data_gt) - - step = axis_slice // (slices + 2) - # store for the result - data_result = np.zeros((slices, sizeX, sizeY), dtype=np.float32) - - path_to_data = "data/" - h5_file_name = "FBP3d_tomobar" - for file_to_open in h5_files: - if h5_file_name in file_to_open: - h5f = h5py.File(file_to_open, "r") - index_prog = step - for i in range(slices): - data_result[i, :, :] = h5f[path_to_data][:, index_prog, :] - index_prog += step - h5f.close() - else: - message_str = f"File name with {h5_file_name} string cannot be found." - raise FileNotFoundError(message_str) - - residual_im = data_gt - data_result - res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") + res_norm = calculate_gt_residual( + path_to_data="data/", + h5_file_name="FBP3d_tomobar", + h5_files=h5_files, + data_gt=data_gt, + axis_slice=axis_slice, + ) + assert res_norm < 1e-4