Skip to content

Refactor output export logic in dementpy.py #49

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
eusimeo merged 6 commits into from
Jul 16, 2026
+157 −160
Merged

Refactor output export logic in dementpy.py #49

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
9eeb434
Refactor output export logic in dementpy.py
jgwalkup Jul 15, 2026
2db0b16
Disable NetCDF export functionality
jgwalkup Jul 15, 2026
2499361
Comment out export_to_netcdf method and related logic
jgwalkup Jul 15, 2026
e9f69f7
Comment out xarray import in output.py
jgwalkup Jul 15, 2026
4c6adce
Update runtime.txt
jgwalkup Jul 15, 2026
9d8a211
Change export function for Initialization dictionary
jgwalkup Jul 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion input/runtime.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,6 @@ Init_PO4 0
Input_NH4 0
Input_PO4 0
direct 0.95
dispersal 0
dist 1
interval 1
output_CSV 1
9 changes: 3 additions & 6 deletions src/dementpy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ def main():
pulse = int(runtime.loc['pulse',1]) # number of pulses
cycle = int(runtime.loc['end_time',1]) # number of time steps in each pulse
interval = int(runtime.loc['interval',1]) # interval of time step to record outputs
Export_format = int(runtime.loc['output_CSV',1]) # interval of time step to record outputs
#Export_format = int(runtime.loc['output_CSV',1]) # interval of time step to record outputs
mode = int(runtime.loc['dispersal',1]) # 0:'default' or 1:'dispersal'

#...Initialize data by calling the Function: Initialize_Data()
Expand Down Expand Up @@ -102,10 +102,7 @@ def main():

#...export the Output_init object to the output_folder using the export() funtion in the utility module
os.chdir('../'+output_folder)
if Export_format == 1:
Output_init.export_to_csv(outname)
else:
Output_init.export_to_netcdf(outname)

Output_init.export_to_csv(outname)

if __name__ == '__main__':
main()
142 changes: 71 additions & 71 deletions src/initialization.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"""
import pandas as pd
import numpy as np
import xarray as xr
# import xarray as xr

import warnings
import numbers
Expand Down Expand Up @@ -219,83 +219,83 @@ def export_initialization_dict_to_csv(base_path: Path | str, d: dict) -> None:
# Print numbers
pd.Series(scalar_numbers).to_csv(base_path / "scalars.csv")

def export_to_netcdf(self, base_path: Path | str) -> None:
"""Export contents of the output file to a directory in NetCDF format.
- Each pandas.DataFrame member is saved to a separate .nc file.
- All pandas.Series members are combined and saved to a single 'series.nc' file.
- All scalar numerical members are grouped and saved to 'scalars.nc'.
# def export_to_netcdf(self, base_path: Path | str) -> None:
# """Export contents of the output file to a directory in NetCDF format.
# - Each pandas.DataFrame member is saved to a separate .nc file.
# - All pandas.Series members are combined and saved to a single 'series.nc' file.
# - All scalar numerical members are grouped and saved to 'scalars.nc'.

Parameters:
base_path : Path
A path that names the root directory where contents will be exported.
If the directory does not exist it will be created.
"""
# Create space for output
base_path = Path(base_path)
base_path.mkdir(parents=True, exist_ok=True)
# Parameters:
# base_path : Path
# A path that names the root directory where contents will be exported.
# If the directory does not exist it will be created.
# """
# # Create space for output
# base_path = Path(base_path)
# base_path.mkdir(parents=True, exist_ok=True)

# Collect all series and scalar data
series_data = dict()
scalar_numbers = dict()
# # Collect all series and scalar data
# series_data = dict()
# scalar_numbers = dict()

for name, member in vars(self).items():
# convert each DataFrame to an xarray Dataset and save to .nc
if isinstance(member, pd.DataFrame):
# Ensure column names are strings
member.columns = member.columns.astype(str)
if member.index.name is not None:
member.index.name = str(member.index.name)
fname = name + ".nc" # use the .nc extension
try:
xarray_member = xr.Dataset.from_dataframe(member)
xarray_member.to_netcdf(base_path / fname)
except Exception as e:
warnings.warn(
f"Could not export DataFrame '{name}' to NetCDF. Error: {e}"
)
# for name, member in vars(self).items():
# # convert each DataFrame to an xarray Dataset and save to .nc
# if isinstance(member, pd.DataFrame):
# # Ensure column names are strings
# member.columns = member.columns.astype(str)
# if member.index.name is not None:
# member.index.name = str(member.index.name)
# fname = name + ".nc" # use the .nc extension
# try:
# xarray_member = xr.Dataset.from_dataframe(member)
# xarray_member.to_netcdf(base_path / fname)
# except Exception as e:
# warnings.warn(
# f"Could not export DataFrame '{name}' to NetCDF. Error: {e}"
# )

elif isinstance(member, pd.Series):
series_data[name] = member
# elif isinstance(member, pd.Series):
# series_data[name] = member

elif isinstance(member, numbers.Number):
scalar_numbers[name] = member
# elif isinstance(member, numbers.Number):
# scalar_numbers[name] = member

elif name == "Initialization":
# Special case - Initialization dictionary
# Serialise it to a subfolder
path = base_path / name
export_initialization_dict_to_netcdf(path, member)
elif isinstance(member, pd.Series):
xrmember = xr.DataArray(member)
fname = name + ".nc"
xrmember.to_netcdf(base_path / fname)
# elif name == "Initialization":
# # Special case - Initialization dictionary
# # Serialise it to a subfolder
# path = base_path / name
# export_initialization_dict_to_netcdf(path, member)
# elif isinstance(member, pd.Series):
# xrmember = xr.DataArray(member)
# fname = name + ".nc"
# xrmember.to_netcdf(base_path / fname)

else:
warnings.warn(
f"Output member '{name}' has unsupported type '{type(member)}'. "
f"It has not been exported to the output directory '{base_path}'."
)
# else:
# warnings.warn(
# f"Output member '{name}' has unsupported type '{type(member)}'. "
# f"It has not been exported to the output directory '{base_path}'."
# )

# process and save Series
if series_data:
try:
# Combine all Series into a single DataFrame.
combined_series_df = pd.concat(series_data, axis=1)
# Convert the combined DataFrame to an xarray Dataset.
series_dataset = xr.Dataset.from_dataframe(combined_series_df)
# Save the Series Dataset to a single NetCDF file.
series_dataset.to_netcdf(base_path / "series.nc")
except ValueError as e:
# This handles the "duplicate labels" error if it occurs.
warnings.warn(
f"Could not export combined series due to an error: {e}. "
"Consider cleaning the index of your Series data first."
)
# # process and save Series
# if series_data:
# try:
# # Combine all Series into a single DataFrame.
# combined_series_df = pd.concat(series_data, axis=1)
# # Convert the combined DataFrame to an xarray Dataset.
# series_dataset = xr.Dataset.from_dataframe(combined_series_df)
# # Save the Series Dataset to a single NetCDF file.
# series_dataset.to_netcdf(base_path / "series.nc")
# except ValueError as e:
# # This handles the "duplicate labels" error if it occurs.
# warnings.warn(
# f"Could not export combined series due to an error: {e}. "
# "Consider cleaning the index of your Series data first."
# )

if scalar_numbers:
# Create an xarray Dataset directly from the dictionary of scalars.
# Each key will become a variable in the NetCDF file.
scalars_dataset = xr.Dataset(scalar_numbers)
# Save the scalars Dataset to a NetCDF file.
scalars_dataset.to_netcdf(base_path / "scalars.nc")
# if scalar_numbers:
# # Create an xarray Dataset directly from the dictionary of scalars.
# # Each key will become a variable in the NetCDF file.
# scalars_dataset = xr.Dataset(scalar_numbers)
# # Save the scalars Dataset to a NetCDF file.
# scalars_dataset.to_netcdf(base_path / "scalars.nc")

164 changes: 82 additions & 82 deletions src/output.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@
import numbers

from initialization import export_initialization_dict_to_csv
from initialization import export_initialization_dict_to_netcdf
# from initialization import export_initialization_dict_to_netcdf

import numpy as np
import pandas as pd
import xarray as xr
# import xarray as xr

class Output():
"""
Expand Down Expand Up @@ -336,7 +336,7 @@ def export_to_csv(self, base_path: Path | str) -> None:
# Special case - Initialization dictionary
# Serialise it to a subfolder
path = base_path / name
export_initialization_dict(path, member)
export_initialization_dict_to_csv(path, member)
else:
warnings.warn(
f"Output member '{name}' has unsupported type '{type(member)}'. "
Expand All @@ -351,82 +351,82 @@ def export_to_csv(self, base_path: Path | str) -> None:
# Print numbers
pd.Series(scalar_numbers).to_csv(base_path / "scalars.csv")

def export_to_netcdf(self, base_path: Path | str) -> None:
"""Export contents of the output file to a directory in NetCDF format.
- Each pandas.DataFrame member is saved to a separate .nc file.
- All pandas.Series members are combined and saved to a single 'series.nc' file.
- All scalar numerical members are grouped and saved to 'scalars.nc'.

Parameters:
base_path : Path
A path that names the root directory where contents will be exported.
If the directory does not exist it will be created.
"""
# Create space for output
base_path = Path(base_path)
base_path.mkdir(parents=True, exist_ok=True)

# Collect all series and scalar data
series_data = dict()
scalar_numbers = dict()

for name, member in vars(self).items():
# convert each DataFrame to an xarray Dataset and save to .nc
if isinstance(member, pd.DataFrame):
# Ensure column names are strings
member.columns = member.columns.astype(str)
if member.index.name is not None:
member.index.name = str(member.index.name)
fname = name + ".nc" # use the .nc extension
try:
xarray_member = xr.Dataset.from_dataframe(member)
xarray_member.to_netcdf(base_path / fname)
except Exception as e:
warnings.warn(
f"Could not export DataFrame '{name}' to NetCDF. Error: {e}"
)

elif isinstance(member, pd.Series):
series_data[name] = member

elif isinstance(member, numbers.Number):
scalar_numbers[name] = member

elif name == "Initialization":
# Special case - Initialization dictionary
# Serialise it to a subfolder
path = base_path / name
export_initialization_dict_to_netcdf(path, member)
elif isinstance(member, pd.Series):
xrmember = xr.DataArray(member)
fname = name + ".nc"
xrmember.to_netcdf(base_path / fname)

else:
warnings.warn(
f"Output member '{name}' has unsupported type '{type(member)}'. "
f"It has not been exported to the output directory '{base_path}'."
)

# process and save Series
if series_data:
try:
# Combine all Series into a single DataFrame.
combined_series_df = pd.concat(series_data, axis=1)
# Convert the combined DataFrame to an xarray Dataset.
series_dataset = xr.Dataset.from_dataframe(combined_series_df)
# Save the Series Dataset to a single NetCDF file.
series_dataset.to_netcdf(base_path / "series.nc")
except ValueError as e:
# This handles the "duplicate labels" error if it occurs.
warnings.warn(
f"Could not export combined series due to an error: {e}. "
"Consider cleaning the index of your Series data first."
)

if scalar_numbers:
# Create an xarray Dataset directly from the dictionary of scalars.
# Each key will become a variable in the NetCDF file.
scalars_dataset = xr.Dataset(scalar_numbers)
# Save the scalars Dataset to a NetCDF file.
scalars_dataset.to_netcdf(base_path / "scalars.nc")
# def export_to_netcdf(self, base_path: Path | str) -> None:
# """Export contents of the output file to a directory in NetCDF format.
# - Each pandas.DataFrame member is saved to a separate .nc file.
# - All pandas.Series members are combined and saved to a single 'series.nc' file.
# - All scalar numerical members are grouped and saved to 'scalars.nc'.

# Parameters:
# base_path : Path
# A path that names the root directory where contents will be exported.
# If the directory does not exist it will be created.
# """
# # Create space for output
# base_path = Path(base_path)
# base_path.mkdir(parents=True, exist_ok=True)

# # Collect all series and scalar data
# series_data = dict()
# scalar_numbers = dict()

# for name, member in vars(self).items():
# # convert each DataFrame to an xarray Dataset and save to .nc
# if isinstance(member, pd.DataFrame):
# # Ensure column names are strings
# member.columns = member.columns.astype(str)
# if member.index.name is not None:
# member.index.name = str(member.index.name)
# fname = name + ".nc" # use the .nc extension
# try:
# xarray_member = xr.Dataset.from_dataframe(member)
# xarray_member.to_netcdf(base_path / fname)
# except Exception as e:
# warnings.warn(
# f"Could not export DataFrame '{name}' to NetCDF. Error: {e}"
# )

# elif isinstance(member, pd.Series):
# series_data[name] = member

# elif isinstance(member, numbers.Number):
# scalar_numbers[name] = member

# elif name == "Initialization":
# # Special case - Initialization dictionary
# # Serialise it to a subfolder
# path = base_path / name
# export_initialization_dict_to_netcdf(path, member)
# elif isinstance(member, pd.Series):
# xrmember = xr.DataArray(member)
# fname = name + ".nc"
# xrmember.to_netcdf(base_path / fname)

# else:
# warnings.warn(
# f"Output member '{name}' has unsupported type '{type(member)}'. "
# f"It has not been exported to the output directory '{base_path}'."
# )

# # process and save Series
# if series_data:
# try:
# # Combine all Series into a single DataFrame.
# combined_series_df = pd.concat(series_data, axis=1)
# # Convert the combined DataFrame to an xarray Dataset.
# series_dataset = xr.Dataset.from_dataframe(combined_series_df)
# # Save the Series Dataset to a single NetCDF file.
# series_dataset.to_netcdf(base_path / "series.nc")
# except ValueError as e:
# # This handles the "duplicate labels" error if it occurs.
# warnings.warn(
# f"Could not export combined series due to an error: {e}. "
# "Consider cleaning the index of your Series data first."
# )

# if scalar_numbers:
# # Create an xarray Dataset directly from the dictionary of scalars.
# # Each key will become a variable in the NetCDF file.
# scalars_dataset = xr.Dataset(scalar_numbers)
# # Save the scalars Dataset to a NetCDF file.
# scalars_dataset.to_netcdf(base_path / "scalars.nc")
Loading