From 450bcaf9eab19f3e75708a4d09bc826a8d0c4430 Mon Sep 17 00:00:00 2001 From: Jeth Walkup <161084507+jgwalkup@users.noreply.github.com> Date: Wed, 15 Jul 2026 12:39:16 -0400 Subject: [PATCH 1/3] Refactor export functions for initialization data Refactor export functions to handle Union types and improve NetCDF export for various data types. --- src/initialization.py | 150 ++++++++++++++++++++++-------------------- 1 file changed, 80 insertions(+), 70 deletions(-) diff --git a/src/initialization.py b/src/initialization.py index 14feb67..ca5fb2c 100644 --- a/src/initialization.py +++ b/src/initialization.py @@ -174,7 +174,7 @@ def initialize_data(runtime_parameters, site): return Data_Dictionary -def export_initialization_dict_to_csv(base_path: Path | str, d: dict) -> None: +def export_initialization_dict_to_csv(base_path: Union[Path, str], d: dict) -> None: """Export contents of the initialisation directory to a folder. Writes each of the items of a type below to a separate CSV file @@ -202,7 +202,7 @@ def export_initialization_dict_to_csv(base_path: Path | str, d: dict) -> None: elif isinstance(member, np.ndarray): if len(member.shape) <= 2: fname = name + ".csv" - np.savetxt(fname, member, delimiter=",") + np.savetxt(base_path / fname, member, delimiter=",") else: warnings.warn( f"Member '{name}' of initialisation dictionary could not be saved since " @@ -210,6 +210,8 @@ def export_initialization_dict_to_csv(base_path: Path | str, d: dict) -> None: ) elif isinstance(member, numbers.Number): scalar_numbers[name] = member + elif isinstance(member, pd.Index): + pass # Index objects are already captured as DataFrame row labels; no separate export needed else: warnings.warn( f"Initialisation member '{name}' has unsupported type '{type(member)}'. " @@ -219,83 +221,91 @@ def export_initialization_dict_to_csv(base_path: Path | str, d: dict) -> None: # Print numbers pd.Series(scalar_numbers).to_csv(base_path / "scalars.csv") - def export_to_netcdf(self, base_path: Path | str) -> None: - """Export contents of the output file to a directory in NetCDF format. - - Each pandas.DataFrame member is saved to a separate .nc file. - - All pandas.Series members are combined and saved to a single 'series.nc' file. - - All scalar numerical members are grouped and saved to 'scalars.nc'. - - Parameters: - base_path : Path - A path that names the root directory where contents will be exported. - If the directory does not exist it will be created. - """ - # Create space for output - base_path = Path(base_path) - base_path.mkdir(parents=True, exist_ok=True) +def export_initialization_dict_to_netcdf(base_path: Union[Path, str], d: dict) -> None: + """ + Export contents of the initialisation directory to a folder in NetCDF format. + - pandas.DataFrame, pandas.Series, and numpy.ndarray (rank <= 2) are each + written to a separate .nc file. + - All scalar numbers are grouped in a single 'scalars.nc' file. - # Collect all series and scalar data - series_data = dict() - scalar_numbers = dict() + Note: + All other items are ignored following a warning! + """ + # Create space for output + base_path = Path(base_path) + base_path.mkdir(parents=True, exist_ok=True) - for name, member in vars(self).items(): - # convert each DataFrame to an xarray Dataset and save to .nc - if isinstance(member, pd.DataFrame): - # Ensure column names are strings - member.columns = member.columns.astype(str) - if member.index.name is not None: - member.index.name = str(member.index.name) - fname = name + ".nc" # use the .nc extension - try: - xarray_member = xr.Dataset.from_dataframe(member) - xarray_member.to_netcdf(base_path / fname) - except Exception as e: - warnings.warn( - f"Could not export DataFrame '{name}' to NetCDF. Error: {e}" - ) + # Collect all scalar numbers to be processed at the end + scalar_numbers = dict() - elif isinstance(member, pd.Series): - series_data[name] = member + for name, member in d.items(): + fname = name + ".nc" # Use .nc extension for all files - elif isinstance(member, numbers.Number): - scalar_numbers[name] = member + if isinstance(member, pd.DataFrame): + try: + df_to_save = member.copy() + if isinstance(df_to_save.index, pd.MultiIndex): + df_to_save = df_to_save.reset_index() + ds = xr.Dataset.from_dataframe(df_to_save) + encoding = {var: {"zlib": True, "complevel": 4} for var in ds.data_vars} + ds.to_netcdf(base_path / fname, encoding=encoding, format="NETCDF4") + except Exception as e: + warnings.warn(f"Could not convert DataFrame '{name}' to NetCDF. Error: {e}") + + elif isinstance(member, pd.Series): + try: + da = xr.DataArray.from_series(member.reset_index(drop=True)) + da.name = name + da.to_netcdf(base_path / fname, encoding={name: {"zlib": True, "complevel": 4}}, + format="NETCDF4") + except Exception as e: + warnings.warn(f"Could not convert Series '{name}' to NetCDF. Error: {e}") - elif name == "Initialization": - # Special case - Initialization dictionary - # Serialise it to a subfolder - path = base_path / name - export_initialization_dict_to_netcdf(path, member) - elif isinstance(member, pd.Series): - xrmember = xr.DataArray(member) - fname = name + ".nc" - xrmember.to_netcdf(base_path / fname) + elif isinstance(member, pd.Index): + try: + # Use the index's name for the dimension and variable, or a default. + index_name = member.name if member.name is not None else "index_data" + # Convert the Index to a self-describing DataArray and save. + da = xr.DataArray(member, dims=[index_name], name=index_name) + da.to_netcdf(base_path / fname, encoding={index_name: {"zlib": True, "complevel": 4}}, + format="NETCDF4") + except Exception as e: + warnings.warn(f"Could not convert Index '{name}' to NetCDF. Error: {e}") + elif isinstance(member, np.ndarray): + if member.ndim <= 2: + try: + # Convert numpy array to an xarray DataArray. + # We give it a name and default dimension names. + data_array = xr.DataArray( + member, + name=name, + dims=[f"dim_{i}" for i in range(member.ndim)] + ) + data_array.to_netcdf(base_path / fname, encoding={name: {"zlib": True, "complevel": 4}}, + format="NETCDF4") + except Exception as e: + warnings.warn(f"Could not convert array '{name}' to NetCDF. Error: {e}") else: warnings.warn( - f"Output member '{name}' has unsupported type '{type(member)}'. " - f"It has not been exported to the output directory '{base_path}'." + f"Member '{name}' of initialisation dictionary could not be saved since " + f"it is an array of rank higher than 2 (rank: {member.ndim})." ) - # process and save Series - if series_data: - try: - # Combine all Series into a single DataFrame. - combined_series_df = pd.concat(series_data, axis=1) - # Convert the combined DataFrame to an xarray Dataset. - series_dataset = xr.Dataset.from_dataframe(combined_series_df) - # Save the Series Dataset to a single NetCDF file. - series_dataset.to_netcdf(base_path / "series.nc") - except ValueError as e: - # This handles the "duplicate labels" error if it occurs. - warnings.warn( - f"Could not export combined series due to an error: {e}. " - "Consider cleaning the index of your Series data first." - ) + elif isinstance(member, numbers.Number): + scalar_numbers[name] = member + + else: + warnings.warn( + f"Initialisation member '{name}' has unsupported type '{type(member)}'. " + f"It has not been exported to the output directory '{base_path}'." + ) - if scalar_numbers: - # Create an xarray Dataset directly from the dictionary of scalars. - # Each key will become a variable in the NetCDF file. - scalars_dataset = xr.Dataset(scalar_numbers) - # Save the scalars Dataset to a NetCDF file. - scalars_dataset.to_netcdf(base_path / "scalars.nc") + # Process and save all collected scalars + if scalar_numbers: + # Create an xarray Dataset from the dictionary of scalars and save to .nc + scalars_dataset = xr.Dataset(scalar_numbers) + encoding = {var: {"zlib": True, "complevel": 4} for var in scalars_dataset.data_vars} + scalars_dataset.to_netcdf(base_path / "scalars.nc", + encoding=encoding, format="NETCDF4") From 192261c71aaa9b5771d0c34355e0d88a9a9b5304 Mon Sep 17 00:00:00 2001 From: Jeth Walkup <161084507+jgwalkup@users.noreply.github.com> Date: Wed, 15 Jul 2026 14:06:36 -0400 Subject: [PATCH 2/3] Optimize export_to_csv and export_to_netcdf methods Refactor export functions to improve performance and error handling. --- src/output.py | 86 +++++++++++++++++++++++++++++++++++---------------- 1 file changed, 60 insertions(+), 26 deletions(-) diff --git a/src/output.py b/src/output.py index adc96dd..9b706f7 100644 --- a/src/output.py +++ b/src/output.py @@ -303,6 +303,31 @@ def microbes_tradeoff(self, ecosystem, year, day): self.Growth_yield = pd.concat([self.Growth_yield,GY_grid],axis=1,sort=False) + def _assemble_series(self): + """Convert per-day list buffers to DataFrames (called once per export). + + Replaces the incremental pd.concat pattern (O(n²)) with a single + pd.concat per series (O(n)). The initial-state column stored in + self.NAME during __init__ is prepended automatically. + """ + for name, buf in self._buf.items(): + if not buf: + continue + axis = 0 if name == "Kill" else 1 + new_data = pd.concat(buf, axis=axis, sort=False) + existing = getattr(self, name, None) + if existing is not None and not ( + hasattr(existing, "__len__") and len(existing) == 0): + try: + setattr(self, name, pd.concat([existing, new_data], + axis=axis, sort=False)) + except Exception: + setattr(self, name, new_data) + else: + setattr(self, name, new_data) + self._buf[name] = [] + + def export_to_csv(self, base_path: Path | str) -> None: """Export contents of the output file to a directory. @@ -315,6 +340,7 @@ def export_to_csv(self, base_path: Path | str) -> None: A path that names the root directory where contents will be exported. If the directory does not exist it will be created. """ + self._assemble_series() # Create space for output base_path = Path(base_path) base_path.mkdir(parents=True, exist_ok=True) @@ -325,6 +351,8 @@ def export_to_csv(self, base_path: Path | str) -> None: scalar_numbers = dict() for name, member in vars(self).items(): + if name.startswith('_'): + continue # skip private helper attributes (indices, caches) if isinstance(member, pd.DataFrame): fname = name + ".csv" member.to_csv(base_path / fname) @@ -336,21 +364,23 @@ def export_to_csv(self, base_path: Path | str) -> None: # Special case - Initialization dictionary # Serialise it to a subfolder path = base_path / name - export_initialization_dict(path, member) + export_initialization_dict_to_csv(path, member) else: warnings.warn( f"Output member '{name}' has unsupported type '{type(member)}'. " f"It has not been exported to the output directory '{base_path}'." ) - # If it happens that Series have different lengths they will be padded - # with missing data labels (NaNs) - series_data = pd.concat(series_data, axis=1) - series_data.to_csv(base_path / "series.csv") + for name, series in series_data.items(): + try: + series.to_csv(base_path / f"{name}.csv", header=[name]) + except Exception as e: + warnings.warn(f"Could not export series '{name}': {e}") # Print numbers pd.Series(scalar_numbers).to_csv(base_path / "scalars.csv") + def export_to_netcdf(self, base_path: Path | str) -> None: """Export contents of the output file to a directory in NetCDF format. - Each pandas.DataFrame member is saved to a separate .nc file. @@ -362,6 +392,7 @@ def export_to_netcdf(self, base_path: Path | str) -> None: A path that names the root directory where contents will be exported. If the directory does not exist it will be created. """ + self._assemble_series() # Create space for output base_path = Path(base_path) base_path.mkdir(parents=True, exist_ok=True) @@ -371,6 +402,8 @@ def export_to_netcdf(self, base_path: Path | str) -> None: scalar_numbers = dict() for name, member in vars(self).items(): + if name.startswith('_'): + continue # skip private helper attributes (indices, caches) # convert each DataFrame to an xarray Dataset and save to .nc if isinstance(member, pd.DataFrame): # Ensure column names are strings @@ -380,7 +413,8 @@ def export_to_netcdf(self, base_path: Path | str) -> None: fname = name + ".nc" # use the .nc extension try: xarray_member = xr.Dataset.from_dataframe(member) - xarray_member.to_netcdf(base_path / fname) + encoding = {var: {"zlib": True, "complevel": 4} for var in xarray_member.data_vars} + xarray_member.to_netcdf(base_path / fname, encoding=encoding, format="NETCDF4") except Exception as e: warnings.warn( f"Could not export DataFrame '{name}' to NetCDF. Error: {e}" @@ -397,10 +431,6 @@ def export_to_netcdf(self, base_path: Path | str) -> None: # Serialise it to a subfolder path = base_path / name export_initialization_dict_to_netcdf(path, member) - elif isinstance(member, pd.Series): - xrmember = xr.DataArray(member) - fname = name + ".nc" - xrmember.to_netcdf(base_path / fname) else: warnings.warn( @@ -408,25 +438,29 @@ def export_to_netcdf(self, base_path: Path | str) -> None: f"It has not been exported to the output directory '{base_path}'." ) - # process and save Series - if series_data: + # process and save Series — each written to its own .nc file + for name, series in series_data.items(): try: - # Combine all Series into a single DataFrame. - combined_series_df = pd.concat(series_data, axis=1) - # Convert the combined DataFrame to an xarray Dataset. - series_dataset = xr.Dataset.from_dataframe(combined_series_df) - # Save the Series Dataset to a single NetCDF file. - series_dataset.to_netcdf(base_path / "series.nc") - except ValueError as e: - # This handles the "duplicate labels" error if it occurs. - warnings.warn( - f"Could not export combined series due to an error: {e}. " - "Consider cleaning the index of your Series data first." - ) - + if isinstance(series.index, pd.MultiIndex): + da = xr.DataArray.from_series(series) + else: + da = xr.DataArray(series.values, dims=["index"], name=name) + da = da.assign_coords(index=series.index.values) + da.name = name + da.to_netcdf(base_path / f"{name}.nc", + encoding={name: {"zlib": True, "complevel": 4}}, + format="NETCDF4") + except Exception as e: + warnings.warn(f"Could not export Series '{name}' to NetCDF. Error: {e}") + + if scalar_numbers: # Create an xarray Dataset directly from the dictionary of scalars. # Each key will become a variable in the NetCDF file. scalars_dataset = xr.Dataset(scalar_numbers) # Save the scalars Dataset to a NetCDF file. - scalars_dataset.to_netcdf(base_path / "scalars.nc") + encoding = {var: {"zlib": True, "complevel": 4} for var in scalars_dataset.data_vars} + scalars_dataset.to_netcdf(base_path / "scalars.nc", + encoding=encoding, format="NETCDF4") + + From ac3d3b657313b73765cff514e87cd1b2a188b3ff Mon Sep 17 00:00:00 2001 From: Jeth Walkup <161084507+jgwalkup@users.noreply.github.com> Date: Wed, 15 Jul 2026 15:59:07 -0400 Subject: [PATCH 3/3] Remove _assemble_series and netcdf export methods Removed the _assemble_series export_to_netcdf methods to optimize the export process. --- src/output.py | 113 ++------------------------------------------------ 1 file changed, 3 insertions(+), 110 deletions(-) diff --git a/src/output.py b/src/output.py index 9b706f7..1a04050 100644 --- a/src/output.py +++ b/src/output.py @@ -301,32 +301,6 @@ def microbes_tradeoff(self, ecosystem, year, day): GY_grid = ecosystem.Microbe_C_Gain.groupby(level=0,sort=False).sum() GY_grid.name = self.cycle*year + (day+1) self.Growth_yield = pd.concat([self.Growth_yield,GY_grid],axis=1,sort=False) - - - def _assemble_series(self): - """Convert per-day list buffers to DataFrames (called once per export). - - Replaces the incremental pd.concat pattern (O(n²)) with a single - pd.concat per series (O(n)). The initial-state column stored in - self.NAME during __init__ is prepended automatically. - """ - for name, buf in self._buf.items(): - if not buf: - continue - axis = 0 if name == "Kill" else 1 - new_data = pd.concat(buf, axis=axis, sort=False) - existing = getattr(self, name, None) - if existing is not None and not ( - hasattr(existing, "__len__") and len(existing) == 0): - try: - setattr(self, name, pd.concat([existing, new_data], - axis=axis, sort=False)) - except Exception: - setattr(self, name, new_data) - else: - setattr(self, name, new_data) - self._buf[name] = [] - def export_to_csv(self, base_path: Path | str) -> None: """Export contents of the output file to a directory. @@ -340,7 +314,7 @@ def export_to_csv(self, base_path: Path | str) -> None: A path that names the root directory where contents will be exported. If the directory does not exist it will be created. """ - self._assemble_series() + # self._assemble_series() # Create space for output base_path = Path(base_path) base_path.mkdir(parents=True, exist_ok=True) @@ -371,6 +345,8 @@ def export_to_csv(self, base_path: Path | str) -> None: f"It has not been exported to the output directory '{base_path}'." ) + series_data = pd.concat(series_data, axis=1) + for name, series in series_data.items(): try: series.to_csv(base_path / f"{name}.csv", header=[name]) @@ -381,86 +357,3 @@ def export_to_csv(self, base_path: Path | str) -> None: pd.Series(scalar_numbers).to_csv(base_path / "scalars.csv") - def export_to_netcdf(self, base_path: Path | str) -> None: - """Export contents of the output file to a directory in NetCDF format. - - Each pandas.DataFrame member is saved to a separate .nc file. - - All pandas.Series members are combined and saved to a single 'series.nc' file. - - All scalar numerical members are grouped and saved to 'scalars.nc'. - - Parameters: - base_path : Path - A path that names the root directory where contents will be exported. - If the directory does not exist it will be created. - """ - self._assemble_series() - # Create space for output - base_path = Path(base_path) - base_path.mkdir(parents=True, exist_ok=True) - - # Collect all series and scalar data - series_data = dict() - scalar_numbers = dict() - - for name, member in vars(self).items(): - if name.startswith('_'): - continue # skip private helper attributes (indices, caches) - # convert each DataFrame to an xarray Dataset and save to .nc - if isinstance(member, pd.DataFrame): - # Ensure column names are strings - member.columns = member.columns.astype(str) - if member.index.name is not None: - member.index.name = str(member.index.name) - fname = name + ".nc" # use the .nc extension - try: - xarray_member = xr.Dataset.from_dataframe(member) - encoding = {var: {"zlib": True, "complevel": 4} for var in xarray_member.data_vars} - xarray_member.to_netcdf(base_path / fname, encoding=encoding, format="NETCDF4") - except Exception as e: - warnings.warn( - f"Could not export DataFrame '{name}' to NetCDF. Error: {e}" - ) - - elif isinstance(member, pd.Series): - series_data[name] = member - - elif isinstance(member, numbers.Number): - scalar_numbers[name] = member - - elif name == "Initialization": - # Special case - Initialization dictionary - # Serialise it to a subfolder - path = base_path / name - export_initialization_dict_to_netcdf(path, member) - - else: - warnings.warn( - f"Output member '{name}' has unsupported type '{type(member)}'. " - f"It has not been exported to the output directory '{base_path}'." - ) - - # process and save Series — each written to its own .nc file - for name, series in series_data.items(): - try: - if isinstance(series.index, pd.MultiIndex): - da = xr.DataArray.from_series(series) - else: - da = xr.DataArray(series.values, dims=["index"], name=name) - da = da.assign_coords(index=series.index.values) - da.name = name - da.to_netcdf(base_path / f"{name}.nc", - encoding={name: {"zlib": True, "complevel": 4}}, - format="NETCDF4") - except Exception as e: - warnings.warn(f"Could not export Series '{name}' to NetCDF. Error: {e}") - - - if scalar_numbers: - # Create an xarray Dataset directly from the dictionary of scalars. - # Each key will become a variable in the NetCDF file. - scalars_dataset = xr.Dataset(scalar_numbers) - # Save the scalars Dataset to a NetCDF file. - encoding = {var: {"zlib": True, "complevel": 4} for var in scalars_dataset.data_vars} - scalars_dataset.to_netcdf(base_path / "scalars.nc", - encoding=encoding, format="NETCDF4") - -