From 8b5be32a0c2d5f15183607df9294e151d136bd90 Mon Sep 17 00:00:00 2001 From: James Warner Date: Fri, 7 Aug 2026 15:41:11 +0100 Subject: [PATCH 01/28] minor changes --- src/CSET/operators/plot.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/CSET/operators/plot.py b/src/CSET/operators/plot.py index 9460d493f..4b05878a4 100644 --- a/src/CSET/operators/plot.py +++ b/src/CSET/operators/plot.py @@ -747,7 +747,7 @@ def _plot_and_save_spatial_plot( # Add watermark with min/max/mean. Currently not user togglable. # In the bbox dictionary, fc and ec are hex colour codes for grey shade. axes.annotate( - f"Min: {np.min(cube.data):.3g} Max: {np.max(cube.data):.3g} Mean: {np.mean(cube.data):.3g}", + f"Min: {np.nanmin(cube.data):.3g} Max: {np.nanmax(cube.data):.3g} Mean: {np.nanmean(cube.data):.3g}", xy=(0.025, yinfopad), xycoords="axes fraction", xytext=(-5, 5), @@ -1469,7 +1469,7 @@ def _plot_and_save_vector_plot( # Add watermark with min/max/mean. Currently not user togglable. # In the bbox dictionary, fc and ec are hex colour codes for grey shade. axes.annotate( - f"Min: {np.min(cube_vec_mag.data):.3g} Max: {np.max(cube_vec_mag.data):.3g} Mean: {np.mean(cube_vec_mag.data):.3g}", + f"Min: {np.nanmin(cube_vec_mag.data):.3g} Max: {np.nanmax(cube_vec_mag.data):.3g} Mean: {np.nanmean(cube_vec_mag.data):.3g}", xy=(0.05, -0.05), xycoords="axes fraction", xytext=(-5, 5), @@ -1605,7 +1605,10 @@ def _plot_and_save_histogram_series( ax.set_ylabel( f"Contribution to mean ({iter_maybe(cubes)[0].units})", fontsize=14 ) - ax.set_xlim(vmin, vmax) + try: + ax.set_xlim(vmin, vmax) + except ValueError: + pass ax.tick_params(axis="both", labelsize=12) # Overlay grid-lines onto histogram plot. From 7cc12cdb61a3f81e5b07af6bd65c117ba1642aa9 Mon Sep 17 00:00:00 2001 From: James Warner Date: Fri, 7 Aug 2026 15:54:24 +0100 Subject: [PATCH 02/28] initial code lift/lodge --- utils/proc_fastnetuk/proc_fastnetuk.py | 330 +++++++++++++++++++++++++ 1 file changed, 330 insertions(+) create mode 100644 utils/proc_fastnetuk/proc_fastnetuk.py diff --git a/utils/proc_fastnetuk/proc_fastnetuk.py b/utils/proc_fastnetuk/proc_fastnetuk.py new file mode 100644 index 000000000..a02068cff --- /dev/null +++ b/utils/proc_fastnetuk/proc_fastnetuk.py @@ -0,0 +1,330 @@ +"""TODO""" + +import iris +import iris.coord_systems +import iris.coords as icoords +import iris.cube +import numpy as np +from scipy.interpolate import LinearNDInterpolator +from iris.analysis.cartography import rotate_pole + + +# Lookup dictionary to translate to LFRic long_names. +UGRID_VAR_LOOKUP = { + "t": {"long_name": "temperature_at_pressure_levels", "units": "K"}, + "u": {"long_name": "zonal_wind_at_pressure_levels", "units": "m s-1"}, + "v": {"long_name": "meridional_wind_at_pressure_levels", "units": "m s-1"}, + "w": {"long_name": "vertical_wind_at_pressure_levels", "units": "m s-1"}, + "q": { + "long_name": "vapour_specific_humidity_at_pressure_levels_for_climate_averaging", + "units": "kg kg-1", + }, + "z": {"long_name": "geopotential_height_at_pressure_levels", "units": "m"}, + "sp": {"long_name": "surface_air_pressure", "units": "Pa"}, + "10u": {"long_name": "eastward_wind_at_10m", "units": "m s-1"}, + "10v": {"long_name": "northward_wind_at_10m", "units": "m s-1"}, + "lsm": {"long_name": "land_binary_mask", "units": "1"}, + "2t": {"long_name": "temperature_at_screen_level", "units": "K"}, + "2d": {"long_name": "dew_point_temperature_at_screen_level", "units": "K"}, + "skt": {"long_name": "grid_surface_temperature", "units": "K"}, + "tp": {"long_name": "surface_microphysical_rainfall_rate", "units": "mm 6hr-1"}, + "latitude": {"long_name": "latitude", "units": "degrees"}, + "longitude": {"long_name": "longitude", "units": "degrees"}, +} + + +def _rebuild_ugrid_meta_firstfix(cube): + """ + Rebuild iris cube metadata. + + The cube will have metadata within its name and an additional pressure auxiliary + coordinate inferred from the cube name if present. + + Parameters + ---------- + cube : iris.cube.Cube + Original unstructured source cube, used for fixing metadata. + + Returns + ------- + iris.cube.Cube + A structured iris cube with appropriate metadata. + """ + # Get original cube time coordinate dimension. + try: + time_coord = cube.coord("time") + except iris.exceptions.CoordinateNotFoundError: + return None + + # Create new ugrid coordinate placeholder. + # ugrid_coord = icoords.DimCoord(np.arange(cube.shape[1])) + + # Parse cube name, to determine if it contains a likely pressure variable/level. + # If it can't parse this pattern, returns None + m = re.match(r"^([a-zA-Z][a-zA-Z0-9]*|\d+[a-zA-Z]+)(?:_(\d+))?$", cube.name()) + + # Extract variable and pressure from cube name components. + # If it can't find, returns None. + var_key, pressure_hpa = m.group(1), m.group(2) + + # Rename cube using lookup dictionary, if a lookup exists. + meta = UGRID_VAR_LOOKUP.get(var_key) + + if meta is None: + return + else: + # If there is a number in cube name that can be split. + if pressure_hpa is not None: + # Create new pressure coordinate dimension. + pressure_coord = icoords.DimCoord( + [int(pressure_hpa)], + long_name="pressure", + units="hPa", + ) + + # If ndim = 1, a single 2D timeslice with pressure and time. + if cube.ndim == 1: + arr = cube.core_data()[np.newaxis, np.newaxis, :] + else: + arr = cube.core_data()[:, np.newaxis, :] + + out_cube = iris.cube.Cube( + arr, + dim_coords_and_dims=[ + (time_coord, 0), + (pressure_coord, 1), + (icoords.DimCoord(np.arange(arr.shape[-1])), 2), + ], + ) + + else: + # Not a pressure level variable, so only 3 dimensions. + # If ndim = 1, a single 2D timeslice withd time. + if cube.ndim == 1: + arr = cube.core_data()[np.newaxis, :] + else: + arr = cube.core_data() + + out_cube = iris.cube.Cube( + arr, + dim_coords_and_dims=[ + (time_coord, 0), + (icoords.DimCoord(np.arange(arr.shape[-1])), 1), + ], + ) + + # Fix cube metadata + out_cube.long_name = meta["long_name"] + out_cube.units = meta["units"] + out_cube.rename(meta["long_name"]) + + # Add forecast reference time as 'time_origin' to mimic lfric where it will + # reconstruct forecast_period in a later callback. + # Extract the origin string from the units + time_origin = time_coord.units.origin + + # Strip the "seconds since " part. + time_origin = time_origin.split("since ")[1] + + # Add to cube attributes as str. + out_cube.coord("time").attributes["time_origin"] = time_origin + + return out_cube + + + +def _rebuild_ugrid_meta(cube, arr, lat, lon): + """ + Rebuild iris cube metadata. + + The cube will have metadata within its name and an additional pressure auxiliary + coordinate inferred from the cube name if present. + + Parameters + ---------- + cube : iris.cube.Cube + Original unstructured source cube, used for fixing metadata. + arr : np.ndarray + Numpy array of UGRID data. + lat : np.ndarray + 1D latitude coordinate values of regridded data + lon : np.ndarray + 1D longitude coordinate values of regridded data + + Returns + ------- + iris.cube.Cube + A structured iris cube with appropriate metadata. + """ + # Create new latitude coordinate. + lat_coord = icoords.DimCoord( + lat, + standard_name="latitude", + units="degrees", + ) + + # Create new longitude coordinate. + lon_coord = icoords.DimCoord( + lon, + standard_name="longitude", + units="degrees", + ) + + # Get original cube time coordinate dimension. + time_coord = cube.coord("time") + + try: + pressure_coord = cube.coord("pressure") + except iris.exceptions.CoordinateNotFoundError: + pressure_coord = None + + if pressure_coord is not None: + # Create length 1 axis to match shape for pressure + arr = arr[:, np.newaxis, :, :] + + # Create new cube with these dimensions. + out_cube = iris.cube.Cube( + arr, + dim_coords_and_dims=[ + (time_coord, 0), + (pressure_coord, 1), + (lat_coord, 2), + (lon_coord, 3), + ], + ) + + else: + out_cube = iris.cube.Cube( + arr, + dim_coords_and_dims=[ + (time_coord, 0), + (lat_coord, 1), + (lon_coord, 2), + ], + ) + + # Set units/cube name from previous constructed cube. + out_cube.standard_name = cube.standard_name + out_cube.long_name = cube.long_name + out_cube.units = cube.units + + # Copy attributes. + out_cube.attributes = cube.attributes.copy() + + # Change units, geopot in m2 s-2. + if out_cube.long_name == "geopotential_height_at_pressure_levels": + out_cube.data = out_cube.data / 9.81 + + # Raw data in units of 6h accum in meters. + if out_cube.long_name == "surface_microphysical_rainfall_rate": + out_cube.data = out_cube.data * 1000.0 + + return out_cube + + +def _restructure_ugrid_regrid(cube, tri, lat_grid, lon_grid, xy): + """ + Restructure a flattened/unstructured cube. + + Parameters + ---------- + cube : iris.cube + An iris cube to restructure. + tri : scipy.spatial._qhull.Delaunay + A scipy object containing the triangulation mapping of cell points. + lat_grid : np.ndarray + 1D latitude coordinate values of target grid. + lon_grid : np.ndarray + 1D longitude coordinate values of target grid. + xy : np.ndarray + Meshed and flattened target grid points. + + Returns + ------- + iris.cube.Cube + A structured iris cube with appropriate metadata. + + Notes + ----- + This function uses a pre-calculated triangulation, to save rebuilding for + every cube. This therefore assumes all cubes being restructured have the + same flattened structure. + """ + # Create empty numpy array to store regridded data. + out = np.empty((cube.shape[0], lat_grid.size, lon_grid.size)) + + logging.debug(f"Interpolating {cube.name()}") + + # Extract and transpose source data values. + src_vals = cube.data.T + + # Build linear interpolator object mapping target triangulation to source values. + interp = LinearNDInterpolator(tri, src_vals) + + # Interpolate values onto target grid using linear interpolation. + out_flat = interp(xy) + + # Transpose, and reshape to target 2D regular lat/lon grid. + out = out_flat.T.reshape(cube.shape[0], lat_grid.size, lon_grid.size) + + # Rebuild metadata using lookup table (mostly for anemoi ML models). + out_cube = _rebuild_ugrid_meta(cube, out, lat_grid, lon_grid) + + # Return restructured cube with appropriate metadata + return out_cube + + +def restructure_ugrid(cubes, constraint): + """ + Restructure ugrid cubes using parallel processing. + + Parameters + ---------- + cubes : iris.cube.CubeList + A cubelist containing unstructured cubes, along with cubes containing + latitude and longitude information. + + constraint: iris.Constraint + An iris constraint (or combined constraint) to filter cubes on. + + Returns + ------- + fixed_cubes: iris.cube.CubeList + A list of iris cubes, that have been restructured onto a regular grid, + with appropriate corrections to metadata. + """ + # First, parse all cubes and fix their metadata (apart from latitude/longitude, + # which we do later after regridding), and extract required variable from constraint. + cubes = prefilter_fix_metadata(cubes, constraint) + + # First, extract latitude and longitude coordinates + lat = cubes.extract("latitude")[0].data + lon = cubes.extract("longitude")[0].data + points = np.column_stack((lon, lat)) + + # Create output mesh, using standard grid ~2km resolution + # TODO: discussions with ML developers to include metadata so + # we don't have to guess target lat/lon resolution. + # For now, we assume data no higher resolution than 2p2km. + # This will have impacts on PDFs. + lon_grid = np.arange(lon.data.min(), lon.data.max(), 0.02) + lat_grid = np.arange(lat.data.min(), lat.data.max(), 0.02) + Lon2d, Lat2d = np.meshgrid(lon_grid, lat_grid) + + # Flatten target points + xy = np.column_stack((Lon2d.ravel(), Lat2d.ravel())) + + # Build triangulation via a dummy interpolator + tri_interp = LinearNDInterpolator(points, np.zeros(points.shape[0])) + tri = tri_interp.tri + + fixed_cubes = iris.cube.CubeList() + + # For each cube, where ndim > 1 (excluding latitude/longitude array), do regridding + # on array. + for cube in cubes: + if cube.ndim > 1: + result_arr = _restructure_ugrid_regrid(cube, tri, lat_grid, lon_grid, xy) + fixed_cubes.append(result_arr) + + return fixed_cubes.concatenate() \ No newline at end of file From 3cbc07f1e79b94da51776d6e96eb8a615550fde9 Mon Sep 17 00:00:00 2001 From: James Warner Date: Fri, 7 Aug 2026 16:07:56 +0100 Subject: [PATCH 03/28] final code lodge, initial refactor --- utils/proc_fastnetuk/proc_fastnetuk.py | 80 +++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 3 deletions(-) diff --git a/utils/proc_fastnetuk/proc_fastnetuk.py b/utils/proc_fastnetuk/proc_fastnetuk.py index a02068cff..094c5f908 100644 --- a/utils/proc_fastnetuk/proc_fastnetuk.py +++ b/utils/proc_fastnetuk/proc_fastnetuk.py @@ -7,6 +7,8 @@ import numpy as np from scipy.interpolate import LinearNDInterpolator from iris.analysis.cartography import rotate_pole +import argparse +from glob import glob # Lookup dictionary to translate to LFRic long_names. @@ -274,7 +276,51 @@ def _restructure_ugrid_regrid(cube, tri, lat_grid, lon_grid, xy): return out_cube -def restructure_ugrid(cubes, constraint): +def fix_metadata(cubes, constraint): + """ + Pre-filter cubes prior to regridding to reduce excess compute. + + Parse cubes and filter for required variable, alongside latitude and + longitude, for further processing. This reduces compute overhead on + variables that we don't require. This also cleans metadata prior to filtering. + + Parameters + ---------- + cubes : iris.cube.CubeList + A cubelist containing unstructured cubes, along with cubes containing + latitude and longitude information. + + constraint : iris.constraint + Constraint in order to extract required variable. + + Returns + ------- + filterd_cubes : iris.cube.CubeList + A cubelist containing the required cube that matches the constraint, along + with latitude and longitude cubes. + """ + # Add metadata to variables, if appropriate + sanitised_cubes = iris.cube.CubeList() + for cube in cubes: + out = _rebuild_ugrid_meta_firstfix(cube) + if out is not None: + sanitised_cubes.append(out) + + # Create empty cubelist. + filtered_cubes = iris.cube.CubeList() + + # Extract latitude and longitude cubes, and append these to filtered_cubes. + filtered_cubes.append(cubes.extract("latitude")[0]) + filtered_cubes.append(cubes.extract("longitude")[0]) + + # Extract required cube based on constraint. + for c in sanitised_cubes.extract(constraint): + filtered_cubes.append(c) + + return filtered_cubes + + +def restructure_ugrid(cubes): """ Restructure ugrid cubes using parallel processing. @@ -295,7 +341,7 @@ def restructure_ugrid(cubes, constraint): """ # First, parse all cubes and fix their metadata (apart from latitude/longitude, # which we do later after regridding), and extract required variable from constraint. - cubes = prefilter_fix_metadata(cubes, constraint) + cubes = fix_metadata(cubes, constraint) # First, extract latitude and longitude coordinates lat = cubes.extract("latitude")[0].data @@ -327,4 +373,32 @@ def restructure_ugrid(cubes, constraint): result_arr = _restructure_ugrid_regrid(cube, tri, lat_grid, lon_grid, xy) fixed_cubes.append(result_arr) - return fixed_cubes.concatenate() \ No newline at end of file + return fixed_cubes.concatenate() + + +def main() -> None: + """ + Run processing on FastNetUK data. + + Process produces CSET-ready netCDF files for loading. + """ + parser = argparse.ArgumentParser(description="Process arguments.") + parser.add_argument("--inputpath", required=True, help="Path to file(s) to load.") + parser.add_argument( + "--outputpath", + type=str, + required=True, + help="Path to save final output data.", + ) + + args = parser.parse_args() + + inputpath = args.inputpath + outputpath = args.outpath + "/" + + for file in glob(inputpath): + print(f"Running script on {file}") + + # Func1: Load all cubes, get lat, lon + cubes = iris.load(file) + cubes = restructure_ugrid(cubes) From 262b81351817792cfa0ff1784be410e4463452a2 Mon Sep 17 00:00:00 2001 From: James Warner Date: Fri, 7 Aug 2026 16:10:13 +0100 Subject: [PATCH 04/28] tidy --- utils/proc_fastnetuk/proc_fastnetuk.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/proc_fastnetuk/proc_fastnetuk.py b/utils/proc_fastnetuk/proc_fastnetuk.py index 094c5f908..8c39075a5 100644 --- a/utils/proc_fastnetuk/proc_fastnetuk.py +++ b/utils/proc_fastnetuk/proc_fastnetuk.py @@ -341,7 +341,7 @@ def restructure_ugrid(cubes): """ # First, parse all cubes and fix their metadata (apart from latitude/longitude, # which we do later after regridding), and extract required variable from constraint. - cubes = fix_metadata(cubes, constraint) + cubes = fix_metadata(cubes) # First, extract latitude and longitude coordinates lat = cubes.extract("latitude")[0].data @@ -351,6 +351,7 @@ def restructure_ugrid(cubes): # Create output mesh, using standard grid ~2km resolution # TODO: discussions with ML developers to include metadata so # we don't have to guess target lat/lon resolution. + # Need some attributes to capture what is required in terms of resolution. # For now, we assume data no higher resolution than 2p2km. # This will have impacts on PDFs. lon_grid = np.arange(lon.data.min(), lon.data.max(), 0.02) From 01ba6ae12e6299f08ed5a8582949a07271b40811 Mon Sep 17 00:00:00 2001 From: James Warner Date: Thu, 13 Aug 2026 11:01:14 +0100 Subject: [PATCH 05/28] small fixes for end to end working example --- utils/proc_fastnetuk/proc_fastnetuk.py | 37 +++++++++++++------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/utils/proc_fastnetuk/proc_fastnetuk.py b/utils/proc_fastnetuk/proc_fastnetuk.py index 8c39075a5..7a8772a75 100644 --- a/utils/proc_fastnetuk/proc_fastnetuk.py +++ b/utils/proc_fastnetuk/proc_fastnetuk.py @@ -9,7 +9,7 @@ from iris.analysis.cartography import rotate_pole import argparse from glob import glob - +import re # Lookup dictionary to translate to LFRic long_names. UGRID_VAR_LOOKUP = { @@ -73,7 +73,7 @@ def _rebuild_ugrid_meta_firstfix(cube): meta = UGRID_VAR_LOOKUP.get(var_key) if meta is None: - return + return None else: # If there is a number in cube name that can be split. if pressure_hpa is not None: @@ -255,7 +255,7 @@ def _restructure_ugrid_regrid(cube, tri, lat_grid, lon_grid, xy): # Create empty numpy array to store regridded data. out = np.empty((cube.shape[0], lat_grid.size, lon_grid.size)) - logging.debug(f"Interpolating {cube.name()}") + print(f"Interpolating: {cube}") # Extract and transpose source data values. src_vals = cube.data.T @@ -276,7 +276,7 @@ def _restructure_ugrid_regrid(cube, tri, lat_grid, lon_grid, xy): return out_cube -def fix_metadata(cubes, constraint): +def fix_metadata(cubes): """ Pre-filter cubes prior to regridding to reduce excess compute. @@ -290,9 +290,6 @@ def fix_metadata(cubes, constraint): A cubelist containing unstructured cubes, along with cubes containing latitude and longitude information. - constraint : iris.constraint - Constraint in order to extract required variable. - Returns ------- filterd_cubes : iris.cube.CubeList @@ -306,18 +303,11 @@ def fix_metadata(cubes, constraint): if out is not None: sanitised_cubes.append(out) - # Create empty cubelist. - filtered_cubes = iris.cube.CubeList() - # Extract latitude and longitude cubes, and append these to filtered_cubes. - filtered_cubes.append(cubes.extract("latitude")[0]) - filtered_cubes.append(cubes.extract("longitude")[0]) + sanitised_cubes.append(cubes.extract("latitude")[0]) + sanitised_cubes.append(cubes.extract("longitude")[0]) - # Extract required cube based on constraint. - for c in sanitised_cubes.extract(constraint): - filtered_cubes.append(c) - - return filtered_cubes + return sanitised_cubes def restructure_ugrid(cubes): @@ -340,7 +330,7 @@ def restructure_ugrid(cubes): with appropriate corrections to metadata. """ # First, parse all cubes and fix their metadata (apart from latitude/longitude, - # which we do later after regridding), and extract required variable from constraint. + # which we do later after regridding). cubes = fix_metadata(cubes) # First, extract latitude and longitude coordinates @@ -394,8 +384,9 @@ def main() -> None: args = parser.parse_args() + # Get file paths inputpath = args.inputpath - outputpath = args.outpath + "/" + outputpath = args.outputpath + "/" for file in glob(inputpath): print(f"Running script on {file}") @@ -403,3 +394,11 @@ def main() -> None: # Func1: Load all cubes, get lat, lon cubes = iris.load(file) cubes = restructure_ugrid(cubes) + + print(f"Saving restructured cubes") + iris.save(cubes, f"{outputpath}/fixed_{file.split("/")[-1]}") + print(f"Done file {file}") + + +if __name__ == '__main__': + main() \ No newline at end of file From 6703a99ac4d14325d62ab699a40944abd5e3ad46 Mon Sep 17 00:00:00 2001 From: James Warner Date: Thu, 13 Aug 2026 11:01:49 +0100 Subject: [PATCH 06/28] small fixes for end to end working example --- utils/proc_fastnetuk/proc_fastnetuk.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/utils/proc_fastnetuk/proc_fastnetuk.py b/utils/proc_fastnetuk/proc_fastnetuk.py index 7a8772a75..3a2d403d3 100644 --- a/utils/proc_fastnetuk/proc_fastnetuk.py +++ b/utils/proc_fastnetuk/proc_fastnetuk.py @@ -1,15 +1,15 @@ """TODO""" +import argparse +import re +from glob import glob + import iris import iris.coord_systems import iris.coords as icoords import iris.cube import numpy as np from scipy.interpolate import LinearNDInterpolator -from iris.analysis.cartography import rotate_pole -import argparse -from glob import glob -import re # Lookup dictionary to translate to LFRic long_names. UGRID_VAR_LOOKUP = { @@ -134,7 +134,6 @@ def _rebuild_ugrid_meta_firstfix(cube): return out_cube - def _rebuild_ugrid_meta(cube, arr, lat, lon): """ Rebuild iris cube metadata. @@ -395,10 +394,10 @@ def main() -> None: cubes = iris.load(file) cubes = restructure_ugrid(cubes) - print(f"Saving restructured cubes") - iris.save(cubes, f"{outputpath}/fixed_{file.split("/")[-1]}") + print("Saving restructured cubes") + iris.save(cubes, f"{outputpath}/fixed_{file.split('/')[-1]}") print(f"Done file {file}") -if __name__ == '__main__': - main() \ No newline at end of file +if __name__ == "__main__": + main() From 1f28939a97d5e8c94e71920ed1d1eefa0a65375f Mon Sep 17 00:00:00 2001 From: James Warner Date: Thu, 13 Aug 2026 11:43:23 +0100 Subject: [PATCH 07/28] further clean-ups --- utils/proc_fastnetuk/proc_fastnetuk.py | 29 +++++++++++++------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/utils/proc_fastnetuk/proc_fastnetuk.py b/utils/proc_fastnetuk/proc_fastnetuk.py index 3a2d403d3..ce8870ffc 100644 --- a/utils/proc_fastnetuk/proc_fastnetuk.py +++ b/utils/proc_fastnetuk/proc_fastnetuk.py @@ -277,21 +277,17 @@ def _restructure_ugrid_regrid(cube, tri, lat_grid, lon_grid, xy): def fix_metadata(cubes): """ - Pre-filter cubes prior to regridding to reduce excess compute. - - Parse cubes and filter for required variable, alongside latitude and - longitude, for further processing. This reduces compute overhead on - variables that we don't require. This also cleans metadata prior to filtering. + Pre-filter cubes prior to regridding. Parameters ---------- - cubes : iris.cube.CubeList + cubes: iris.cube.CubeList A cubelist containing unstructured cubes, along with cubes containing latitude and longitude information. Returns ------- - filterd_cubes : iris.cube.CubeList + sanitised_cubes: iris.cube.CubeList A cubelist containing the required cube that matches the constraint, along with latitude and longitude cubes. """ @@ -311,7 +307,10 @@ def fix_metadata(cubes): def restructure_ugrid(cubes): """ - Restructure ugrid cubes using parallel processing. + Restructure ugrid cubes. + + First, fixes cube metadata names as a first fix, and then regrids, and then + finally adds metadata associated with new coordinates. Parameters ---------- @@ -319,14 +318,17 @@ def restructure_ugrid(cubes): A cubelist containing unstructured cubes, along with cubes containing latitude and longitude information. - constraint: iris.Constraint - An iris constraint (or combined constraint) to filter cubes on. - Returns ------- fixed_cubes: iris.cube.CubeList A list of iris cubes, that have been restructured onto a regular grid, with appropriate corrections to metadata. + + Notes + ----- + Currently, data is regridded to a 0.02 rectilinear grid. This is because + there is no metada in the source file that describes the target resolution + of what it should be unpacked to. """ # First, parse all cubes and fix their metadata (apart from latitude/longitude, # which we do later after regridding). @@ -340,9 +342,6 @@ def restructure_ugrid(cubes): # Create output mesh, using standard grid ~2km resolution # TODO: discussions with ML developers to include metadata so # we don't have to guess target lat/lon resolution. - # Need some attributes to capture what is required in terms of resolution. - # For now, we assume data no higher resolution than 2p2km. - # This will have impacts on PDFs. lon_grid = np.arange(lon.data.min(), lon.data.max(), 0.02) lat_grid = np.arange(lat.data.min(), lat.data.max(), 0.02) Lon2d, Lat2d = np.meshgrid(lon_grid, lat_grid) @@ -390,7 +389,7 @@ def main() -> None: for file in glob(inputpath): print(f"Running script on {file}") - # Func1: Load all cubes, get lat, lon + # Load data and restructure. cubes = iris.load(file) cubes = restructure_ugrid(cubes) From 2320f0a29aee3a781344964a330287be10cbcc48 Mon Sep 17 00:00:00 2001 From: James Warner Date: Thu, 13 Aug 2026 13:52:41 +0100 Subject: [PATCH 08/28] add comments and tidyup --- utils/proc_fastnetuk/proc_fastnetuk.py | 274 ++++++++----------------- 1 file changed, 90 insertions(+), 184 deletions(-) diff --git a/utils/proc_fastnetuk/proc_fastnetuk.py b/utils/proc_fastnetuk/proc_fastnetuk.py index ce8870ffc..f54c47a88 100644 --- a/utils/proc_fastnetuk/proc_fastnetuk.py +++ b/utils/proc_fastnetuk/proc_fastnetuk.py @@ -1,4 +1,7 @@ -"""TODO""" +"""Fix FastNetUK inference data on UGRID with limited metadata. + +For more information on this script and how to use it, see the README.md +""" import argparse import re @@ -35,7 +38,7 @@ } -def _rebuild_ugrid_meta_firstfix(cube): +def rebuild_metadata(cube, arr, lat, lon): """ Rebuild iris cube metadata. @@ -46,190 +49,130 @@ def _rebuild_ugrid_meta_firstfix(cube): ---------- cube : iris.cube.Cube Original unstructured source cube, used for fixing metadata. + arr : np.ndarray + Numpy array of restructured (2D) data. + lat : np.ndarray + 1D latitude coordinate values of regridded data + lon : np.ndarray + 1D longitude coordinate values of regridded data Returns ------- iris.cube.Cube A structured iris cube with appropriate metadata. """ - # Get original cube time coordinate dimension. - try: - time_coord = cube.coord("time") - except iris.exceptions.CoordinateNotFoundError: - return None - - # Create new ugrid coordinate placeholder. - # ugrid_coord = icoords.DimCoord(np.arange(cube.shape[1])) + # Determine if cube matches certain string pattern. + match = re.match( + r"^([a-zA-Z][a-zA-Z0-9]*|\d+[a-zA-Z]+)(?:_(\d+))?$", + cube.name(), + ) - # Parse cube name, to determine if it contains a likely pressure variable/level. - # If it can't parse this pattern, returns None - m = re.match(r"^([a-zA-Z][a-zA-Z0-9]*|\d+[a-zA-Z]+)(?:_(\d+))?$", cube.name()) + if match is None: + return None - # Extract variable and pressure from cube name components. - # If it can't find, returns None. - var_key, pressure_hpa = m.group(1), m.group(2) + # Extract var and pressure, if present. + var_key, pressure_hpa = match.groups() - # Rename cube using lookup dictionary, if a lookup exists. + # See if there is an entry for the variable, if not return. meta = UGRID_VAR_LOOKUP.get(var_key) if meta is None: return None - else: - # If there is a number in cube name that can be split. - if pressure_hpa is not None: - # Create new pressure coordinate dimension. - pressure_coord = icoords.DimCoord( - [int(pressure_hpa)], - long_name="pressure", - units="hPa", - ) - - # If ndim = 1, a single 2D timeslice with pressure and time. - if cube.ndim == 1: - arr = cube.core_data()[np.newaxis, np.newaxis, :] - else: - arr = cube.core_data()[:, np.newaxis, :] - - out_cube = iris.cube.Cube( - arr, - dim_coords_and_dims=[ - (time_coord, 0), - (pressure_coord, 1), - (icoords.DimCoord(np.arange(arr.shape[-1])), 2), - ], - ) - - else: - # Not a pressure level variable, so only 3 dimensions. - # If ndim = 1, a single 2D timeslice withd time. - if cube.ndim == 1: - arr = cube.core_data()[np.newaxis, :] - else: - arr = cube.core_data() - - out_cube = iris.cube.Cube( - arr, - dim_coords_and_dims=[ - (time_coord, 0), - (icoords.DimCoord(np.arange(arr.shape[-1])), 1), - ], - ) - - # Fix cube metadata - out_cube.long_name = meta["long_name"] - out_cube.units = meta["units"] - out_cube.rename(meta["long_name"]) - - # Add forecast reference time as 'time_origin' to mimic lfric where it will - # reconstruct forecast_period in a later callback. - # Extract the origin string from the units - time_origin = time_coord.units.origin - - # Strip the "seconds since " part. - time_origin = time_origin.split("since ")[1] - - # Add to cube attributes as str. - out_cube.coord("time").attributes["time_origin"] = time_origin - - return out_cube - -def _rebuild_ugrid_meta(cube, arr, lat, lon): - """ - Rebuild iris cube metadata. - - The cube will have metadata within its name and an additional pressure auxiliary - coordinate inferred from the cube name if present. - - Parameters - ---------- - cube : iris.cube.Cube - Original unstructured source cube, used for fixing metadata. - arr : np.ndarray - Numpy array of UGRID data. - lat : np.ndarray - 1D latitude coordinate values of regridded data - lon : np.ndarray - 1D longitude coordinate values of regridded data - - Returns - ------- - iris.cube.Cube - A structured iris cube with appropriate metadata. - """ - # Create new latitude coordinate. + # Create latitude, longitude coordinate objects. lat_coord = icoords.DimCoord( lat, standard_name="latitude", units="degrees", ) - # Create new longitude coordinate. lon_coord = icoords.DimCoord( lon, standard_name="longitude", units="degrees", ) - # Get original cube time coordinate dimension. - time_coord = cube.coord("time") + # Create time dimensions, including forecast_period and forecast_reference_time. + time_coord = cube.coord("time").copy() + + forecast_reference_time = icoords.AuxCoord( + time_coord.points[0], + standard_name="forecast_reference_time", + units=time_coord.units, + ) + + forecast_period = icoords.DimCoord( + (time_coord.points - time_coord.points[0]) / 3600, + standard_name="forecast_period", + units="hours", + ) - try: - pressure_coord = cube.coord("pressure") - except iris.exceptions.CoordinateNotFoundError: - pressure_coord = None + # Start with coordinates of just forecast_period. + coords = [ + (forecast_period, 0), + ] + + # If pressure exists, create additional size 1 dimension for future concatenation. + if pressure_hpa is not None: + pressure_coord = icoords.DimCoord( + [int(pressure_hpa)], + long_name="pressure", + units="hPa", + ) - if pressure_coord is not None: - # Create length 1 axis to match shape for pressure arr = arr[:, np.newaxis, :, :] - # Create new cube with these dimensions. - out_cube = iris.cube.Cube( - arr, - dim_coords_and_dims=[ - (time_coord, 0), + coords.extend( + [ (pressure_coord, 1), (lat_coord, 2), (lon_coord, 3), - ], + ] ) + # If pressure doesn't exist, just use latitude/longitude in addition to time. else: - out_cube = iris.cube.Cube( - arr, - dim_coords_and_dims=[ - (time_coord, 0), + coords.extend( + [ (lat_coord, 1), (lon_coord, 2), - ], + ] ) - # Set units/cube name from previous constructed cube. - out_cube.standard_name = cube.standard_name - out_cube.long_name = cube.long_name - out_cube.units = cube.units + # Create cube with coordinates + out_cube = iris.cube.Cube( + arr, + dim_coords_and_dims=coords, + ) + + # Add scalar time coordinates + out_cube.add_aux_coord(forecast_reference_time) + out_cube.add_aux_coord(time_coord, data_dims=(0,)) + + # Add metadata for long name, units, and preserve other attributes. + out_cube.rename(meta["long_name"]) + out_cube.long_name = meta["long_name"] + out_cube.units = meta["units"] - # Copy attributes. out_cube.attributes = cube.attributes.copy() - # Change units, geopot in m2 s-2. + # Some unit corrections for specific variables. if out_cube.long_name == "geopotential_height_at_pressure_levels": - out_cube.data = out_cube.data / 9.81 + out_cube.data /= 9.81 - # Raw data in units of 6h accum in meters. - if out_cube.long_name == "surface_microphysical_rainfall_rate": - out_cube.data = out_cube.data * 1000.0 + elif out_cube.long_name == "surface_microphysical_rainfall_rate": + out_cube.data *= 1000.0 return out_cube -def _restructure_ugrid_regrid(cube, tri, lat_grid, lon_grid, xy): +def ugrid_transform(arr, tri, lat_grid, lon_grid, xy): """ Restructure a flattened/unstructured cube. Parameters ---------- - cube : iris.cube + arr : arrayy An iris cube to restructure. tri : scipy.spatial._qhull.Delaunay A scipy object containing the triangulation mapping of cell points. @@ -252,12 +195,10 @@ def _restructure_ugrid_regrid(cube, tri, lat_grid, lon_grid, xy): same flattened structure. """ # Create empty numpy array to store regridded data. - out = np.empty((cube.shape[0], lat_grid.size, lon_grid.size)) - - print(f"Interpolating: {cube}") + out = np.empty((arr.shape[0], lat_grid.size, lon_grid.size)) # Extract and transpose source data values. - src_vals = cube.data.T + src_vals = arr.T # Build linear interpolator object mapping target triangulation to source values. interp = LinearNDInterpolator(tri, src_vals) @@ -266,48 +207,14 @@ def _restructure_ugrid_regrid(cube, tri, lat_grid, lon_grid, xy): out_flat = interp(xy) # Transpose, and reshape to target 2D regular lat/lon grid. - out = out_flat.T.reshape(cube.shape[0], lat_grid.size, lon_grid.size) - - # Rebuild metadata using lookup table (mostly for anemoi ML models). - out_cube = _rebuild_ugrid_meta(cube, out, lat_grid, lon_grid) + out = out_flat.T.reshape(arr.shape[0], lat_grid.size, lon_grid.size) - # Return restructured cube with appropriate metadata - return out_cube + return out -def fix_metadata(cubes): +def fix_cubes(cubes): """ - Pre-filter cubes prior to regridding. - - Parameters - ---------- - cubes: iris.cube.CubeList - A cubelist containing unstructured cubes, along with cubes containing - latitude and longitude information. - - Returns - ------- - sanitised_cubes: iris.cube.CubeList - A cubelist containing the required cube that matches the constraint, along - with latitude and longitude cubes. - """ - # Add metadata to variables, if appropriate - sanitised_cubes = iris.cube.CubeList() - for cube in cubes: - out = _rebuild_ugrid_meta_firstfix(cube) - if out is not None: - sanitised_cubes.append(out) - - # Extract latitude and longitude cubes, and append these to filtered_cubes. - sanitised_cubes.append(cubes.extract("latitude")[0]) - sanitised_cubes.append(cubes.extract("longitude")[0]) - - return sanitised_cubes - - -def restructure_ugrid(cubes): - """ - Restructure ugrid cubes. + Restructure ugrid cubes and then fix metadata. First, fixes cube metadata names as a first fix, and then regrids, and then finally adds metadata associated with new coordinates. @@ -326,14 +233,10 @@ def restructure_ugrid(cubes): Notes ----- - Currently, data is regridded to a 0.02 rectilinear grid. This is because + Currently, data is regridded to a 0.02degree rectilinear grid. This is because there is no metada in the source file that describes the target resolution - of what it should be unpacked to. + of what it should be regridded to. """ - # First, parse all cubes and fix their metadata (apart from latitude/longitude, - # which we do later after regridding). - cubes = fix_metadata(cubes) - # First, extract latitude and longitude coordinates lat = cubes.extract("latitude")[0].data lon = cubes.extract("longitude")[0].data @@ -356,11 +259,14 @@ def restructure_ugrid(cubes): fixed_cubes = iris.cube.CubeList() # For each cube, where ndim > 1 (excluding latitude/longitude array), do regridding - # on array. + # on array, and correct metadata. for cube in cubes: if cube.ndim > 1: - result_arr = _restructure_ugrid_regrid(cube, tri, lat_grid, lon_grid, xy) - fixed_cubes.append(result_arr) + print(f"Fixing {cube.name()}") + result_arr = ugrid_transform(cube.data, tri, lat_grid, lon_grid, xy) + cube = rebuild_metadata(cube, result_arr, lat_grid, lon_grid) + if cube: + fixed_cubes.append(cube) return fixed_cubes.concatenate() @@ -391,7 +297,7 @@ def main() -> None: # Load data and restructure. cubes = iris.load(file) - cubes = restructure_ugrid(cubes) + cubes = fix_cubes(cubes) print("Saving restructured cubes") iris.save(cubes, f"{outputpath}/fixed_{file.split('/')[-1]}") From 1dffed07f842b405a303ee30e117455b4feec958 Mon Sep 17 00:00:00 2001 From: James Warner Date: Thu, 13 Aug 2026 14:05:22 +0100 Subject: [PATCH 09/28] add README --- utils/proc_fastnetuk/README.md | 223 +++++++++++++++++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 utils/proc_fastnetuk/README.md diff --git a/utils/proc_fastnetuk/README.md b/utils/proc_fastnetuk/README.md new file mode 100644 index 000000000..e74ae7dde --- /dev/null +++ b/utils/proc_fastnetuk/README.md @@ -0,0 +1,223 @@ +# fix_fastnetuk_ugrid + +## About + +The script `fix_fastnetuk_ugrid.py` is a utility for converting FastNetUK inference output stored on an unstructured UGRID mesh into CSET-compatible NetCDF files. + +The primary motivation for this tool is to allow FastNetUK machine learning forecast output to be ingested into CSET alongside other forecast systems. The source files contain limited metadata, use an unstructured grid representation, and do not follow the naming conventions expected by CSET. Note that the data is actually flattened, not truly unstructured, but without explaining metadata we have to assume it is a genuine unstructured dataset. + +This script performs several preprocessing steps to make the data suitable for verification and evaluation within CSET: + +- Regrids unstructured UGRID data onto a regular latitude-longitude grid. +- Converts variable names to CSET/LFRic naming conventions. +- Restores appropriate units for meteorological variables. +- Creates a `forecast_period` dimension coordinate. +- Creates a scalar `forecast_reference_time` coordinate. +- Preserves valid times as a `time` auxiliary coordinate. +- Reconstructs pressure-level metadata where present. +- Saves corrected data as CSET-ready NetCDF files. + +The script only performs metadata reconstruction and interpolation onto a structured grid. No scientific modifications are applied to the meteorological fields apart from unit conversions required. + +> [!TIP] +> The script assumes the source file contains latitude and longitude variables describing the UGRID cell locations. These are used to reconstruct a regular latitude-longitude grid. + +> [!TIP] +> On standard data from FastNetUK inference, this script requires 30GB memory to run. + +## Usage + +The script requires the following software to be installed: + +- Python +- Iris +- NumPy +- SciPy + +Run it with: + +```bash +python fix_fastnetuk_ugrid.py \ + --inputpath "" \ + --outputpath "" +``` + +### Required Arguments + +- `--inputpath`: Path to one or more FastNetUK inference files. Wildcards may be used. If wildcards are used, quote the path so the shell passes the pattern to Python unchanged. +- `--outputpath`: Directory where fixed NetCDF files will be written. + +## Processing Details + +### UGRID Restructuring + +FastNetUK inference output is stored on an unstructured mesh with latitude and longitude supplied as separate variables. + +The script: + +1. Extracts latitude and longitude point locations. +2. Builds a triangulation of the source mesh. +3. Creates a regular latitude-longitude target grid. +4. Interpolates each meteorological field onto the regular grid using linear interpolation. + +Currently a fixed grid spacing of: + +```text +0.02° +``` + +is used for the target grid. + +> [!NOTE] +> The target grid resolution is currently inferred because the source files do not contain metadata describing the intended structured output resolution. + +### Metadata Reconstruction + +The source files contain limited metadata, with most information encoded within variable names. + +Examples include: + +```text +t_850 +u_500 +v_250 +2t +10u +sp +``` + +The script extracts: + +- Variable type +- Pressure level (where present) + +and reconstructs metadata required by CSET. + +### Forecast Coordinates + +The script reconstructs forecast metadata using the source time coordinate. + +It creates: + +- `forecast_period` +- `forecast_reference_time` + +while preserving valid times as: + +- `time` + +The first time value in the source file is assumed to represent forecast lead time zero. + +### Pressure Levels + +Variables containing pressure-level information in their names are assigned a pressure dimension coordinate. + +For example: + +```text +t_850 +``` + +becomes: + +```text +temperature_at_pressure_levels +pressure = 850 hPa +``` + +A length-one pressure dimension is created to allow future concatenation of multiple pressure levels. + +### Variable Renaming + +Variables are translated to CSET/LFRic naming conventions using an internal lookup table. The original names originate +from anemoi [here](https://anemoi.readthedocs.io/projects/inference/en/latest/inference/configs/outputs.html). + +Examples include: + +| Source Name | Output Name | +|-------------|-------------| +| t | temperature_at_pressure_levels | +| u | zonal_wind_at_pressure_levels | +| v | meridional_wind_at_pressure_levels | +| w | vertical_wind_at_pressure_levels | +| q | vapour_specific_humidity_at_pressure_levels_for_climate_averaging | +| z | geopotential_height_at_pressure_levels | +| sp | surface_air_pressure | +| 10u | eastward_wind_at_10m | +| 10v | northward_wind_at_10m | +| 2t | temperature_at_screen_level | +| 2d | dew_point_temperature_at_screen_level | +| skt | grid_surface_temperature | +| tp | surface_microphysical_rainfall_rate | + +### Unit Conversion + +Some variables require unit adjustments before output. + +Examples include: + +- Geopotential (`m² s⁻²`) → Geopotential height (`m`) +- Accumulated precipitation (`m`) → Rainfall amount (`mm 6hr⁻¹`) + +These conversions are applied automatically where required. + +## Examples + +### 1. Process a single FastNetUK file + +```bash +python fix_fastnetuk_ugrid.py \ + --inputpath "/data/fastnetuk/inference.nc" \ + --outputpath "/my/output/path" +``` + +Example output: + +```text +/my/output/path/fixed_inference.nc +``` + +### 2. Process multiple files + +```bash +python fix_fastnetuk_ugrid.py \ + --inputpath "/data/fastnetuk/*.nc" \ + --outputpath "/my/output/path" +``` + +All matching files will be processed and written to the output directory. + +## Output Structure + +The resulting files contain: + +- Structured latitude-longitude grids +- CSET-compliant variable names +- Reconstructed units +- Forecast metadata +- Forecast period dimension +- Forecast reference time coordinate +- Valid time coordinate + +Pressure-level variables will additionally contain: + +```text +pressure +``` + +as a dimension coordinate. + +## Notes + +- Latitude and longitude variables must exist within the source file. +- Variables with names not present in the internal lookup table will be ignored. +- The script currently assumes a target grid spacing of approximately 0.02°. +- Linear interpolation is used to transform data from the unstructured mesh to a rectilinear grid. +- The first time value in the source file is assumed to correspond to forecast lead time zero. +- Pressure-level variables are inferred solely from the variable name. + +## Owners + +The following people should be contacted for queries or issues with this utility: + +[jwarner8](https://github.com/jwarner8) From 2274347c100231b5def0651c92a7a613e48e76d3 Mon Sep 17 00:00:00 2001 From: James Warner Date: Thu, 13 Aug 2026 16:27:06 +0100 Subject: [PATCH 10/28] fix coords not merging appropriately --- utils/proc_fastnetuk/proc_fastnetuk.py | 49 ++++++++++++++++++-------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/utils/proc_fastnetuk/proc_fastnetuk.py b/utils/proc_fastnetuk/proc_fastnetuk.py index f54c47a88..2027dc85d 100644 --- a/utils/proc_fastnetuk/proc_fastnetuk.py +++ b/utils/proc_fastnetuk/proc_fastnetuk.py @@ -12,6 +12,7 @@ import iris.coords as icoords import iris.cube import numpy as np +from cf_units import Unit from scipy.interpolate import LinearNDInterpolator # Lookup dictionary to translate to LFRic long_names. @@ -93,12 +94,17 @@ def rebuild_metadata(cube, arr, lat, lon): ) # Create time dimensions, including forecast_period and forecast_reference_time. - time_coord = cube.coord("time").copy() + time_coord = cube.coord("time") - forecast_reference_time = icoords.AuxCoord( - time_coord.points[0], + base_time_units = Unit("hours since 1970-01-01 00:00:00") + frt_point = base_time_units.date2num( + time_coord.units.num2date(time_coord.points[0]) + ) + + forecast_reference_time = icoords.DimCoord( + [frt_point], standard_name="forecast_reference_time", - units=time_coord.units, + units=base_time_units, ) forecast_period = icoords.DimCoord( @@ -109,7 +115,8 @@ def rebuild_metadata(cube, arr, lat, lon): # Start with coordinates of just forecast_period. coords = [ - (forecast_period, 0), + (forecast_reference_time, 0), + (forecast_period, 1), ] # If pressure exists, create additional size 1 dimension for future concatenation. @@ -120,22 +127,23 @@ def rebuild_metadata(cube, arr, lat, lon): units="hPa", ) - arr = arr[:, np.newaxis, :, :] + arr = arr[np.newaxis, :, np.newaxis, :, :] coords.extend( [ - (pressure_coord, 1), - (lat_coord, 2), - (lon_coord, 3), + (pressure_coord, 2), + (lat_coord, 3), + (lon_coord, 4), ] ) # If pressure doesn't exist, just use latitude/longitude in addition to time. else: + arr = arr[np.newaxis, :, :, :] coords.extend( [ - (lat_coord, 1), - (lon_coord, 2), + (lat_coord, 2), + (lon_coord, 3), ] ) @@ -145,9 +153,17 @@ def rebuild_metadata(cube, arr, lat, lon): dim_coords_and_dims=coords, ) - # Add scalar time coordinates - out_cube.add_aux_coord(forecast_reference_time) - out_cube.add_aux_coord(time_coord, data_dims=(0,)) + # Add auxcoord time coordinate that varies with forecast_period and forecast_reference_time. + time_data = base_time_units.date2num(time_coord.units.num2date(time_coord.points)) + time_data = time_data[np.newaxis, :] + out_cube.add_aux_coord( + iris.coords.AuxCoord( + time_data, + standard_name="time", + units=base_time_units, + ), + data_dims=(0, 1), + ) # Add metadata for long name, units, and preserve other attributes. out_cube.rename(meta["long_name"]) @@ -156,6 +172,9 @@ def rebuild_metadata(cube, arr, lat, lon): out_cube.attributes = cube.attributes.copy() + # Delete fill value attribute, as this tends to be np.float64(nan), which causes iris merge/concat issues. + del out_cube.attributes["fill_value"] + # Some unit corrections for specific variables. if out_cube.long_name == "geopotential_height_at_pressure_levels": out_cube.data /= 9.81 @@ -245,6 +264,8 @@ def fix_cubes(cubes): # Create output mesh, using standard grid ~2km resolution # TODO: discussions with ML developers to include metadata so # we don't have to guess target lat/lon resolution. + + # Regrid to UKV native grid.. lon_grid = np.arange(lon.data.min(), lon.data.max(), 0.02) lat_grid = np.arange(lat.data.min(), lat.data.max(), 0.02) Lon2d, Lat2d = np.meshgrid(lon_grid, lat_grid) From fb85c6c6fbecd72fde456fe6db7906d349a92a63 Mon Sep 17 00:00:00 2001 From: James Warner Date: Thu, 13 Aug 2026 17:39:05 +0100 Subject: [PATCH 11/28] keep backup --- utils/proc_fastnetuk/proc_fastnetuk_regrid.py | 329 ++++++++++++++++++ 1 file changed, 329 insertions(+) create mode 100644 utils/proc_fastnetuk/proc_fastnetuk_regrid.py diff --git a/utils/proc_fastnetuk/proc_fastnetuk_regrid.py b/utils/proc_fastnetuk/proc_fastnetuk_regrid.py new file mode 100644 index 000000000..2027dc85d --- /dev/null +++ b/utils/proc_fastnetuk/proc_fastnetuk_regrid.py @@ -0,0 +1,329 @@ +"""Fix FastNetUK inference data on UGRID with limited metadata. + +For more information on this script and how to use it, see the README.md +""" + +import argparse +import re +from glob import glob + +import iris +import iris.coord_systems +import iris.coords as icoords +import iris.cube +import numpy as np +from cf_units import Unit +from scipy.interpolate import LinearNDInterpolator + +# Lookup dictionary to translate to LFRic long_names. +UGRID_VAR_LOOKUP = { + "t": {"long_name": "temperature_at_pressure_levels", "units": "K"}, + "u": {"long_name": "zonal_wind_at_pressure_levels", "units": "m s-1"}, + "v": {"long_name": "meridional_wind_at_pressure_levels", "units": "m s-1"}, + "w": {"long_name": "vertical_wind_at_pressure_levels", "units": "m s-1"}, + "q": { + "long_name": "vapour_specific_humidity_at_pressure_levels_for_climate_averaging", + "units": "kg kg-1", + }, + "z": {"long_name": "geopotential_height_at_pressure_levels", "units": "m"}, + "sp": {"long_name": "surface_air_pressure", "units": "Pa"}, + "10u": {"long_name": "eastward_wind_at_10m", "units": "m s-1"}, + "10v": {"long_name": "northward_wind_at_10m", "units": "m s-1"}, + "lsm": {"long_name": "land_binary_mask", "units": "1"}, + "2t": {"long_name": "temperature_at_screen_level", "units": "K"}, + "2d": {"long_name": "dew_point_temperature_at_screen_level", "units": "K"}, + "skt": {"long_name": "grid_surface_temperature", "units": "K"}, + "tp": {"long_name": "surface_microphysical_rainfall_rate", "units": "mm 6hr-1"}, + "latitude": {"long_name": "latitude", "units": "degrees"}, + "longitude": {"long_name": "longitude", "units": "degrees"}, +} + + +def rebuild_metadata(cube, arr, lat, lon): + """ + Rebuild iris cube metadata. + + The cube will have metadata within its name and an additional pressure auxiliary + coordinate inferred from the cube name if present. + + Parameters + ---------- + cube : iris.cube.Cube + Original unstructured source cube, used for fixing metadata. + arr : np.ndarray + Numpy array of restructured (2D) data. + lat : np.ndarray + 1D latitude coordinate values of regridded data + lon : np.ndarray + 1D longitude coordinate values of regridded data + + Returns + ------- + iris.cube.Cube + A structured iris cube with appropriate metadata. + """ + # Determine if cube matches certain string pattern. + match = re.match( + r"^([a-zA-Z][a-zA-Z0-9]*|\d+[a-zA-Z]+)(?:_(\d+))?$", + cube.name(), + ) + + if match is None: + return None + + # Extract var and pressure, if present. + var_key, pressure_hpa = match.groups() + + # See if there is an entry for the variable, if not return. + meta = UGRID_VAR_LOOKUP.get(var_key) + + if meta is None: + return None + + # Create latitude, longitude coordinate objects. + lat_coord = icoords.DimCoord( + lat, + standard_name="latitude", + units="degrees", + ) + + lon_coord = icoords.DimCoord( + lon, + standard_name="longitude", + units="degrees", + ) + + # Create time dimensions, including forecast_period and forecast_reference_time. + time_coord = cube.coord("time") + + base_time_units = Unit("hours since 1970-01-01 00:00:00") + frt_point = base_time_units.date2num( + time_coord.units.num2date(time_coord.points[0]) + ) + + forecast_reference_time = icoords.DimCoord( + [frt_point], + standard_name="forecast_reference_time", + units=base_time_units, + ) + + forecast_period = icoords.DimCoord( + (time_coord.points - time_coord.points[0]) / 3600, + standard_name="forecast_period", + units="hours", + ) + + # Start with coordinates of just forecast_period. + coords = [ + (forecast_reference_time, 0), + (forecast_period, 1), + ] + + # If pressure exists, create additional size 1 dimension for future concatenation. + if pressure_hpa is not None: + pressure_coord = icoords.DimCoord( + [int(pressure_hpa)], + long_name="pressure", + units="hPa", + ) + + arr = arr[np.newaxis, :, np.newaxis, :, :] + + coords.extend( + [ + (pressure_coord, 2), + (lat_coord, 3), + (lon_coord, 4), + ] + ) + + # If pressure doesn't exist, just use latitude/longitude in addition to time. + else: + arr = arr[np.newaxis, :, :, :] + coords.extend( + [ + (lat_coord, 2), + (lon_coord, 3), + ] + ) + + # Create cube with coordinates + out_cube = iris.cube.Cube( + arr, + dim_coords_and_dims=coords, + ) + + # Add auxcoord time coordinate that varies with forecast_period and forecast_reference_time. + time_data = base_time_units.date2num(time_coord.units.num2date(time_coord.points)) + time_data = time_data[np.newaxis, :] + out_cube.add_aux_coord( + iris.coords.AuxCoord( + time_data, + standard_name="time", + units=base_time_units, + ), + data_dims=(0, 1), + ) + + # Add metadata for long name, units, and preserve other attributes. + out_cube.rename(meta["long_name"]) + out_cube.long_name = meta["long_name"] + out_cube.units = meta["units"] + + out_cube.attributes = cube.attributes.copy() + + # Delete fill value attribute, as this tends to be np.float64(nan), which causes iris merge/concat issues. + del out_cube.attributes["fill_value"] + + # Some unit corrections for specific variables. + if out_cube.long_name == "geopotential_height_at_pressure_levels": + out_cube.data /= 9.81 + + elif out_cube.long_name == "surface_microphysical_rainfall_rate": + out_cube.data *= 1000.0 + + return out_cube + + +def ugrid_transform(arr, tri, lat_grid, lon_grid, xy): + """ + Restructure a flattened/unstructured cube. + + Parameters + ---------- + arr : arrayy + An iris cube to restructure. + tri : scipy.spatial._qhull.Delaunay + A scipy object containing the triangulation mapping of cell points. + lat_grid : np.ndarray + 1D latitude coordinate values of target grid. + lon_grid : np.ndarray + 1D longitude coordinate values of target grid. + xy : np.ndarray + Meshed and flattened target grid points. + + Returns + ------- + iris.cube.Cube + A structured iris cube with appropriate metadata. + + Notes + ----- + This function uses a pre-calculated triangulation, to save rebuilding for + every cube. This therefore assumes all cubes being restructured have the + same flattened structure. + """ + # Create empty numpy array to store regridded data. + out = np.empty((arr.shape[0], lat_grid.size, lon_grid.size)) + + # Extract and transpose source data values. + src_vals = arr.T + + # Build linear interpolator object mapping target triangulation to source values. + interp = LinearNDInterpolator(tri, src_vals) + + # Interpolate values onto target grid using linear interpolation. + out_flat = interp(xy) + + # Transpose, and reshape to target 2D regular lat/lon grid. + out = out_flat.T.reshape(arr.shape[0], lat_grid.size, lon_grid.size) + + return out + + +def fix_cubes(cubes): + """ + Restructure ugrid cubes and then fix metadata. + + First, fixes cube metadata names as a first fix, and then regrids, and then + finally adds metadata associated with new coordinates. + + Parameters + ---------- + cubes : iris.cube.CubeList + A cubelist containing unstructured cubes, along with cubes containing + latitude and longitude information. + + Returns + ------- + fixed_cubes: iris.cube.CubeList + A list of iris cubes, that have been restructured onto a regular grid, + with appropriate corrections to metadata. + + Notes + ----- + Currently, data is regridded to a 0.02degree rectilinear grid. This is because + there is no metada in the source file that describes the target resolution + of what it should be regridded to. + """ + # First, extract latitude and longitude coordinates + lat = cubes.extract("latitude")[0].data + lon = cubes.extract("longitude")[0].data + points = np.column_stack((lon, lat)) + + # Create output mesh, using standard grid ~2km resolution + # TODO: discussions with ML developers to include metadata so + # we don't have to guess target lat/lon resolution. + + # Regrid to UKV native grid.. + lon_grid = np.arange(lon.data.min(), lon.data.max(), 0.02) + lat_grid = np.arange(lat.data.min(), lat.data.max(), 0.02) + Lon2d, Lat2d = np.meshgrid(lon_grid, lat_grid) + + # Flatten target points + xy = np.column_stack((Lon2d.ravel(), Lat2d.ravel())) + + # Build triangulation via a dummy interpolator + tri_interp = LinearNDInterpolator(points, np.zeros(points.shape[0])) + tri = tri_interp.tri + + fixed_cubes = iris.cube.CubeList() + + # For each cube, where ndim > 1 (excluding latitude/longitude array), do regridding + # on array, and correct metadata. + for cube in cubes: + if cube.ndim > 1: + print(f"Fixing {cube.name()}") + result_arr = ugrid_transform(cube.data, tri, lat_grid, lon_grid, xy) + cube = rebuild_metadata(cube, result_arr, lat_grid, lon_grid) + if cube: + fixed_cubes.append(cube) + + return fixed_cubes.concatenate() + + +def main() -> None: + """ + Run processing on FastNetUK data. + + Process produces CSET-ready netCDF files for loading. + """ + parser = argparse.ArgumentParser(description="Process arguments.") + parser.add_argument("--inputpath", required=True, help="Path to file(s) to load.") + parser.add_argument( + "--outputpath", + type=str, + required=True, + help="Path to save final output data.", + ) + + args = parser.parse_args() + + # Get file paths + inputpath = args.inputpath + outputpath = args.outputpath + "/" + + for file in glob(inputpath): + print(f"Running script on {file}") + + # Load data and restructure. + cubes = iris.load(file) + cubes = fix_cubes(cubes) + + print("Saving restructured cubes") + iris.save(cubes, f"{outputpath}/fixed_{file.split('/')[-1]}") + print(f"Done file {file}") + + +if __name__ == "__main__": + main() From 6d1c8be2387258b1f4d0200266592212f48020f4 Mon Sep 17 00:00:00 2001 From: James Warner Date: Thu, 13 Aug 2026 17:39:23 +0100 Subject: [PATCH 12/28] switch to reshape --- utils/proc_fastnetuk/proc_fastnetuk.py | 159 +++++-------------------- 1 file changed, 27 insertions(+), 132 deletions(-) diff --git a/utils/proc_fastnetuk/proc_fastnetuk.py b/utils/proc_fastnetuk/proc_fastnetuk.py index 2027dc85d..5ab37492b 100644 --- a/utils/proc_fastnetuk/proc_fastnetuk.py +++ b/utils/proc_fastnetuk/proc_fastnetuk.py @@ -13,7 +13,6 @@ import iris.cube import numpy as np from cf_units import Unit -from scipy.interpolate import LinearNDInterpolator # Lookup dictionary to translate to LFRic long_names. UGRID_VAR_LOOKUP = { @@ -39,7 +38,7 @@ } -def rebuild_metadata(cube, arr, lat, lon): +def rebuild_metadata(cube, grid): """ Rebuild iris cube metadata. @@ -50,12 +49,7 @@ def rebuild_metadata(cube, arr, lat, lon): ---------- cube : iris.cube.Cube Original unstructured source cube, used for fixing metadata. - arr : np.ndarray - Numpy array of restructured (2D) data. - lat : np.ndarray - 1D latitude coordinate values of regridded data - lon : np.ndarray - 1D longitude coordinate values of regridded data + grid::: Returns ------- @@ -80,18 +74,9 @@ def rebuild_metadata(cube, arr, lat, lon): if meta is None: return None - # Create latitude, longitude coordinate objects. - lat_coord = icoords.DimCoord( - lat, - standard_name="latitude", - units="degrees", - ) - - lon_coord = icoords.DimCoord( - lon, - standard_name="longitude", - units="degrees", - ) + # Get latitude, longitude coordinate objects. + lat_coord = grid.coord("grid_latitude") + lon_coord = grid.coord("grid_longitude") # Create time dimensions, including forecast_period and forecast_reference_time. time_coord = cube.coord("time") @@ -119,6 +104,9 @@ def rebuild_metadata(cube, arr, lat, lon): (forecast_period, 1), ] + # Reshape cube ADD CHECK IF NOT CORRECT SIZE + cube_data = cube.data.reshape(cube.shape[0], 808, 621) + # If pressure exists, create additional size 1 dimension for future concatenation. if pressure_hpa is not None: pressure_coord = icoords.DimCoord( @@ -127,7 +115,7 @@ def rebuild_metadata(cube, arr, lat, lon): units="hPa", ) - arr = arr[np.newaxis, :, np.newaxis, :, :] + cube_data = cube_data[np.newaxis, :, np.newaxis, :, :] coords.extend( [ @@ -139,7 +127,7 @@ def rebuild_metadata(cube, arr, lat, lon): # If pressure doesn't exist, just use latitude/longitude in addition to time. else: - arr = arr[np.newaxis, :, :, :] + cube_data = cube_data[np.newaxis, :, :, :] coords.extend( [ (lat_coord, 2), @@ -149,7 +137,7 @@ def rebuild_metadata(cube, arr, lat, lon): # Create cube with coordinates out_cube = iris.cube.Cube( - arr, + cube_data, dim_coords_and_dims=coords, ) @@ -185,113 +173,6 @@ def rebuild_metadata(cube, arr, lat, lon): return out_cube -def ugrid_transform(arr, tri, lat_grid, lon_grid, xy): - """ - Restructure a flattened/unstructured cube. - - Parameters - ---------- - arr : arrayy - An iris cube to restructure. - tri : scipy.spatial._qhull.Delaunay - A scipy object containing the triangulation mapping of cell points. - lat_grid : np.ndarray - 1D latitude coordinate values of target grid. - lon_grid : np.ndarray - 1D longitude coordinate values of target grid. - xy : np.ndarray - Meshed and flattened target grid points. - - Returns - ------- - iris.cube.Cube - A structured iris cube with appropriate metadata. - - Notes - ----- - This function uses a pre-calculated triangulation, to save rebuilding for - every cube. This therefore assumes all cubes being restructured have the - same flattened structure. - """ - # Create empty numpy array to store regridded data. - out = np.empty((arr.shape[0], lat_grid.size, lon_grid.size)) - - # Extract and transpose source data values. - src_vals = arr.T - - # Build linear interpolator object mapping target triangulation to source values. - interp = LinearNDInterpolator(tri, src_vals) - - # Interpolate values onto target grid using linear interpolation. - out_flat = interp(xy) - - # Transpose, and reshape to target 2D regular lat/lon grid. - out = out_flat.T.reshape(arr.shape[0], lat_grid.size, lon_grid.size) - - return out - - -def fix_cubes(cubes): - """ - Restructure ugrid cubes and then fix metadata. - - First, fixes cube metadata names as a first fix, and then regrids, and then - finally adds metadata associated with new coordinates. - - Parameters - ---------- - cubes : iris.cube.CubeList - A cubelist containing unstructured cubes, along with cubes containing - latitude and longitude information. - - Returns - ------- - fixed_cubes: iris.cube.CubeList - A list of iris cubes, that have been restructured onto a regular grid, - with appropriate corrections to metadata. - - Notes - ----- - Currently, data is regridded to a 0.02degree rectilinear grid. This is because - there is no metada in the source file that describes the target resolution - of what it should be regridded to. - """ - # First, extract latitude and longitude coordinates - lat = cubes.extract("latitude")[0].data - lon = cubes.extract("longitude")[0].data - points = np.column_stack((lon, lat)) - - # Create output mesh, using standard grid ~2km resolution - # TODO: discussions with ML developers to include metadata so - # we don't have to guess target lat/lon resolution. - - # Regrid to UKV native grid.. - lon_grid = np.arange(lon.data.min(), lon.data.max(), 0.02) - lat_grid = np.arange(lat.data.min(), lat.data.max(), 0.02) - Lon2d, Lat2d = np.meshgrid(lon_grid, lat_grid) - - # Flatten target points - xy = np.column_stack((Lon2d.ravel(), Lat2d.ravel())) - - # Build triangulation via a dummy interpolator - tri_interp = LinearNDInterpolator(points, np.zeros(points.shape[0])) - tri = tri_interp.tri - - fixed_cubes = iris.cube.CubeList() - - # For each cube, where ndim > 1 (excluding latitude/longitude array), do regridding - # on array, and correct metadata. - for cube in cubes: - if cube.ndim > 1: - print(f"Fixing {cube.name()}") - result_arr = ugrid_transform(cube.data, tri, lat_grid, lon_grid, xy) - cube = rebuild_metadata(cube, result_arr, lat_grid, lon_grid) - if cube: - fixed_cubes.append(cube) - - return fixed_cubes.concatenate() - - def main() -> None: """ Run processing on FastNetUK data. @@ -313,15 +194,29 @@ def main() -> None: inputpath = args.inputpath outputpath = args.outputpath + "/" + # Load mask containing lat/lon to project onto. + ukv_mask = iris.load_cube("/data/scratch/james.warner/1308_tmpfastnet/ukv_mesh.nc") + for file in glob(inputpath): print(f"Running script on {file}") # Load data and restructure. cubes = iris.load(file) - cubes = fix_cubes(cubes) + + fixed_cubes = iris.cube.CubeList() + # For each cube, where ndim > 1 (excluding latitude/longitude array), do regridding + # on array, and correct metadata. + for cube in cubes: + if cube.ndim > 1: + print(f"Fixing {cube.name()}") + cube = rebuild_metadata(cube, ukv_mask) + if cube: + fixed_cubes.append(cube) print("Saving restructured cubes") - iris.save(cubes, f"{outputpath}/fixed_{file.split('/')[-1]}") + iris.save( + fixed_cubes.concatenate(), f"{outputpath}/fixed_{file.split('/')[-1]}" + ) print(f"Done file {file}") From ad2578cfcb9f86ace8b2fd22409376a885ed8956 Mon Sep 17 00:00:00 2001 From: James Warner Date: Thu, 13 Aug 2026 18:02:43 +0100 Subject: [PATCH 13/28] add fill to prevent masked array --- utils/proc_fastnetuk/proc_fastnetuk.py | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/proc_fastnetuk/proc_fastnetuk.py b/utils/proc_fastnetuk/proc_fastnetuk.py index 5ab37492b..e78cce05c 100644 --- a/utils/proc_fastnetuk/proc_fastnetuk.py +++ b/utils/proc_fastnetuk/proc_fastnetuk.py @@ -106,6 +106,7 @@ def rebuild_metadata(cube, grid): # Reshape cube ADD CHECK IF NOT CORRECT SIZE cube_data = cube.data.reshape(cube.shape[0], 808, 621) + cube_data = cube_data.filled(np.nan) # stop it being masked # If pressure exists, create additional size 1 dimension for future concatenation. if pressure_hpa is not None: From 10321790562ff85dd216c5bb392f727378e373db Mon Sep 17 00:00:00 2001 From: James Warner Date: Fri, 14 Aug 2026 09:52:39 +0100 Subject: [PATCH 14/28] add fill/reduce mask end of fix --- utils/proc_fastnetuk/proc_fastnetuk.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/utils/proc_fastnetuk/proc_fastnetuk.py b/utils/proc_fastnetuk/proc_fastnetuk.py index e78cce05c..abe062125 100644 --- a/utils/proc_fastnetuk/proc_fastnetuk.py +++ b/utils/proc_fastnetuk/proc_fastnetuk.py @@ -106,7 +106,6 @@ def rebuild_metadata(cube, grid): # Reshape cube ADD CHECK IF NOT CORRECT SIZE cube_data = cube.data.reshape(cube.shape[0], 808, 621) - cube_data = cube_data.filled(np.nan) # stop it being masked # If pressure exists, create additional size 1 dimension for future concatenation. if pressure_hpa is not None: @@ -171,6 +170,9 @@ def rebuild_metadata(cube, grid): elif out_cube.long_name == "surface_microphysical_rainfall_rate": out_cube.data *= 1000.0 + # Fill any np.nan, as issues with read-only arrays in plotting. + out_cube.data = np.array(out_cube.data.filled(np.nan), copy=True) + return out_cube From fb19b63085cdda3294f6385ac560e75b2f5959b7 Mon Sep 17 00:00:00 2001 From: James Warner Date: Fri, 14 Aug 2026 10:25:04 +0100 Subject: [PATCH 15/28] fill masks ahead of stats --- src/CSET/operators/plot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/CSET/operators/plot.py b/src/CSET/operators/plot.py index b12f423ed..fc27c7df4 100644 --- a/src/CSET/operators/plot.py +++ b/src/CSET/operators/plot.py @@ -745,7 +745,7 @@ def _plot_and_save_spatial_plot( # Add watermark with min/max/mean. Currently not user togglable. # In the bbox dictionary, fc and ec are hex colour codes for grey shade. axes.annotate( - f"Min: {np.nanmin(cube.data):.3g} Max: {np.nanmax(cube.data):.3g} Mean: {np.nanmean(cube.data):.3g}", + f"Min: {np.nanmin(cube.data.filled(np.nan)):.3g} Max: {np.nanmax(cube.data.filled(np.nan)):.3g} Mean: {np.nanmean(cube.data.filled(np.nan)):.3g}", xy=(0.025, yinfopad), xycoords="axes fraction", xytext=(-5, 5), @@ -1474,7 +1474,7 @@ def _plot_and_save_vector_plot( # Add watermark with min/max/mean. Currently not user togglable. # In the bbox dictionary, fc and ec are hex colour codes for grey shade. axes.annotate( - f"Min: {np.nanmin(cube_vec_mag.data):.3g} Max: {np.nanmax(cube_vec_mag.data):.3g} Mean: {np.nanmean(cube_vec_mag.data):.3g}", + f"Min: {np.nanmin(cube_vec_mag.data.filled(np.nan)):.3g} Max: {np.nanmax(cube_vec_mag.data.filled(np.nan)):.3g} Mean: {np.nanmean(cube_vec_mag.data.filled(np.nan)):.3g}", xy=(0.05, -0.05), xycoords="axes fraction", xytext=(-5, 5), From 9e86f150d64630dce7ab0820296ef2c00f755478 Mon Sep 17 00:00:00 2001 From: James Warner Date: Fri, 14 Aug 2026 10:25:22 +0100 Subject: [PATCH 16/28] switch to local mask path --- utils/proc_fastnetuk/proc_fastnetuk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/proc_fastnetuk/proc_fastnetuk.py b/utils/proc_fastnetuk/proc_fastnetuk.py index abe062125..30fc6d9f7 100644 --- a/utils/proc_fastnetuk/proc_fastnetuk.py +++ b/utils/proc_fastnetuk/proc_fastnetuk.py @@ -198,7 +198,7 @@ def main() -> None: outputpath = args.outputpath + "/" # Load mask containing lat/lon to project onto. - ukv_mask = iris.load_cube("/data/scratch/james.warner/1308_tmpfastnet/ukv_mesh.nc") + ukv_mask = iris.load_cube("ukv_mesh.nc") for file in glob(inputpath): print(f"Running script on {file}") From b4ee246bbca762c1d66e8893ab97e01669f3a4f4 Mon Sep 17 00:00:00 2001 From: James Warner Date: Fri, 14 Aug 2026 10:25:34 +0100 Subject: [PATCH 17/28] add mesh file --- utils/proc_fastnetuk/ukv_mesh.nc | Bin 0 -> 2026111 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 utils/proc_fastnetuk/ukv_mesh.nc diff --git a/utils/proc_fastnetuk/ukv_mesh.nc b/utils/proc_fastnetuk/ukv_mesh.nc new file mode 100644 index 0000000000000000000000000000000000000000..e1b9708140d798ce07e67fe7df05064d6d748626 GIT binary patch literal 2026111 zcmeI(cbpVu-Y)ziiZ}uy3L>i`vWkc#5fx?nMuH*&Dx#?9h%mq?$zed2MQjxlV$NAa zMa(%MOqW?vSyW`rIp>_v<<#8+uDfS_pZ7WE^ErQ<^DShr>At$FtG>VLo@%WZ?!Y*r`H~PmGcMs{a$H3b5#gA%Vb;_Eyq;rxqNa`icl16d$Qzw*9sUA6H zT=|qqQ>#W*BpYTAYn=V9oBf^8rR$U~-7C9x?NYL9k~GYow0`ZA##U91E}t-R^5n{i zV~fvdz2+HJld4BnSBx$nKeD>Adg|zk^6`@%r!eR{BUHj+>abL1!ZPsfMpZK5F-JnTq>UH?%A#%m;wTkVsSB}e&+DO*UrqLhH zrlfa#XuWI~OX6;p53Z;lHF`|=1 zd-WbVv`8Kgw76q^2Mp>rc<6v3gUg5Z>AhFK+B>uRciN%D9g(}?7TJxYPW?3pSJI?7 zyx-J6iZ7Ba2h6zso_`)RyF5{Q(7an)CMDa%_14P{oZ>+HKqb)DHA zl|4#UHR-e4FSXOSw@#gQaZPoz;k92Qxpo@=etYe7Hvf&LWuv6!vDvd5G_IXaE!med zYbWzXTyeHsSR4O;CX*rkHz%{^2rBODn%5~Fd?h1lr_(fhJ7q)u=PdRbFm%6UlkA~! zOf9p&@sNs#?!T|8$iL=zi$DMVc6@A{f0S>M9Y)!XuRVs=uZPqjz>kK z=GpC}UY$!Hs68&ODv7tV7i(Mlx$r-&BYq^sb&ReUTUAjpr6l`kU8lGg#n;L{aVCx) zSvC5f8@B$M$JZW6Yd$@e+_~VD(*O9*IC{-NRMg6zde3{^>K8v-){M4E?LL(c?z4B* zq{$Uk)s@-XEq=6R+gseOX%$sdDkn|s)ve>i;`_2!_f9)^>eg}G=rKF=>e{Jm_fFf_ z9$n2}uifEn`)ikmF=`*tJ6UIo-#%@M2W8yjdL?m&zkh50{3y{f&DVmjx29ej zvtB$R6hCdYi}%#2zh*v3>p%XDa6pkVNm^Vpe3Rnr`p414-!aAC3l$@)s>hXAjT~J$ zb;^JIu>ONz1mS~0!+Za9_soaaZdz^b-e<^O#f^_|$ZT1=BWpkEyWL;AhjD~)S$qH5 z&&S$(lO&BFc@_c$2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0{@Q*4C%ASz>gbAOgVJ2msOS@t?&~GpSJz9D zx^=Q;joI2P-b{B3B}0>>ezxA$f3V(36UY89thmdV;))ymbH%^g*5bC0|6f~iw%Nn4 zZM)r&J^Jojcm1SJos6P(|JrB!8JD$LuSJsW>IS8oXL}cy(xhdhq~)>MPYoK!re25I z`@fvotXb?&U9?$@r1=`4joK8SSg%pC`VW?p-tqDEGPjaAk@CS6)uTp_>E373q^i-C z6Gv88Oi7w$lUV!me>8-WZIf(!>Lm@cv1O0#vsdq-Lz4}&HOI}WoBi!OU{JroLkA2Q zTt2i<@4fmZ*&}~j7Omgtpm=mNERA>7sbBkINwR*e@tT98+x@lsR=cO)#g%4D=h}#` z9+%f&I;dS-Rqca%HmGf9)kad%K5k6Cq(SXg4(>guUvVRwWPM!Uzr0=&_bDE%O){En zlh)Got(;If`d?Aju71|tw` zu{$g|`-iEM(QC#1X7nUJp6ZhZP`|0C0zE00KL&8} z)wg5PhgyzF&+S=}jy_~`+C4uiZ7~1vbmgNX(*++MmQG*quyjDD^0f6nho+xQIV4?t z!H9J3{RgLqyn0aDY4w5W>MagTm-ii>&Kr9`S~X|?w9n1M(iY42OW*x^XnOZXL(`LX z+BZGm;C<5ev-VDZx@t(e^x?tjrSA_)$FDmm-L>OhX_FxX)7OvMGrje^0crl;J<@$% z-aT#mYyb3{&HJa1?A9+mucB`{=9E5Zj~jPO*ICv(eeuhXF5EDrGkWCGfd`hQZD#gL zKf7YrwC2HG(lg%enU=59Gwsr0=d{kCozmwf?U>F#r$<_S_YP^l7rUn$ui8F+|4-Ye z_l0ihDWkfk2cFa=?Xa+Oy6UM;>C>NYmtMZXcIm|K9n-P{woRMO=#aj7**59zh4$(3 zZ*QFrZLoFPuKiZ&cYFRheQZLz^n$atOvl~XHr;9E7HQ+3O4FA&ElqDM+dQ2)aM z>_4TOEohT|{^X`<`f2O*tfsBg!@F&gb{p0ztv9`8y5f?J(`%P(lumoIMcTh!i?r2N z8>S!b(L7yr-X!Y%Qjg*y>i!*baMH6X*j-d+Wgve)3+aQ zl-~LAI_d27*GY$UX_#&`v_bmA)cV=X>!lYxP&b=-owR42I%!F}B)#wSe-sXS{+Gfp ztyUGTIP|ANxaNn#TOWN_IHB{mg+EXIy72g-FAL+6FA9zOe^$8Zv`-3ypZ%!t<;EWt z&K~i8Vf(A!Exho-+l8aId#liD${U46_q|pa@sC#u|LF5_;i^+#EbR8Tm4$a(yihpl zpyvx)UG;3?iTD0iIAYsng_5J5F5Gg@(!$=qJW=?%_v3~0PJXn|{d7-AsWrf4`yQEO}l8Xx0 zym3LH|DVq7;|nj3%NKqeaZF*?5C2lgPnlMDARj$I0&Y^TEP zC%YG(*sp7$PS3r?Au5`h}TaCxz*&ey&NHe_L}(`_F2c<=(HkXxJMyrQ=_$x&HWPYr0&# zwC28BAFc_HEv`vly}M?QZ*HkszV7uk!%OGYyxHTDnsI~9t@(1)+?we#POM4V9#fMH znN-tk=E$1Tg+pt)tn614>g`mscb5(|haJ+UX7ZdSHTip!n$joVTT;5^(k0=TTb3O5 z_j8scqb4p1Eq7d!T)291e)U<4ldDdBASrLPYyR;?yXJ3g(JQPP*egH(xL$`1xV%^R z;=W#`=db9M?DA!=$zL`oJM7^N%MQC>i?Y2dJC}uBdzE!*I(_xamrn zw!JUT&02Ly?)AGb%au*OB6mrLt8zcTJ1@8YmGg7=9C&T6d5Z! z#V_OzpZ51$O_!H)n|<>O zO`QI5ZfUnqa~(eWEO*j^FLLjX{3_SK%{RHLSALtTd-nIaLkIqtt7-6au64R9H)F=H zxs}~l=Po@u2~X`;C;Zg4ZrG-2y>P&%^}@+d)(>}I*C4!iLc_4($aO-WvPNObrt5}z zKdu{|eW7t!eaCvC^I0X~kO}LD)Awu=9@x5R_@sWbu<6^)!hpgC;V+jp4-5XXVfg$0 zEkc9t8-*P<*f@;(eB*HLQ!T@T*S88^ow!NZVpQueBy1Xv{ZpH8^UrO*e;xS#h=5Sf88p)Gkoi?L67#K_lDbq zqrTiGT=`6g@V6Vb4Zoh;F>F_{T{w8RPT{o8JBRy!=^Q?KsY}@8-mYQy^SXtjkJ>(5 zJGgszVcQ)-y>)wp?ceVaMn1e_IQy!dLgAR5!xsnj44dz?OBl4#uHl%kcMUiGtyg&E zrn0c^oLty>YzSlf_6`?rv0HfjAG?L`Ug;D5e1G3C^n!ljgem>Q?R)PY-r8=D&~&{4 zA@|{cFzL}f!{t{G49^_9S6DS-P}sKT;Ba8eA>q_-hlG2d-8+16%RZsy+%x7gYKQ@ftHXrIRd}X_MJu|6^u{&p(+RPCDqsu-TI*hI@8ADeQIK zN#WfVCx@9QoE(~eeR8)|--0RdZ`KD7tqfO_8t4^I0cKUHnSU%>oFy@8R z!aw@V4Hw)oH+0(K^zg`;r-u>$I6Ztj{){m9l`}%SJ2uxb3a8!ybE|6W**jCrsb&+|cB*bHjCw&kJRLIWMgI;Jh$?|MNrrN6!zJZGS=N ze)R?6>1G#(!;if%{QTL4;hYf{g$_?$6w;k94#Tg%IDEPBCE=74FA1gJUJ~v({L(P^ z*-OLwxy!<=TP_PN{&ab`>9os3-=8lJua3JSRR8^o(76AV;p)4t3_Z8JDm-`gRbgB* zFC>%Zg^OOB7rN|qb$DXY)#1=>=7;YuoFC3;cum-9>NR2UJJ*E$_PsWInqC`bcfKyP zx#GHTN6CV)=U*3uw?A4CW(>bRG<*E|aQzMoL%3#Pc(M5nVd8N&goa<<5Ux1%#?a%L z8^f|)ZVIDsxGDV7>gI6X$v20N-`^Y_8huMR==od1*S&8Ib8fpewB7u+aNp^-g}r~d zEqr*y?IHit?P23R?g+Qsb4Td^=R3pe=iV8n)x9f}9CcT?=8e0;u7mFmD;D1!j@b5| zZ07f5Gru>R`Muf9?+u6TcV9O1`?8tepUwRKP}5{lHuFW<%ok-de;}Lr1L3Y67iTkH zoXvbmHuELf%$H;{ugPX!lg+#ks%|P|Gf%Ubr`gQYY~~MUGk-9f`9s;vAIfI_aCq#@ zhqIYKoXz}^Z03(-Gk-Lj`J>s)AIoO`ST^&=L+dGzXET31oB0#j%%8|+{$w`uC$pI^ z&1SwdoB7gg=1*lae=3{#)7i|Q&Sw5hHuGn)nLm@wd|5X0W!cRCmd*Tc+02(`Ghd#~ ze0etWXS10WZ065pGk-pt`SaP#pU-B#BAfY&Z00XyGk+nQ`O0kOE3=vZJ)8O8 zvzh-roB4~`%wNoA{!%vcm$I3^oXz~@Z00X#Gk+zU`77DXU(IIzYBuxNvYEe@&HS}& z=C5Zne?6P|8`;d?$Y%a#n0(xu+05U}X8u+-^S82@zn#te?QG`nWHWy!oB2E8k-`PJGldGRZ5T>e|%!Q%IB_t z9T+q0SMP+F$)-aV#Z2G2X>QEpKgqqf4%QJ#s1o}T6c^6_*eIdnRaV`YRvpkQ_`3rE&NdIPhPL{_OTyy zz?hi%7V|HU8D9B(Ma<;to+aI5KklSHG4q|Dn-Md7|ED`*CQB+mD)uMbUAjZ;_pCEK zX4qiRnK6?WFM2Fydd0iH7W;}WJM@Ts$C=|}Cf__hKW2LWhOflT=gXV#82jCC91t^U z@6qb|b?fgO`@X|=iJ3N@ zR~0k=?57K3hKqK3tJs&%Zq+mPe_lQ$X8zMnPmUSx99t7JnQ`ZL#XhX=xJ&HU88|v- zSa!jsF_ZJ(elBKuaHqz*#=h;0-Z8^Rk4}%7+_u4OG1F;>d{FFrE!?bE?CXCwEN1dl zI6Y>1_S{Ee=7+zys@QMdZu_#>zdz}SnCVRq%!`?yvd&8}L(iejapdt!!R$A4Dr+bnArV*mCgBVy(kR-79%Oul_-%w)%3>-3KOuY2qqGylkWlVgU{ z-dqqf*|*~x#lF?_jdzRv8xIeP8Ln%3V$5X1!HZ+2+b{U0*#G=>hd!}?u}x(88Z({U{j`|* zf%%7GhUQQIRP0}B*|mS{uN*ZlW`4}ASH=w6ty&o~`L=(P-DAJ#oPII$$Gtu?X4s>{ zT``lU(>^KoFFe?GkJw+f{y{Orr~}WAnRK}JiJ0k^U#>3p_wKsmfY{GEWn#>v-->Hv zrt6iy7Bm0ckuCO&{l)hVjF}u(?}V7?RznuW%zt*tSH=FW_uCJQ{a?Es7BlTV>%y4% zMoX8)41e3G;a;)7U}UeD>4=-FW9GN~>BgAh(>FdO_MbdFC}wg;(-UK+(+^%8GoM@V zO|ftIO@~8c|4iB8G1GJBTpTlh(8}d8!xn8DmB;?W3Avc*E%#50nV;I=mY8AJecmhf z^)7F7SnQwtaA?f@S=~>I84k!l6f@cE>7R=IyDhtpjQx$H#>EU(w_X`D*?HB&TTlC>*uV8)+flKo`&@;Sj_yOSr^6(EtW2enY^-5!?CfSH?mjE{J5K|V}{N@-54|ZzVAE5e(_nY z$Ho4HSNDk-_H2J@%w&V=G-mqu!Vks%iuF2I#=hc!F)@>l^DmE?e*O81nECsAmK+iL z{7HRcCc8g3BWAk(W_QHQuc-W}*k5{g>5;KNyw32LY5PHE#>{_l(PJ^gJ@5Wn?2qZP z!}!?unK?dYzVQ?DV}@rpd?jXbQF-$Tu|M?20WtG`{vjVTe7f6xF_Syb_@dbV<>jp= z#y)IyXw0xq)%h`#WlNrpnVz?9{YkMuc-Ss6L)&>(F_VuzT^KXHb*HzA{j}MwCdaaFn>373o=HCvd#|#VSJ`yvT{Nk!&-=p33)v^C| z(h)JkBM;1rnVh!HOEJ@ZhBliT`&L)(9y7f0@vNB1bv^EhnT|jHvtqyfvUbyA|MMmz zVkQq(oEtMe`SztT^Mik_b9C%C8nAQB3+w7sm_(7kpFfn}5^cFR_2I?C_ZRE9YDsGmKfeJZ7?8+eR~D|ILJ4%>1JJ zr^O7%HMk{avd2E}75gTaxA|-AU-)on%y3!v(_$v0@(;yKw|V-fV*h2!t}|nQ@2GJx zlUcW388hv>YGus)db>9{Cic&r(=TRn@#`~VrsW;(ikaVP+9$>Svj^MGiv3;d9~3j4 zap2i8^S!TqB4%jx)#_sZw_SHUHumSAGBIX;#ENTUhAm59i2>y zPKX)GhAfJiG`RGuV*k|p?T?H7Io%G684jFvVa%j->9UyV`x`YpKK3_{>=iRq-&`Ft z+2yAjW2SZby;JO;IIHyuu|M&B~98{ertoPm2A7I>Te;yAL`u zW>|I6V=X1?UH=`q9e8{8H%x#W-!iv3{=H#;NtTYon!X81gu9y7Un z?jteNzrMJt*zeYE`!i#|?&Kq4Cd(g~7c;$JotI+f4;k9*tk}1^a`%|YCm+v>ncm*x zo|yUR$A4DrbIaPD9s7o@N5o8@t~fVl{@mM_#taAjTIZbDZ!uu!nCS=SO^%tr<;?{# z!_pn6lX8y^hC&mnC9lSVZa=?Obiv6Fy>2O}`-z_^lX1HFzP!x^v488sp)r%|yPp;_otS?p zW`2jKe=7FBwCs9e>>nOAE@pb_tyjj(@4aee%+PZ8CKtv2^>g~gOs{!;X3YGN9qx)5 zx=s6}*#Gok+lymgSpT4y`I8PjJ7ySk?GrJR7GJF{_Al?c<0Y}5cgn<=Vcd#qV^Wpn%(U61Ulsemzu*3{*k94@u$W23tP5kN+b&%e zGynBQ4KI)V{UdwD%--2?bz9{x*zr58ou^+zGp)u3Vs?Lv@e{adtF~d#k z*1tCPQ-jjs_OdfgrxtQskPK~dR{XR2##|*6=n;tWHV}sjbrq>?wL9rjd zaI=N6-~PK{F_WLe=`quE?jtetC%?F=*bi>E{SC3-Xz~#;(^npt7c+nLIxocxl|!4| z82c_)?jAG!{^MCO^GkZ%6En;{{GR-79%TzUJ_n8}!5>)ags zjstd%ng8az$uYyCHy6ZAj_deFvEP0A#<#@2$s>beh8LQi7&E!-;Kec1!xwy0?6>)* z!>zIZvh47f$-Q$fj+q{_a(T>r-?oizi~V{Nb1{?W?w=Mjy{N%0G4tj7yjSeEy1dQp zvH$GDp)u1tyPp;_KO_H8%+UMkpNf5>R$cFi{jyQxV&>1k^~#uG#Hy7slPz~|a%b#6 zI;UUE{B5t#j2VvZa97NvY}zNqzWzgP?~46X>mL*|oO9sWF_QzYeIjPM`B$s6jQ=jN zQS&5Q>ebo&$=ZLLI4)j^k}V}|<8>#$U8OR0Vr6yh6&mYgWxPUTbj8@Jii#;EU9)#Z z_P8ds*NIH29yxLJ$g1p8rI8aV{=)?$YcB$tJ@?Q{e!Hfpc$G$6XT3Go@chqLG&N`v zuUe_sAztHAyIisR+D+f>_1aBe+pWElr|utL$x|BF@gH_BUIY_2WZa~wRZ~j3)n2Ic zhr8Bv&7;Rms;U?@a!Pgi!bY5H&LZTMR4 zes0~T?i;^feb{re-tqR@{C($V0t5&UAVA>%9)bVebEn#ie&>Hsdq%O>8kn_bPWAry z%&BL5=zlnK+C01aw=<{GDU}mPRg`w!dB-l>b?KV@Q`)6V&*DEN+4JMY&wqGM6))tC z>-p{6r>bI1MO8(%y7KDm+TNsb*8OhTb6(T7|NKwlt3IsVpV!ttqV~TlH!Plz)u|s( zUE|X74_3b0`?XKMyY%^Ye}8Vg=f9m`mTVQbv0k!n?K$tLk>e}kNpJCVH=cyX?O%H$ z{Ko7#C;smaYwd~dzW=rz{|Cmk^?xz0cT$TEn@fV-r}<6!t(6rdsqLp zP(0C%L(Wcg;}hdjKVD3}_HyioYs$4}^Y!DZzgTmimXy_Q&pK^hm09hnH`A3CRUEAtQcK7XwvA4@!6HV#p$hAyN;<7%BNLS z6)zc&Uwzi@P}l7{?NpNOeY~o@r1p`0CQY1HF|j(kxV-k8P&`kMFWIO6c3nH|*n8+l zxBmX1Nm^x9wv=8~>rpbO_KVV*m5ut{|7#$=ZpZ9zT-k4jX6<*0(%)|X>-(TIs$6tg S#iQ{h|2xOyZTDYw`~LuAqNj@h literal 0 HcmV?d00001 From 7b41620a868df0457e059a814b2bc6bf09ab7f41 Mon Sep 17 00:00:00 2001 From: James Warner Date: Fri, 14 Aug 2026 10:27:08 +0100 Subject: [PATCH 18/28] remove old script out of repo --- utils/proc_fastnetuk/proc_fastnetuk_regrid.py | 329 ------------------ 1 file changed, 329 deletions(-) delete mode 100644 utils/proc_fastnetuk/proc_fastnetuk_regrid.py diff --git a/utils/proc_fastnetuk/proc_fastnetuk_regrid.py b/utils/proc_fastnetuk/proc_fastnetuk_regrid.py deleted file mode 100644 index 2027dc85d..000000000 --- a/utils/proc_fastnetuk/proc_fastnetuk_regrid.py +++ /dev/null @@ -1,329 +0,0 @@ -"""Fix FastNetUK inference data on UGRID with limited metadata. - -For more information on this script and how to use it, see the README.md -""" - -import argparse -import re -from glob import glob - -import iris -import iris.coord_systems -import iris.coords as icoords -import iris.cube -import numpy as np -from cf_units import Unit -from scipy.interpolate import LinearNDInterpolator - -# Lookup dictionary to translate to LFRic long_names. -UGRID_VAR_LOOKUP = { - "t": {"long_name": "temperature_at_pressure_levels", "units": "K"}, - "u": {"long_name": "zonal_wind_at_pressure_levels", "units": "m s-1"}, - "v": {"long_name": "meridional_wind_at_pressure_levels", "units": "m s-1"}, - "w": {"long_name": "vertical_wind_at_pressure_levels", "units": "m s-1"}, - "q": { - "long_name": "vapour_specific_humidity_at_pressure_levels_for_climate_averaging", - "units": "kg kg-1", - }, - "z": {"long_name": "geopotential_height_at_pressure_levels", "units": "m"}, - "sp": {"long_name": "surface_air_pressure", "units": "Pa"}, - "10u": {"long_name": "eastward_wind_at_10m", "units": "m s-1"}, - "10v": {"long_name": "northward_wind_at_10m", "units": "m s-1"}, - "lsm": {"long_name": "land_binary_mask", "units": "1"}, - "2t": {"long_name": "temperature_at_screen_level", "units": "K"}, - "2d": {"long_name": "dew_point_temperature_at_screen_level", "units": "K"}, - "skt": {"long_name": "grid_surface_temperature", "units": "K"}, - "tp": {"long_name": "surface_microphysical_rainfall_rate", "units": "mm 6hr-1"}, - "latitude": {"long_name": "latitude", "units": "degrees"}, - "longitude": {"long_name": "longitude", "units": "degrees"}, -} - - -def rebuild_metadata(cube, arr, lat, lon): - """ - Rebuild iris cube metadata. - - The cube will have metadata within its name and an additional pressure auxiliary - coordinate inferred from the cube name if present. - - Parameters - ---------- - cube : iris.cube.Cube - Original unstructured source cube, used for fixing metadata. - arr : np.ndarray - Numpy array of restructured (2D) data. - lat : np.ndarray - 1D latitude coordinate values of regridded data - lon : np.ndarray - 1D longitude coordinate values of regridded data - - Returns - ------- - iris.cube.Cube - A structured iris cube with appropriate metadata. - """ - # Determine if cube matches certain string pattern. - match = re.match( - r"^([a-zA-Z][a-zA-Z0-9]*|\d+[a-zA-Z]+)(?:_(\d+))?$", - cube.name(), - ) - - if match is None: - return None - - # Extract var and pressure, if present. - var_key, pressure_hpa = match.groups() - - # See if there is an entry for the variable, if not return. - meta = UGRID_VAR_LOOKUP.get(var_key) - - if meta is None: - return None - - # Create latitude, longitude coordinate objects. - lat_coord = icoords.DimCoord( - lat, - standard_name="latitude", - units="degrees", - ) - - lon_coord = icoords.DimCoord( - lon, - standard_name="longitude", - units="degrees", - ) - - # Create time dimensions, including forecast_period and forecast_reference_time. - time_coord = cube.coord("time") - - base_time_units = Unit("hours since 1970-01-01 00:00:00") - frt_point = base_time_units.date2num( - time_coord.units.num2date(time_coord.points[0]) - ) - - forecast_reference_time = icoords.DimCoord( - [frt_point], - standard_name="forecast_reference_time", - units=base_time_units, - ) - - forecast_period = icoords.DimCoord( - (time_coord.points - time_coord.points[0]) / 3600, - standard_name="forecast_period", - units="hours", - ) - - # Start with coordinates of just forecast_period. - coords = [ - (forecast_reference_time, 0), - (forecast_period, 1), - ] - - # If pressure exists, create additional size 1 dimension for future concatenation. - if pressure_hpa is not None: - pressure_coord = icoords.DimCoord( - [int(pressure_hpa)], - long_name="pressure", - units="hPa", - ) - - arr = arr[np.newaxis, :, np.newaxis, :, :] - - coords.extend( - [ - (pressure_coord, 2), - (lat_coord, 3), - (lon_coord, 4), - ] - ) - - # If pressure doesn't exist, just use latitude/longitude in addition to time. - else: - arr = arr[np.newaxis, :, :, :] - coords.extend( - [ - (lat_coord, 2), - (lon_coord, 3), - ] - ) - - # Create cube with coordinates - out_cube = iris.cube.Cube( - arr, - dim_coords_and_dims=coords, - ) - - # Add auxcoord time coordinate that varies with forecast_period and forecast_reference_time. - time_data = base_time_units.date2num(time_coord.units.num2date(time_coord.points)) - time_data = time_data[np.newaxis, :] - out_cube.add_aux_coord( - iris.coords.AuxCoord( - time_data, - standard_name="time", - units=base_time_units, - ), - data_dims=(0, 1), - ) - - # Add metadata for long name, units, and preserve other attributes. - out_cube.rename(meta["long_name"]) - out_cube.long_name = meta["long_name"] - out_cube.units = meta["units"] - - out_cube.attributes = cube.attributes.copy() - - # Delete fill value attribute, as this tends to be np.float64(nan), which causes iris merge/concat issues. - del out_cube.attributes["fill_value"] - - # Some unit corrections for specific variables. - if out_cube.long_name == "geopotential_height_at_pressure_levels": - out_cube.data /= 9.81 - - elif out_cube.long_name == "surface_microphysical_rainfall_rate": - out_cube.data *= 1000.0 - - return out_cube - - -def ugrid_transform(arr, tri, lat_grid, lon_grid, xy): - """ - Restructure a flattened/unstructured cube. - - Parameters - ---------- - arr : arrayy - An iris cube to restructure. - tri : scipy.spatial._qhull.Delaunay - A scipy object containing the triangulation mapping of cell points. - lat_grid : np.ndarray - 1D latitude coordinate values of target grid. - lon_grid : np.ndarray - 1D longitude coordinate values of target grid. - xy : np.ndarray - Meshed and flattened target grid points. - - Returns - ------- - iris.cube.Cube - A structured iris cube with appropriate metadata. - - Notes - ----- - This function uses a pre-calculated triangulation, to save rebuilding for - every cube. This therefore assumes all cubes being restructured have the - same flattened structure. - """ - # Create empty numpy array to store regridded data. - out = np.empty((arr.shape[0], lat_grid.size, lon_grid.size)) - - # Extract and transpose source data values. - src_vals = arr.T - - # Build linear interpolator object mapping target triangulation to source values. - interp = LinearNDInterpolator(tri, src_vals) - - # Interpolate values onto target grid using linear interpolation. - out_flat = interp(xy) - - # Transpose, and reshape to target 2D regular lat/lon grid. - out = out_flat.T.reshape(arr.shape[0], lat_grid.size, lon_grid.size) - - return out - - -def fix_cubes(cubes): - """ - Restructure ugrid cubes and then fix metadata. - - First, fixes cube metadata names as a first fix, and then regrids, and then - finally adds metadata associated with new coordinates. - - Parameters - ---------- - cubes : iris.cube.CubeList - A cubelist containing unstructured cubes, along with cubes containing - latitude and longitude information. - - Returns - ------- - fixed_cubes: iris.cube.CubeList - A list of iris cubes, that have been restructured onto a regular grid, - with appropriate corrections to metadata. - - Notes - ----- - Currently, data is regridded to a 0.02degree rectilinear grid. This is because - there is no metada in the source file that describes the target resolution - of what it should be regridded to. - """ - # First, extract latitude and longitude coordinates - lat = cubes.extract("latitude")[0].data - lon = cubes.extract("longitude")[0].data - points = np.column_stack((lon, lat)) - - # Create output mesh, using standard grid ~2km resolution - # TODO: discussions with ML developers to include metadata so - # we don't have to guess target lat/lon resolution. - - # Regrid to UKV native grid.. - lon_grid = np.arange(lon.data.min(), lon.data.max(), 0.02) - lat_grid = np.arange(lat.data.min(), lat.data.max(), 0.02) - Lon2d, Lat2d = np.meshgrid(lon_grid, lat_grid) - - # Flatten target points - xy = np.column_stack((Lon2d.ravel(), Lat2d.ravel())) - - # Build triangulation via a dummy interpolator - tri_interp = LinearNDInterpolator(points, np.zeros(points.shape[0])) - tri = tri_interp.tri - - fixed_cubes = iris.cube.CubeList() - - # For each cube, where ndim > 1 (excluding latitude/longitude array), do regridding - # on array, and correct metadata. - for cube in cubes: - if cube.ndim > 1: - print(f"Fixing {cube.name()}") - result_arr = ugrid_transform(cube.data, tri, lat_grid, lon_grid, xy) - cube = rebuild_metadata(cube, result_arr, lat_grid, lon_grid) - if cube: - fixed_cubes.append(cube) - - return fixed_cubes.concatenate() - - -def main() -> None: - """ - Run processing on FastNetUK data. - - Process produces CSET-ready netCDF files for loading. - """ - parser = argparse.ArgumentParser(description="Process arguments.") - parser.add_argument("--inputpath", required=True, help="Path to file(s) to load.") - parser.add_argument( - "--outputpath", - type=str, - required=True, - help="Path to save final output data.", - ) - - args = parser.parse_args() - - # Get file paths - inputpath = args.inputpath - outputpath = args.outputpath + "/" - - for file in glob(inputpath): - print(f"Running script on {file}") - - # Load data and restructure. - cubes = iris.load(file) - cubes = fix_cubes(cubes) - - print("Saving restructured cubes") - iris.save(cubes, f"{outputpath}/fixed_{file.split('/')[-1]}") - print(f"Done file {file}") - - -if __name__ == "__main__": - main() From 07797e578679bb688314d6e0b03b9d206b9480be Mon Sep 17 00:00:00 2001 From: James Warner Date: Fri, 14 Aug 2026 10:35:00 +0100 Subject: [PATCH 19/28] update readme --- utils/proc_fastnetuk/README.md | 165 +++++++++++++-------------------- 1 file changed, 66 insertions(+), 99 deletions(-) diff --git a/utils/proc_fastnetuk/README.md b/utils/proc_fastnetuk/README.md index e74ae7dde..b12c0a8b8 100644 --- a/utils/proc_fastnetuk/README.md +++ b/utils/proc_fastnetuk/README.md @@ -1,40 +1,42 @@ -# fix_fastnetuk_ugrid +# proc_fastnetuk.py ## About -The script `fix_fastnetuk_ugrid.py` is a utility for converting FastNetUK inference output stored on an unstructured UGRID mesh into CSET-compatible NetCDF files. +The script `proc_fastnetuk.py` converts FastNetUK inference output into CSET-compatible NetCDF files. -The primary motivation for this tool is to allow FastNetUK machine learning forecast output to be ingested into CSET alongside other forecast systems. The source files contain limited metadata, use an unstructured grid representation, and do not follow the naming conventions expected by CSET. Note that the data is actually flattened, not truly unstructured, but without explaining metadata we have to assume it is a genuine unstructured dataset. +FastNetUK output contains limited metadata and uses variable naming conventions that do not match those expected by CSET. Although the source files are described as UGRID data, the forecast fields are stored as flattened arrays representing a regular grid. This utility reconstructs the missing metadata and reshapes the flattened fields back onto a structured latitude-longitude grid. -This script performs several preprocessing steps to make the data suitable for verification and evaluation within CSET: +The script performs the following preprocessing steps: -- Regrids unstructured UGRID data onto a regular latitude-longitude grid. -- Converts variable names to CSET/LFRic naming conventions. -- Restores appropriate units for meteorological variables. -- Creates a `forecast_period` dimension coordinate. -- Creates a scalar `forecast_reference_time` coordinate. +- Converts FastNetUK variable names to CSET/LFRic conventions. +- Rebuilds forecast metadata and coordinates. +- Reconstructs pressure-level information from variable names. +- Reshapes flattened fields onto the UKV latitude-longitude grid. +- Creates `forecast_period` and `forecast_reference_time` coordinates. - Preserves valid times as a `time` auxiliary coordinate. -- Reconstructs pressure-level metadata where present. -- Saves corrected data as CSET-ready NetCDF files. +- Applies required unit conversions. +- Saves the result as CSET-ready NetCDF files. -The script only performs metadata reconstruction and interpolation onto a structured grid. No scientific modifications are applied to the meteorological fields apart from unit conversions required. +No interpolation or scientific modification of the meteorological fields is performed other than the documented unit conversions. > [!TIP] -> The script assumes the source file contains latitude and longitude variables describing the UGRID cell locations. These are used to reconstruct a regular latitude-longitude grid. +> The script uses the grid definition stored in `ukv_mesh.nc` to reconstruct the latitude-longitude coordinates of the output data. > [!TIP] -> On standard data from FastNetUK inference, this script requires 30GB memory to run. +> Typical FastNetUK inference datasets require 30G memory due to reshaping and reconstruction of multiple variables. ## Usage -The script requires the following software to be installed: +### Requirements + +The script requires: - Python - Iris - NumPy -- SciPy +- cf-units -Run it with: +Run with: ```bash python fix_fastnetuk_ugrid.py \ @@ -44,38 +46,36 @@ python fix_fastnetuk_ugrid.py \ ### Required Arguments -- `--inputpath`: Path to one or more FastNetUK inference files. Wildcards may be used. If wildcards are used, quote the path so the shell passes the pattern to Python unchanged. -- `--outputpath`: Directory where fixed NetCDF files will be written. +- `--inputpath` - Input NetCDF file or wildcard pattern. +- `--outputpath` - Directory for processed output file. + +If wildcards are used, quote the pattern so it is passed unchanged to Python. This script assumes each forecast is stored in one single file, and can be run on multiple files each containing a forecast. ## Processing Details -### UGRID Restructuring +### Grid Reconstruction -FastNetUK inference output is stored on an unstructured mesh with latitude and longitude supplied as separate variables. +FastNetUK variables are stored as flattened arrays. -The script: +The script reconstructs the original structured grid by reshaping forecast data using the dimensions: -1. Extracts latitude and longitude point locations. -2. Builds a triangulation of the source mesh. -3. Creates a regular latitude-longitude target grid. -4. Interpolates each meteorological field onto the regular grid using linear interpolation. +```text +808 × 621 +``` -Currently a fixed grid spacing of: +Latitude and longitude coordinates are obtained from the reference UKV mesh file: ```text -0.02° +ukv_mesh.nc ``` -is used for the target grid. - -> [!NOTE] -> The target grid resolution is currently inferred because the source files do not contain metadata describing the intended structured output resolution. +No interpolation or regridding is performed. ### Metadata Reconstruction -The source files contain limited metadata, with most information encoded within variable names. +Variable metadata is reconstructed from the source variable name. -Examples include: +Examples: ```text t_850 @@ -88,29 +88,31 @@ sp The script extracts: -- Variable type -- Pressure level (where present) +- Variable identifier +- Pressure level (if present) -and reconstructs metadata required by CSET. +and rebuilds metadata required by CSET. + +Variables that cannot be matched to the internal lookup table are skipped. ### Forecast Coordinates -The script reconstructs forecast metadata using the source time coordinate. +Forecast metadata is reconstructed from the source time coordinate. -It creates: +The following coordinates are generated: -- `forecast_period` - `forecast_reference_time` +- `forecast_period` -while preserving valid times as: +Valid times are retained as: - `time` -The first time value in the source file is assumed to represent forecast lead time zero. +The first time step is assumed to represent lead time zero. ### Pressure Levels -Variables containing pressure-level information in their names are assigned a pressure dimension coordinate. +Variables containing pressure information in their name are given an explicit pressure dimension coordinate. For example: @@ -125,12 +127,11 @@ temperature_at_pressure_levels pressure = 850 hPa ``` -A length-one pressure dimension is created to allow future concatenation of multiple pressure levels. +A length-one pressure dimension is added so that multiple pressure levels can be concatenated later by Iris. ### Variable Renaming -Variables are translated to CSET/LFRic naming conventions using an internal lookup table. The original names originate -from anemoi [here](https://anemoi.readthedocs.io/projects/inference/en/latest/inference/configs/outputs.html). +Variables are translated to CSET/LFRic naming conventions using an internal lookup table. Examples include: @@ -145,6 +146,7 @@ Examples include: | sp | surface_air_pressure | | 10u | eastward_wind_at_10m | | 10v | northward_wind_at_10m | +| lsm | land_binary_mask | | 2t | temperature_at_screen_level | | 2d | dew_point_temperature_at_screen_level | | skt | grid_surface_temperature | @@ -152,72 +154,37 @@ Examples include: ### Unit Conversion -Some variables require unit adjustments before output. - -Examples include: - -- Geopotential (`m² s⁻²`) → Geopotential height (`m`) -- Accumulated precipitation (`m`) → Rainfall amount (`mm 6hr⁻¹`) - -These conversions are applied automatically where required. - -## Examples - -### 1. Process a single FastNetUK file +The following variable-specific adjustments are performed automatically. -```bash -python fix_fastnetuk_ugrid.py \ - --inputpath "/data/fastnetuk/inference.nc" \ - --outputpath "/my/output/path" -``` +#### Geopotential Height -Example output: +FastNetUK geopotential is converted to geopotential height: -```text -/my/output/path/fixed_inference.nc +```python +height = geopotential / 9.81 ``` -### 2. Process multiple files - -```bash -python fix_fastnetuk_ugrid.py \ - --inputpath "/data/fastnetuk/*.nc" \ - --outputpath "/my/output/path" -``` +#### Rainfall -All matching files will be processed and written to the output directory. +Rainfall fields are converted from metres to millimetres: -## Output Structure - -The resulting files contain: - -- Structured latitude-longitude grids -- CSET-compliant variable names -- Reconstructed units -- Forecast metadata -- Forecast period dimension -- Forecast reference time coordinate -- Valid time coordinate - -Pressure-level variables will additionally contain: - -```text -pressure +```python +rainfall *= 1000.0 ``` -as a dimension coordinate. - ## Notes -- Latitude and longitude variables must exist within the source file. -- Variables with names not present in the internal lookup table will be ignored. -- The script currently assumes a target grid spacing of approximately 0.02°. -- Linear interpolation is used to transform data from the unstructured mesh to a rectilinear grid. -- The first time value in the source file is assumed to correspond to forecast lead time zero. -- Pressure-level variables are inferred solely from the variable name. +- `ukv_mesh.nc` must be available when running the script. +- Variables not present in the lookup table are ignored. +- Pressure levels are inferred solely from variable names. +- The latitude-longitude grid is reconstructed by reshaping flattened fields and not by interpolation. +- The first valid time is assumed to be forecast lead time zero. +- Missing values are converted to `NaN` before output generation. + +--- ## Owners The following people should be contacted for queries or issues with this utility: -[jwarner8](https://github.com/jwarner8) +- [jwarner8](https://github.com/jwarner8) From 6e4ae2b7c40e0fe1f5b8dcddd51ba32f84d9f4e4 Mon Sep 17 00:00:00 2001 From: James Warner Date: Fri, 14 Aug 2026 10:55:08 +0100 Subject: [PATCH 20/28] remove readme aspects related to fill values and masking --- utils/proc_fastnetuk/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/utils/proc_fastnetuk/README.md b/utils/proc_fastnetuk/README.md index b12c0a8b8..8238c0466 100644 --- a/utils/proc_fastnetuk/README.md +++ b/utils/proc_fastnetuk/README.md @@ -4,7 +4,7 @@ The script `proc_fastnetuk.py` converts FastNetUK inference output into CSET-compatible NetCDF files. -FastNetUK output contains limited metadata and uses variable naming conventions that do not match those expected by CSET. Although the source files are described as UGRID data, the forecast fields are stored as flattened arrays representing a regular grid. This utility reconstructs the missing metadata and reshapes the flattened fields back onto a structured latitude-longitude grid. +FastNetUK output contains limited metadata and uses variable naming conventions that do not match those expected by CSET. Although the source files are described as UGRID data, the forecast fields are stored as flattened arrays representing a regular grid. This utility reshapes the flattened fields back onto a structured latitude-longitude grid. The script performs the following preprocessing steps: @@ -179,7 +179,6 @@ rainfall *= 1000.0 - Pressure levels are inferred solely from variable names. - The latitude-longitude grid is reconstructed by reshaping flattened fields and not by interpolation. - The first valid time is assumed to be forecast lead time zero. -- Missing values are converted to `NaN` before output generation. --- From ed02e61eabf013472d21a737d6f9bd66645d4e96 Mon Sep 17 00:00:00 2001 From: James Warner Date: Fri, 14 Aug 2026 10:55:33 +0100 Subject: [PATCH 21/28] remove fill value part --- utils/proc_fastnetuk/proc_fastnetuk.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/utils/proc_fastnetuk/proc_fastnetuk.py b/utils/proc_fastnetuk/proc_fastnetuk.py index 30fc6d9f7..000f6bc51 100644 --- a/utils/proc_fastnetuk/proc_fastnetuk.py +++ b/utils/proc_fastnetuk/proc_fastnetuk.py @@ -170,9 +170,6 @@ def rebuild_metadata(cube, grid): elif out_cube.long_name == "surface_microphysical_rainfall_rate": out_cube.data *= 1000.0 - # Fill any np.nan, as issues with read-only arrays in plotting. - out_cube.data = np.array(out_cube.data.filled(np.nan), copy=True) - return out_cube From 453ad00b59dcef012c70208deb33ea9c0ceb7c34 Mon Sep 17 00:00:00 2001 From: James Warner Date: Fri, 14 Aug 2026 10:56:39 +0100 Subject: [PATCH 22/28] add unit tests --- utils/proc_fastnetuk/test_proc_fastnetuk.py | 170 ++++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 utils/proc_fastnetuk/test_proc_fastnetuk.py diff --git a/utils/proc_fastnetuk/test_proc_fastnetuk.py b/utils/proc_fastnetuk/test_proc_fastnetuk.py new file mode 100644 index 000000000..a8bd63226 --- /dev/null +++ b/utils/proc_fastnetuk/test_proc_fastnetuk.py @@ -0,0 +1,170 @@ +"""Tests for the proc_fastnetuk.py script.""" + +import iris +import iris.coords as icoords +import iris.cube +import numpy as np +import pytest +from cf_units import Unit +from utils.proc_fastnetuk import proc_fastnetuk + +# Setup standard mesh variables that will be reused in tests. +NY = 808 +NX = 621 +NMESH = NY * NX + + +@pytest.fixture +def grid_cube(): + """Create a minimal UKV shaped grid cube.""" + lat = icoords.DimCoord( + np.arange(NY), + long_name="grid_latitude", + units="degrees", + ) + + lon = icoords.DimCoord( + np.arange(NX), + long_name="grid_longitude", + units="degrees", + ) + + data = np.zeros((NY, NX)) + + return iris.cube.Cube( + data, + dim_coords_and_dims=[ + (lat, 0), + (lon, 1), + ], + ) + + +@pytest.fixture +def time_coord(): + """Create a representative time coordinate.""" + return icoords.DimCoord( + np.array([0, 21600]), # 0h and +6h + standard_name="time", + units=Unit("seconds since 2024-01-01 00:00:00"), + ) + + +def make_cube(name, time_coord, value=1.0): + """Build a FastNetUK cube.""" + # Create data array. + data = np.full((2, NMESH), value) + + cube = iris.cube.Cube( + data, + long_name=name, + dim_coords_and_dims=[(time_coord, 0)], + ) + + cube.rename(name) + + cube.attributes["fill_value"] = np.nan + cube.attributes["source"] = "test" + + return cube + + +def test_unknown_variable_returns_none(grid_cube, time_coord): + """Test that unknown variable returns None. + + If a variable is not matched in the meta lookup, return None. + """ + cube = make_cube("foobar", time_coord) + + result = proc_fastnetuk.rebuild_metadata(cube, grid_cube) + + assert result is None + + +def test_invalid_name_returns_none(grid_cube, time_coord): + """Test name that does not match pattern returns None.""" + cube = make_cube("foo-bar!", time_coord) + + result = proc_fastnetuk.rebuild_metadata(cube, grid_cube) + + assert result is None + + +def test_surface_variable_metadata(grid_cube, time_coord): + """Test metadata is fixed for surface variable.""" + cube = make_cube("2t", time_coord) + + result = proc_fastnetuk.rebuild_metadata(cube, grid_cube) + + assert result is not None + assert result.long_name == "temperature_at_screen_level" + assert str(result.units) == "K" + + assert result.shape == (1, 2, NY, NX) + + assert result.coord("forecast_period").points.tolist() == [0.0, 6.0] + + +def test_pressure_variable_metadata(grid_cube, time_coord): + """Test metadata fixed for pressure level variable.""" + cube = make_cube("t_850", time_coord) + + result = proc_fastnetuk.rebuild_metadata(cube, grid_cube) + + assert result.long_name == "temperature_at_pressure_levels" + + pressure = result.coord("pressure") + + assert pressure.points[0] == 850 + assert str(pressure.units) == "hPa" + + assert result.shape == (1, 2, 1, NY, NX) + + +def test_time_auxcoord_created(grid_cube, time_coord): + """Test that corresponding time auxcoord created.""" + cube = make_cube("2t", time_coord) + + result = proc_fastnetuk.rebuild_metadata(cube, grid_cube) + + time_aux = result.coord("time") + + assert time_aux.shape == (1, 2) + + +def test_forecast_reference_time_created(grid_cube, time_coord): + """Check that forecast reference time created.""" + cube = make_cube("2t", time_coord) + + result = proc_fastnetuk.rebuild_metadata(cube, grid_cube) + + frt = result.coord("forecast_reference_time") + + assert frt.shape == (1,) + + +def test_attributes_preserved(grid_cube, time_coord): + """Check that additional attributes preserved.""" + cube = make_cube("2t", time_coord) + + result = proc_fastnetuk.rebuild_metadata(cube, grid_cube) + + assert result.attributes["source"] == "test" + + +def test_geopotential_conversion(grid_cube, time_coord): + """Check that geopotential converted to height.""" + cube = make_cube("z_500", time_coord, value=9.81) + + result = proc_fastnetuk.rebuild_metadata(cube, grid_cube) + + assert np.allclose(result.data, 1.0) + + +def test_precipitation_conversion(grid_cube, time_coord): + """Check that precipitation converted.""" + cube = make_cube("tp", time_coord, value=1.0) + + result = proc_fastnetuk.rebuild_metadata(cube, grid_cube) + + assert np.allclose(result.data, 1000.0) From 38d6cf1eaa530e3a4937bb79a22138dcf8649b59 Mon Sep 17 00:00:00 2001 From: James Warner Date: Wed, 19 Aug 2026 18:10:48 +0100 Subject: [PATCH 23/28] edits from code review --- utils/proc_fastnetuk/proc_fastnetuk.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/utils/proc_fastnetuk/proc_fastnetuk.py b/utils/proc_fastnetuk/proc_fastnetuk.py index 000f6bc51..567e12051 100644 --- a/utils/proc_fastnetuk/proc_fastnetuk.py +++ b/utils/proc_fastnetuk/proc_fastnetuk.py @@ -98,13 +98,14 @@ def rebuild_metadata(cube, grid): units="hours", ) - # Start with coordinates of just forecast_period. + # Start with coordinates of forecast_reference_time and forecast_period. coords = [ (forecast_reference_time, 0), (forecast_period, 1), ] - # Reshape cube ADD CHECK IF NOT CORRECT SIZE + # Reshape cube to standard UKV. This is hard-coded, as this script only + # supports UKV data. cube_data = cube.data.reshape(cube.shape[0], 808, 621) # If pressure exists, create additional size 1 dimension for future concatenation. @@ -160,8 +161,9 @@ def rebuild_metadata(cube, grid): out_cube.attributes = cube.attributes.copy() - # Delete fill value attribute, as this tends to be np.float64(nan), which causes iris merge/concat issues. - del out_cube.attributes["fill_value"] + # Delete fill value attribute if exists, as this tends to be np.float64(nan), which causes iris merge/concat issues. + if "fill_value" in out_cube.attributes: + del out_cube.attributes["fill_value"] # Some unit corrections for specific variables. if out_cube.long_name == "geopotential_height_at_pressure_levels": From 5a13dc0289eca2da319fc5b5779c8c4c5803ad24 Mon Sep 17 00:00:00 2001 From: James Warner Date: Wed, 19 Aug 2026 18:12:49 +0100 Subject: [PATCH 24/28] update comment --- utils/proc_fastnetuk/proc_fastnetuk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/proc_fastnetuk/proc_fastnetuk.py b/utils/proc_fastnetuk/proc_fastnetuk.py index 567e12051..db2e36521 100644 --- a/utils/proc_fastnetuk/proc_fastnetuk.py +++ b/utils/proc_fastnetuk/proc_fastnetuk.py @@ -165,7 +165,7 @@ def rebuild_metadata(cube, grid): if "fill_value" in out_cube.attributes: del out_cube.attributes["fill_value"] - # Some unit corrections for specific variables. + # Some data corrections for specific variables with certain units. if out_cube.long_name == "geopotential_height_at_pressure_levels": out_cube.data /= 9.81 From a9a756f60a2b115bffaafc38f116bed8f028e6db Mon Sep 17 00:00:00 2001 From: James Warner Date: Thu, 20 Aug 2026 12:08:47 +0100 Subject: [PATCH 25/28] make executable --- utils/proc_fastnetuk/proc_fastnetuk.py | 2 ++ 1 file changed, 2 insertions(+) mode change 100644 => 100755 utils/proc_fastnetuk/proc_fastnetuk.py diff --git a/utils/proc_fastnetuk/proc_fastnetuk.py b/utils/proc_fastnetuk/proc_fastnetuk.py old mode 100644 new mode 100755 index db2e36521..9337428c2 --- a/utils/proc_fastnetuk/proc_fastnetuk.py +++ b/utils/proc_fastnetuk/proc_fastnetuk.py @@ -1,3 +1,5 @@ +#!/usr/bin/python3 + """Fix FastNetUK inference data on UGRID with limited metadata. For more information on this script and how to use it, see the README.md From 2f86100a4739a0067a03429be3d7d008fcee0c79 Mon Sep 17 00:00:00 2001 From: James Warner Date: Thu, 20 Aug 2026 14:29:48 +0100 Subject: [PATCH 26/28] add description of grid in docstring --- utils/proc_fastnetuk/proc_fastnetuk.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/utils/proc_fastnetuk/proc_fastnetuk.py b/utils/proc_fastnetuk/proc_fastnetuk.py index 9337428c2..09f751026 100755 --- a/utils/proc_fastnetuk/proc_fastnetuk.py +++ b/utils/proc_fastnetuk/proc_fastnetuk.py @@ -49,9 +49,11 @@ def rebuild_metadata(cube, grid): Parameters ---------- - cube : iris.cube.Cube + cube: iris.cube.Cube Original unstructured source cube, used for fixing metadata. - grid::: + grid: iris.cube.Cube + An iris cube, containing latitude/longitude coordinates of the + UKV mesh. Returns ------- From 4e09f8f1192ea6eaba2e65c2349933f1de79d142 Mon Sep 17 00:00:00 2001 From: James Warner Date: Thu, 20 Aug 2026 14:35:59 +0100 Subject: [PATCH 27/28] clearer documentation on processing --- utils/proc_fastnetuk/proc_fastnetuk.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/proc_fastnetuk/proc_fastnetuk.py b/utils/proc_fastnetuk/proc_fastnetuk.py index 09f751026..9fc6e9005 100755 --- a/utils/proc_fastnetuk/proc_fastnetuk.py +++ b/utils/proc_fastnetuk/proc_fastnetuk.py @@ -172,7 +172,9 @@ def rebuild_metadata(cube, grid): # Some data corrections for specific variables with certain units. if out_cube.long_name == "geopotential_height_at_pressure_levels": out_cube.data /= 9.81 + out_cube.units = "m" + # Convert meters to mm. elif out_cube.long_name == "surface_microphysical_rainfall_rate": out_cube.data *= 1000.0 From e59b98b63683d597ef85787f121d849a8f27d796 Mon Sep 17 00:00:00 2001 From: James Warner Date: Thu, 20 Aug 2026 14:45:22 +0100 Subject: [PATCH 28/28] add additional docstring comments --- utils/proc_fastnetuk/proc_fastnetuk.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/proc_fastnetuk/proc_fastnetuk.py b/utils/proc_fastnetuk/proc_fastnetuk.py index 9fc6e9005..cd7daa8a7 100755 --- a/utils/proc_fastnetuk/proc_fastnetuk.py +++ b/utils/proc_fastnetuk/proc_fastnetuk.py @@ -183,6 +183,8 @@ def rebuild_metadata(cube, grid): def main() -> None: """ + Define and parse input and output path arguments. + Run processing on FastNetUK data. Process produces CSET-ready netCDF files for loading.