Merge develop to master

Author: shahramn

.github/workflows/ci.yml

@@ -4,10 +4,10 @@ on:
   # Trigger the workflow on push to master or develop, except tag creation
   push:
     branches:
-      - 'master'
-      - 'develop'
+      - "master"
+      - "develop"
     tags-ignore:
-      - '**'
+      - "**"
 
   # Trigger the workflow on pull request
   pull_request: ~
@@ -27,6 +27,7 @@ jobs:
     uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main
     with:
       eccodes-python: ecmwf/eccodes-python@${{ github.event.pull_request.head.sha || github.sha }}
+      codecov_upload: true
     secrets: inherit
 
   # Build downstream packages on HPC

CHANGELOG.rst

@@ -2,6 +2,11 @@
 Changelog for eccodes-python
 ============================
 
+1.7.0 (2024-02-26)
+--------------------
+- ECC-1761: Add function to extract message offsets and sizes
+- ECC-1742: Add function to clone only the meta-data of a message
+
 1.6.1 (2023-10-02)
 --------------------
 

README.rst

@@ -8,7 +8,7 @@ Features:
 
 - reads and writes GRIB 1 and 2 files,
 - reads and writes BUFR 3 and 4 files,
-- supports all modern versions of Python 3.11, 3.10, 3.9, 3.8 and PyPy3,
+- supports all modern versions of Python and PyPy3,
 - works on most *Linux* distributions and *MacOS*, the *ecCodes* C-library
   is the only system dependency,
 - PyPI package can be installed without compiling,

docs/index.rst

@@ -15,7 +15,7 @@ Features:
 
 - reads and writes GRIB 1 and 2 files,
 - reads and writes BUFR 3 and 4 files,
-- supports all modern versions of Python 3.11, 3.10, 3.9, 3.8 and PyPy3,
+- supports all modern versions of Python and PyPy3,
 - works on most *Linux* distributions and *MacOS*, the *ecCodes* C-library is the only system dependency,
 - PyPI package can be installed without compiling,
   at the cost of being twice as slow as the original *ecCodes* module,

eccodes/eccodes.py

@@ -43,6 +43,7 @@
     codes_definition_path,
     codes_dump,
     codes_extract_offsets,
+    codes_extract_offsets_sizes,
     codes_get_gaussian_latitudes,
     codes_get_library_path,
     codes_get_version_info,
@@ -245,6 +246,7 @@
     "codes_count_in_file",
     "codes_definition_path",
     "codes_extract_offsets",
+    "codes_extract_offsets_sizes",
     "codes_get_api_version",
     "codes_get_array",
     "codes_get_double_array",

gribapi/bindings.py

@@ -21,7 +21,7 @@
 
 import cffi
 
-__version__ = "1.6.1"
+__version__ = "1.7.0"
 
 LOG = logging.getLogger(__name__)
 

gribapi/eccodes.h

@@ -22,6 +22,7 @@ void codes_bufr_multi_element_constant_arrays_on(codes_context* c);
 void codes_bufr_multi_element_constant_arrays_off(codes_context* c);
 int codes_bufr_extract_headers_malloc(codes_context* c, const char* filename, codes_bufr_header** result, int* num_messages, int strict_mode);
 int codes_extract_offsets_malloc(codes_context* c, const char* filename, ProductKind product, long int** offsets, int* num_messages, int strict_mode);
+int codes_extract_offsets_sizes_malloc(codes_context* c, const char* filename, ProductKind product, long int** offsets, size_t** sizes, int* num_messages, int strict_mode);
 int codes_bufr_key_is_header(const codes_handle* h, const char* key, int* err);
 int codes_bufr_key_is_coordinate(const codes_handle* h, const char* key, int* err);
 

gribapi/errors.py

@@ -192,7 +192,7 @@ class InvalidIndexError(GribInternalError):
 
 
 class InvalidGribError(GribInternalError):
-    """Invalid grib id."""
+    """Invalid GRIB id."""
 
 
 class InvalidFileError(GribInternalError):

gribapi/grib_api.h

@@ -38,7 +38,7 @@ struct grib_values {
   int         has_value;
   int         equal;
   grib_values* next;
-} ;
+};
 
 typedef struct grib_handle    grib_handle;
 typedef struct grib_multi_handle    grib_multi_handle;
@@ -74,7 +74,8 @@ int grib_count_in_file(grib_context* c, FILE* f,int* n);
 grib_handle* grib_handle_new_from_file(grib_context* c, FILE* f, int* error);
 grib_handle* grib_handle_new_from_message_copy(grib_context* c, const void* data, size_t data_len);
 grib_handle* grib_handle_new_from_samples (grib_context* c, const char* sample_name);
-grib_handle* grib_handle_clone(const grib_handle* h)                 ;
+grib_handle* grib_handle_clone(const grib_handle* h);
+grib_handle* grib_handle_clone_headers_only(const grib_handle* h);
 int grib_handle_delete(grib_handle* h);
 grib_multi_handle* grib_multi_handle_new(grib_context* c);
 int grib_multi_handle_append(grib_handle* h,int start_section,grib_multi_handle* mh);
@@ -125,7 +126,7 @@ void grib_dump_content(const grib_handle* h, FILE* out, const char* mode, unsign
 grib_context* grib_context_get_default(void);
 void grib_context_delete(grib_context* c);
 
-void grib_gts_header_on(grib_context* c) ;
+void grib_gts_header_on(grib_context* c);
 void grib_gts_header_off(grib_context* c);
 void grib_gribex_mode_on(grib_context* c);
 void grib_gribex_mode_off(grib_context* c);

gribapi/gribapi.py

@@ -162,13 +162,13 @@ def wrapper(*args):
 def get_handle(msgid):
     h = ffi.cast("grib_handle*", msgid)
     if h == ffi.NULL:
-        raise errors.InvalidGribError(f"get_handle: Bad message ID {msgid}")
+        raise errors.NullHandleError(f"get_handle: Bad message ID {msgid}")
     return h
 
 
 def put_handle(handle):
     if handle == ffi.NULL:
-        raise errors.InvalidGribError(f"put_handle: Bad message ID {handle}")
+        raise errors.NullHandleError("put_handle: Bad message ID (handle is NULL)")
     return int(ffi.cast("size_t", handle))
 
 
@@ -752,12 +752,10 @@ def grib_iterator_next(iterid):
     lat_p = ffi.new("double*")
     lon_p = ffi.new("double*")
     value_p = ffi.new("double*")
-    err = lib.grib_iterator_next(iterh, lat_p, lon_p, value_p)
-    if err == 0:
+    retval = lib.grib_iterator_next(iterh, lat_p, lon_p, value_p)
+    if retval == 0:
+        # No more data available. End of iteration
         return []
-    elif err < 0:
-        GRIB_CHECK(err)
-        return None
     else:
         return (lat_p[0], lon_p[0], value_p[0])
 
@@ -803,8 +801,7 @@ def grib_keys_iterator_next(iterid):
     """
     kih = get_grib_keys_iterator(iterid)
     res = lib.grib_keys_iterator_next(kih)
-    if res < 0:
-        GRIB_CHECK(res)
+    # res is 0 or 1
     return res
 
 
@@ -887,8 +884,7 @@ def codes_bufr_keys_iterator_next(iterid):
     """
     bki = get_bufr_keys_iterator(iterid)
     res = lib.codes_bufr_keys_iterator_next(bki)
-    if res < 0:
-        GRIB_CHECK(res)
+    # res is 0 or 1
     return res
 
 
@@ -1121,23 +1117,29 @@ def codes_bufr_copy_data(msgid_src, msgid_dst):
 
 
 @require(msgid_src=int)
-def grib_clone(msgid_src):
+def grib_clone(msgid_src, headers_only=False):
     r"""
     @brief Create a copy of a message.
 
     Create a copy of a given message (\em msgid_src) resulting in a new
     message in memory (\em msgid_dest) identical to the original one.
+    If the headers_only option is enabled, the clone will not contain
+    the Bitmap and Data sections
 
     \b Examples: \ref grib_clone.py "grib_clone.py"
 
-    @param msgid_src   id of message to be cloned
-    @return            id of clone
+    @param msgid_src    id of message to be cloned
+    @param headers_only whether or not to clone the message with the headers only
+    @return             id of clone
     @exception CodesInternalError
     """
     h_src = get_handle(msgid_src)
-    h_dest = lib.grib_handle_clone(h_src)
+    if headers_only:
+        h_dest = lib.grib_handle_clone_headers_only(h_src)
+    else:
+        h_dest = lib.grib_handle_clone(h_src)
     if h_dest == ffi.NULL:
-        raise errors.InvalidGribError("clone failed")
+        raise errors.MessageInvalidError("clone failed")
     return put_handle(h_dest)
 
 
@@ -2381,7 +2383,7 @@ def grib_new_from_message(message):
         message = message.encode(ENC)
     h = lib.grib_handle_new_from_message_copy(ffi.NULL, message, len(message))
     if h == ffi.NULL:
-        raise errors.InvalidGribError("new_from_message failed")
+        raise errors.MessageInvalidError("new_from_message failed")
     return put_handle(h)
 
 
@@ -2555,10 +2557,10 @@ def codes_extract_offsets(filepath, product_kind, is_strict=True):
     """
     @brief Message offset extraction
 
-    @param filepath   path of input file
-    @product_kind     one of CODES_PRODUCT_GRIB, CODES_PRODUCT_BUFR, CODES_PRODUCT_ANY or CODES_PRODUCT_GTS
-    @param is_strict  if True, fail as soon as any invalid message is encountered
-    @return           a generator that yields offsets (each offset is an integer)
+    @param filepath      path of input file
+    @param product_kind  one of CODES_PRODUCT_GRIB, CODES_PRODUCT_BUFR, CODES_PRODUCT_ANY or CODES_PRODUCT_GTS
+    @param is_strict     if True, fail as soon as any invalid message is encountered
+    @return              a generator that yields offsets (as integers)
     @exception CodesInternalError
     """
     context = lib.grib_context_get_default()
@@ -2579,6 +2581,42 @@ def codes_extract_offsets(filepath, product_kind, is_strict=True):
         i += 1
 
 
+def codes_extract_offsets_sizes(filepath, product_kind, is_strict=True):
+    """
+    @brief Message offset and size extraction
+
+    @param filepath      path of input file
+    @param product_kind  one of CODES_PRODUCT_GRIB, CODES_PRODUCT_BUFR, CODES_PRODUCT_ANY or CODES_PRODUCT_GTS
+    @param is_strict     if True, fail as soon as any invalid message is encountered
+    @return              a generator that yields lists of pairs of offsets and sizes (as integers)
+    @exception CodesInternalError
+    """
+    context = lib.grib_context_get_default()
+    offsets_p = ffi.new("long int**")
+    sizes_p = ffi.new("size_t**")
+    num_message_p = ffi.new("int*")
+
+    err = lib.codes_extract_offsets_sizes_malloc(
+        context,
+        filepath.encode(ENC),
+        product_kind,
+        offsets_p,
+        sizes_p,
+        num_message_p,
+        is_strict,
+    )
+    GRIB_CHECK(err)
+
+    num_messages = num_message_p[0]
+    offsets = offsets_p[0]
+    sizes = sizes_p[0]
+
+    i = 0
+    while i < num_messages:
+        yield (offsets[i], sizes[i])
+        i += 1
+
+
 # -------------------------------
 # EXPERIMENTAL FEATURES
 # -------------------------------