make varn APIs separated from var APIs

1. the optional arguments of varn APIs are different from var
2. the dimensionality of starts, and counts are also different

Author: wkliao

examples/README.md

@@ -93,8 +93,8 @@ be found at the beginning of each file.
 
 * [put_varn_int.py](./put_varn_int.py)
   + This example shows how to use a single call of `Variable` method
-    `put_var()` to to write a sequence of requests with arbitrary array indices
-    and lengths.
+    `put_varn()` to write a sequence of subarray requests to a variable with
+    arbitrary array indices and lengths.
 
 * [transpose.py](./transpose.py)
   + This example shows how to use `Variable` method `put_var()` to write six 3D

examples/put_varn_int.py

@@ -4,8 +4,8 @@
 #
 
 """
-This example shows how to use a single call of `Variable` method `put_var()` to
-to write a sequence of requests with arbitrary array indices and lengths.
+This example shows how to use a single call of `Variable` method `put_varn()`
+to to write a sequence of requests with arbitrary array indices and lengths.
 
 To run:
   % mpiexec -n num_process python3 put_varn_int.py [test_file_name]
@@ -153,7 +153,7 @@ def pnetcdf_io(file_name, file_format):
     w_buf = np.full(w_len, rank, dtype=np.int32)
 
     # set the buffer pointers to different offsets to the I/O buffe
-    v.put_var_all(w_buf, start = starts, count = counts, num = num_reqs)
+    v.put_varn_all(w_buf, num = num_reqs, starts = starts, counts = counts)
 
     # close the file
     f.close()
@@ -187,7 +187,7 @@ def pnetcdf_io(file_name, file_format):
     filename = args.dir
 
     if verbose and rank == 0:
-        print("{}: example of writing multiple variables in a call".format(os.path.basename(__file__)))
+        print("{}: example of writing multiple subarrays of a variable in a call".format(os.path.basename(__file__)))
 
     try:
         pnetcdf_io(filename, file_format)

src/pnetcdf/_Variable.pyx

@@ -4,13 +4,14 @@
 #
 
 """
-This example program is intended to illustrate the use of the pnetCDF python
-API. The program runs in non-blocking mode and makes a request to write a list
-of subarray of values to a variable into a netCDF variable of an opened netCDF
-file using bput_var method of `Variable` class. This method is a buffered
-version of bput_var and requires the user to attach an internal buffer of size
-equal to the sum of all requests using attach_buff method of `File` class. The
-library will internally invoke ncmpi_bput_vara and ncmpi_attach_buffer in C.
+This program test ibput_varn() method of `Variable` class, a nonblocking,
+buffered API, by making multiple calls to it to write to different NetCDF
+variables of an opened netCDF file.  Each call also writes to multiple
+subarrays of the same variable. This method is a buffered version of
+iput_varn() and requires the user to first attach an internal buffer of size
+equal to the sum of all requests using attach_buff() method of `File` class.
+The library will internally invoke ncmpi_bput_varn() and ncmpi_attach_buffer()
+in C.
 """
 
 import numpy as np
@@ -165,8 +166,8 @@ def run_test(format):
     for i in range(num_reqs):
         v = f.variables[f'data{i}']
         assert(f.inq_buff_size() - f.inq_buff_usage() > 0)
-        # post the request to write an array of values
-        req_id = v.bput_var(data, start = starts, count = counts, num = num_subarrays)
+        # post the request to write multiple subarrays
+        req_id = v.bput_varn(data, num_subarrays, starts, counts)
         # track the request ID for each write request
         req_ids.append(req_id)
 
@@ -182,7 +183,7 @@ def run_test(format):
     # post requests to write the 2nd half of variables w/o tracking req ids
     for i in range(num_reqs):
         v = f.variables[f'data{num_reqs + i}']
-        v.bput_var(data, start = starts, count = counts, num = num_subarrays)
+        v.bput_varn(data, num_subarrays, starts, counts)
 
     # commit the posted requests all at once using wait_all (collective i/o)
     f.wait_all(num = pnetcdf.NC_PUT_REQ_ALL)
@@ -218,3 +219,4 @@ def run_test(format):
         except BaseException as err:
             print("Error: type:", type(err), str(err))
             raise
+

test/tst_var_bput_varn.py

@@ -4,20 +4,17 @@
 #
 
 import pnetcdf
-from numpy.random import seed, randint
-from numpy.testing import assert_array_equal, assert_equal, assert_array_almost_equal
-import tempfile, unittest, os, random, sys
+from numpy.testing import assert_array_equal
+import unittest, os, sys
 import numpy as np
 from mpi4py import MPI
 from utils import validate_nc_file
 import io
 import argparse
 
-
 file_formats = ['NC_64BIT_DATA', 'NC_64BIT_OFFSET', None]
 file_name = "tst_var_get_varn.nc"
 
-
 comm = MPI.COMM_WORLD
 rank = comm.Get_rank()
 size = comm.Get_size()
@@ -85,7 +82,8 @@
     #         -  -  -  -  -  -  -  3  3  3
 else:
     num_reqs = 0
-#Obtain the size of returned array, needed for generating reference array
+
+# Obtain the size of returned array, needed for generating reference array
 buf_len = np.sum(np.prod(counts, axis=1))
 
 class VariablesTestCase(unittest.TestCase):
@@ -96,7 +94,9 @@ def setUp(self):
         else:
             self.file_path = file_name
         self._file_format = file_formats.pop(0)
-        f = pnetcdf.File(filename=self.file_path, mode = 'w', format=self._file_format, comm=comm, info=None)
+        f = pnetcdf.File(filename=self.file_path, mode = 'w',
+                         format=self._file_format, comm=comm, info=None)
+
         # define dimensions and variables
         f.def_dim('x',xdim)
         f.def_dim('y',ydim)
@@ -107,8 +107,8 @@ def setUp(self):
         var1[:] = data
 
         comm.Barrier()
-        assert validate_nc_file(os.environ.get('PNETCDF_DIR'), self.file_path) == 0 if os.environ.get('PNETCDF_DIR') is not None else True
-
+        if os.environ.get('PNETCDF_DIR') is not None:
+            assert validate_nc_file(os.environ.get('PNETCDF_DIR'), self.file_path) == 0
 
     def tearDown(self):
         # Remove the temporary files
@@ -120,16 +120,19 @@ def runTest(self):
         """testing variable get varn for CDF-5/CDF-2/CDF-1 file"""
         dataref = np.full((buf_len,), rank, np.float32)
         f = pnetcdf.File(self.file_path, 'r')
-        # test collective i/o get_var
+
+        # test collective i/o get_varn
         v1 = f.variables['var1']
         buff = np.empty((buf_len,), v1.dtype)
-        v1.get_var_all(buff, start = starts, count = counts, num = num_reqs)
+        v1.get_varn_all(buff, num_reqs, starts, counts)
         assert_array_equal(buff, dataref)
-        # test independent i/o get_var
+
+        # test independent i/o get_varn
         f.begin_indep()
         buff = np.empty((buf_len,), v1.dtype)
-        v1.get_var(buff, start = starts, count = counts, num = num_reqs)
+        v1.get_varn(buff, num_reqs, starts, counts)
         assert_array_equal(buff, dataref)
+
         f.close()
 
 if __name__ == '__main__':
@@ -143,3 +146,4 @@ def runTest(self):
     if not result.wasSuccessful():
         print(output.getvalue())
         sys.exit(1)
+

test/tst_var_get_varn.py

@@ -4,15 +4,15 @@
 #
 
 """
-   This example program is intended to illustrate the use of the pnetCDF python API.
-   The program runs in non-blocking mode and makes a request to read a list of subarray of
-   a netCDF variable of an opened netCDF file using iget_var method of `Variable` class. The
-   library will internally invoke ncmpi_iget_varn in C.
+This program tests iget_varn() method of `Variable` class, by making a
+nonblocking request to read a list of subarrays of a netCDF variable of an
+opened netCDF file.  The library will internally invoke ncmpi_iget_varn() in C.
 """
+
 import pnetcdf
-from numpy.random import seed, randint
-from numpy.testing import assert_array_equal, assert_equal, assert_array_almost_equal
-import tempfile, unittest, os, random, sys
+from numpy.random import seed
+from numpy.testing import assert_array_equal
+import unittest, os, random, sys
 import numpy as np
 from mpi4py import MPI
 from pnetcdf import strerror, strerrno
@@ -90,14 +90,16 @@
     #         -  -  -  -  -  -  -  3  3  3
 else:
     num_subarray_reqs = 0
+
 # obtain the buffer size of returned array
 buf_len = 0
 for i in range(num_subarray_reqs):
     w_req_len = np.prod(counts[i,:])
     buf_len += w_req_len
+
 # generate reference array for comparing at the testing phase
 dataref = np.full((buf_len,), rank, np.float32)
-# total number of iget_var requests for this test program
+# total number of subarray requests for this test program
 num_reqs = 10
 # initialize a list to store references of variable values
 v_datas = []
@@ -110,58 +112,71 @@ def setUp(self):
         else:
             self.file_path = file_name
         self._file_format = file_formats.pop(0)
-        f = pnetcdf.File(filename=self.file_path, mode = 'w', format=self._file_format, comm=comm, info=None)
+
+        f = pnetcdf.File(filename=self.file_path, mode = 'w',
+                         format=self._file_format, comm=comm, info=None)
+
         dx = f.def_dim('x',xdim)
         dy = f.def_dim('y',ydim)
 
         # define 20 netCDF variables
         for i in range(num_reqs * 2):
             v = f.def_var(f'data{i}', pnetcdf.NC_FLOAT, (dx, dy))
-        # initialize variable values
+
         f.enddef()
+
+        # initialize and write variable values
         for i in range(num_reqs * 2):
             v = f.variables[f'data{i}']
             v[:] = data
-        f.close()
-        assert validate_nc_file(os.environ.get('PNETCDF_DIR'), self.file_path) == 0 if os.environ.get('PNETCDF_DIR') is not None else True
 
+        f.close()
+        if os.environ.get('PNETCDF_DIR') is not None:
+            assert validate_nc_file(os.environ.get('PNETCDF_DIR'), self.file_path) == 0
 
 
         f = pnetcdf.File(self.file_path, 'r')
+
         # each process post 10 requests to read a list of subarrays from the variable
         req_ids = []
         v_datas.clear()
+
         for i in range(num_reqs):
             v = f.variables[f'data{i}']
             buff = np.empty(shape = buf_len, dtype = v.datatype)
             # post the request to read multiple slices (subarrays) of the variable
-            req_id = v.iget_var(buff, start = starts, count = counts, num = num_subarray_reqs)
+            req_id = v.iget_varn(buff, num_subarray_reqs, starts, counts)
             # track the reqeust ID for each read reqeust
             req_ids.append(req_id)
             # store the reference of variable values
             v_datas.append(buff)
-        f.end_indep()
+
         # commit those 10 requests to the file at once using wait_all (collective i/o)
         req_errs = [None] * num_reqs
         f.wait_all(num_reqs, req_ids, req_errs)
+
         # check request error msg for each unsuccessful requests
         for i in range(num_reqs):
             if strerrno(req_errs[i]) != "NC_NOERR":
                 print(f"Error on request {i}:",  strerror(req_errs[i]))
 
-         # post 10 requests to read an array of values for the last 10 variables w/o tracking req ids
+        # post 10 requests to read an array of values for the last 10
+        # variables w/o tracking req ids
         for i in range(num_reqs, num_reqs * 2):
             v = f.variables[f'data{i}']
             buff = np.empty(buf_len, dtype = v.datatype)
             # post the request to read a list of subarrays from the variable
-            v.iget_var(buff, start = starts, count = counts, num = num_subarray_reqs)
+            v.iget_varn(buff, num_subarray_reqs, starts, counts)
             # store the reference of variable values
             v_datas.append(buff)
 
-        # commit all pending get requests to the file at once using wait_all (collective i/o)
+        # commit all pending get requests to the file at once using wait_all
+        # (collective i/o)
         req_errs = f.wait_all(num = pnetcdf.NC_GET_REQ_ALL)
+
         f.close()
-        assert validate_nc_file(os.environ.get('PNETCDF_DIR'), self.file_path) == 0 if os.environ.get('PNETCDF_DIR') is not None else True
+        if os.environ.get('PNETCDF_DIR') is not None:
+            assert validate_nc_file(os.environ.get('PNETCDF_DIR'), self.file_path) == 0
 
 
     def tearDown(self):
@@ -188,3 +203,4 @@ def runTest(self):
     if not result.wasSuccessful():
         print(output.getvalue())
         sys.exit(1)
+

test/tst_var_iget_varn.py

@@ -4,22 +4,19 @@
 #
 
 """
-   This example program is intended to illustrate the use of the pnetCDF python API. The program
-   runs in non-blocking mode and makes a request to write a list of subarray of values to a variable
-   into a netCDF variable of an opened netCDF file using iput_var method of `Variable` class. The
-   library will internally invoke ncmpi_iput_varn in C.
+This program tests nonblocking method, iput_varn() of `Variable` class, by
+making a write request consisting of multiple subarrays to a variable of an
+opened netCDF file.  The library will internally invoke ncmpi_iput_varn() in C.
 """
 import pnetcdf
-from numpy.random import seed, randint
-from numpy.testing import assert_array_equal, assert_equal, assert_array_almost_equal
-import tempfile, unittest, os, random, sys
+from numpy.testing import assert_array_equal
+import unittest, os, sys
 import numpy as np
 from mpi4py import MPI
 from pnetcdf import strerror, strerrno
 from utils import validate_nc_file
 import io
 
-seed(0)
 file_formats = ['NC_64BIT_DATA', 'NC_64BIT_OFFSET', None]
 file_name = "tst_var_iput_varn.nc"
 
@@ -115,7 +112,10 @@ def setUp(self):
         else:
             self.file_path = file_name
         self._file_format = file_formats.pop(0)
-        f = pnetcdf.File(filename=self.file_path, mode = 'w', format=self._file_format, comm=comm, info=None)
+
+        f = pnetcdf.File(filename=self.file_path, mode = 'w',
+                         format=self._file_format, comm=comm, info=None)
+
         dx = f.def_dim('x',xdim)
         dy = f.def_dim('y',ydim)
 
@@ -136,11 +136,12 @@ def setUp(self):
         for i in range(num_reqs):
             v = f.variables[f'data{i}']
             # post the request to write an array of values
-            req_id = v.iput_var(data, start = starts, count = counts, num = num_subarrays)
+            req_id = v.iput_varn(data, num_subarrays, starts, counts)
             # track the reqeust ID for each write reqeust
             req_ids.append(req_id)
 
-        # all processes commit those 10 requests to the file at once using wait_all (collective i/o)
+        # all processes commit those 10 requests to the file at once using
+        # wait_all (collective i/o)
         req_errs = [None] * num_reqs
         f.wait_all(num_reqs, req_ids, req_errs)
 
@@ -149,13 +150,15 @@ def setUp(self):
             if strerrno(req_errs[i]) != "NC_NOERR":
                 print(f"Error on request {i}:",  strerror(req_errs[i]))
 
-        # post 10 requests to write an array of values for the last 10 variables w/o tracking req ids
+        # post 10 requests to write an array of values for the last 10
+        # variables w/o tracking req ids
         for i in range(num_reqs):
             v = f.variables[f'data{num_reqs + i}']
             # post the request to write an array of values
-            v.iput_var(data, start = starts, count = counts, num = num_subarrays)
+            v.iput_varn(data, num_subarrays, starts, counts)
 
-        # all processes commit all pending requests to the file at once using wait_all (collective i/o)
+        # all processes commit all pending requests to the file at once using
+        # wait_all (collective I/O)
         f.wait_all(num = pnetcdf.NC_PUT_REQ_ALL)
         f.close()
 
@@ -186,3 +189,4 @@ def runTest(self):
     if not result.wasSuccessful():
         print(output.getvalue())
         sys.exit(1)
+