Skip to content

Commit 0157cc3

Browse files
wkliaowkliao
authored
Merge pull request #42 from wkliao/examples_rev2
simplify examples
2 parents a99831b + 184c4cf commit 0157cc3

15 files changed

Lines changed: 243 additions & 255 deletions

‎examples/collective_write.py‎

Lines changed: 23 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -31,16 +31,12 @@
3131
32 400 x 400 x 200 6.67 45.72
3232
"""
3333

34-
import sys, os, argparse, inspect
34+
import sys, os, argparse
3535
import numpy as np
3636
from mpi4py import MPI
3737
import pnetcdf
3838

39-
NDIMS = 3
40-
NUM_VARS = 10
41-
42-
def parse_help(comm):
43-
rank = comm.Get_rank()
39+
def parse_help():
4440
help_flag = "-h" in sys.argv or "--help" in sys.argv
4541
if help_flag and rank == 0:
4642
help_text = (
@@ -54,21 +50,22 @@ def parse_help(comm):
5450
print(help_text)
5551
return help_flag
5652

57-
def print_info(info_used):
58-
print("MPI hint: cb_nodes =", info_used.Get("cb_nodes"))
59-
print("MPI hint: cb_buffer_size =", info_used.Get("cb_buffer_size"))
60-
print("MPI hint: striping_factor =", info_used.Get("striping_factor"))
61-
print("MPI hint: striping_unit =", info_used.Get("striping_unit"))
53+
def pnetcdf_io(filename, file_format, length):
54+
# number of dimensions
55+
NDIMS = 3
56+
# number of variables
57+
NUM_VARS = 10
6258

63-
def pnetcdf_io(comm, filename, file_format, length):
64-
rank = comm.Get_rank()
65-
nprocs = comm.Get_size()
59+
if verbose and rank == 0:
60+
print("Number of variables = ", NUM_VARS)
61+
print("Number of dimensions = ", NDIMS)
6662

6763
starts = np.zeros(NDIMS, dtype=np.int32)
6864
counts = np.zeros(NDIMS, dtype=np.int32)
6965
gsizes = np.zeros(NDIMS, dtype=np.int32)
7066
buf = []
7167

68+
# calculate local subarray access pattern
7269
psizes = MPI.Compute_dims(nprocs, NDIMS)
7370
starts[0] = rank % psizes[0]
7471
starts[1] = (rank // psizes[1]) % psizes[1]
@@ -87,20 +84,12 @@ def pnetcdf_io(comm, filename, file_format, length):
8784
for j in range(bufsize):
8885
buf[i][j] = rank * i + 123 + j
8986

90-
comm.Barrier()
91-
write_timing = MPI.Wtime()
92-
9387
# Create the file using file clobber mode
94-
try:
95-
f = pnetcdf.File(filename = filename, \
96-
mode = 'w', \
97-
format = file_format, \
98-
comm = comm, \
99-
info = None)
100-
except OSError as e:
101-
print("Error at {}:{} ncmpi_create() file {} ({})".format(__file__,inspect.currentframe().f_back.f_lineno, filename, e))
102-
comm.Abort()
103-
exit(1)
88+
f = pnetcdf.File(filename = filename,
89+
mode = 'w',
90+
format = file_format,
91+
comm = comm,
92+
info = None)
10493

10594
# Define dimensions
10695
dims = []
@@ -127,34 +116,14 @@ def pnetcdf_io(comm, filename, file_format, length):
127116
# Close the file
128117
f.close()
129118

130-
write_timing = MPI.Wtime() - write_timing
131-
132-
# calculate write amount across all processes in total
133-
write_size = bufsize * NUM_VARS * np.dtype(np.int32).itemsize
134-
sum_write_size = comm.reduce(write_size, MPI.SUM, root=0)
135-
max_write_timing = comm.reduce(write_timing, MPI.MAX, root=0)
136-
137-
if rank == 0 and verbose:
138-
subarray_size = (bufsize * np.dtype(np.int32).itemsize) / 1048576.0
139-
print_info(info_used)
140-
print("Local array size {} x {} x {} integers, size = {:.2f} MB".format(length, length, length, subarray_size))
141-
sum_write_size /= 1048576.0
142-
print("Global array size {} x {} x {} integers, write size = {:.2f} GB".format(gsizes[0], gsizes[1], gsizes[2], sum_write_size/1024.0))
143-
144-
write_bw = sum_write_size / max_write_timing
145-
print(" procs Global array size exec(sec) write(MB/s)")
146-
print("------- ------------------ --------- -----------")
147-
print(" {:4d} {:4d} x {:4d} x {:4d} {:8.2f} {:10.2f}\n".format(nprocs, gsizes[0], gsizes[1], gsizes[2], max_write_timing, write_bw))
148-
149119

150120
if __name__ == "__main__":
151121

152-
verbose = True
153122
comm = MPI.COMM_WORLD
154123
rank = comm.Get_rank()
155124
nprocs = comm.Get_size()
156125

157-
if parse_help(comm):
126+
if parse_help():
158127
MPI.Finalize()
159128
sys.exit(1)
160129

@@ -168,25 +137,25 @@ def pnetcdf_io(comm, filename, file_format, length):
168137
parser.add_argument("-l", help="Size of each dimension of the local array\n")
169138
args = parser.parse_args()
170139

171-
file_format = None
172-
length = 10
173-
174-
if args.q: verbose = False
140+
verbose = False if args.q else True
175141

142+
file_format = None
176143
if args.k:
177-
kind_dict = {'1':None, '2':"NETCDF3_64BIT_OFFSET", '5':"NETCDF3_64BIT_DATA"}
144+
kind_dict = {'1':None, '2':"NC_64BIT_OFFSET", '5':"NC_64BIT_DATA"}
178145
file_format = kind_dict[args.k]
179146

147+
length = 10
180148
if args.l and int(args.l) > 0:
181149
length = int(args.l)
182150

183151
filename = args.dir
152+
184153
if verbose and rank == 0:
185154
print("{}: example of collective writes".format(os.path.basename(__file__)))
186155

187156
# Run I/O
188157
try:
189-
pnetcdf_io(comm, filename, file_format, length)
158+
pnetcdf_io(filename, file_format, length)
190159
except BaseException as err:
191160
print("Error: type:", type(err), str(err))
192161
raise

‎examples/create_open.py‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,12 @@ def parse_help():
3434
return help_flag
3535

3636
def pnetcdf_io(filename):
37-
if verbose and rank == 0:
38-
print("{}: example of file create and open".format(os.path.basename(__file__)))
3937

4038
# create a new file using file clobber mode, i.e. flag "-w"
41-
f = pnetcdf.File(filename=filename, mode = 'w', comm=comm, info=None)
39+
f = pnetcdf.File(filename = filename,
40+
mode = 'w',
41+
comm = comm,
42+
info = None)
4243

4344
# close the file
4445
f.close()
@@ -51,7 +52,6 @@ def pnetcdf_io(filename):
5152

5253

5354
if __name__ == "__main__":
54-
verbose = True
5555
comm = MPI.COMM_WORLD
5656
rank = comm.Get_rank()
5757
nprocs = comm.Get_size()
@@ -68,10 +68,13 @@ def pnetcdf_io(filename):
6868
parser.add_argument("-q", help="Quiet mode (reports when fail)", action="store_true")
6969
args = parser.parse_args()
7070

71-
if args.q: verbose = False
71+
verbose = False if args.q else True
7272

7373
filename = args.dir
7474

75+
if verbose and rank == 0:
76+
print("{}: example of file create and open".format(os.path.basename(__file__)))
77+
7578
try:
7679
pnetcdf_io(filename)
7780
except BaseException as err:

‎examples/fill_mode.py‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -63,10 +63,14 @@ def pnetcdf_io(filename):
6363
NX = 4
6464

6565
if verbose and rank == 0:
66-
print("{}: example of setting fill mode".format(os.path.basename(__file__)))
66+
print("Y dimension size = ", NY)
67+
print("X dimension size = ", NX)
6768

6869
# create a new file using clobber "w" mode
69-
f = pnetcdf.File(filename=filename, mode = 'w', comm=comm, info=None)
70+
f = pnetcdf.File(filename = filename,
71+
mode = 'w',
72+
comm = comm,
73+
info = None)
7074

7175
# the global array is NY * (NX * nprocs)
7276
global_ny = NY
@@ -131,8 +135,11 @@ def pnetcdf_io(filename):
131135

132136
# write to the 2nd record
133137
rec_var.put_var_all(buf, start = starts, count = counts)
138+
139+
# close file
134140
f.close()
135141

142+
136143
if __name__ == "__main__":
137144
verbose = True
138145

@@ -152,10 +159,14 @@ def pnetcdf_io(filename):
152159
parser.add_argument("-q", help="Quiet mode (reports when fail)", action="store_true")
153160

154161
args = parser.parse_args()
155-
if args.q: verbose = False
162+
163+
verbose = False if args.q else True
156164

157165
filename = args.dir
158166

167+
if verbose and rank == 0:
168+
print("{}: example of setting fill mode".format(os.path.basename(__file__)))
169+
159170
try:
160171
pnetcdf_io(filename)
161172
except BaseException as err:

‎examples/flexible_api.py‎

Lines changed: 29 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -89,14 +89,21 @@ def pnetcdf_io(filename, file_format):
8989
NY = 5
9090
NX = 5
9191
NZ = 5
92-
ghost_len = 3
9392

9493
if verbose and rank == 0:
95-
print("{}: example of using flexible APIs".format(os.path.basename(__file__)))
94+
print("Z dimension size = ", NZ)
95+
print("Y dimension size = ", NY)
96+
print("X dimension size = ", NX)
9697

98+
# number of cells at both end of each dimension
99+
ghost_len = 3
97100

98101
# Create the file
99-
f = pnetcdf.File(filename=filename, mode = 'w', format = file_format, comm=comm, info=None)
102+
f = pnetcdf.File(filename = filename,
103+
mode = 'w',
104+
format = file_format,
105+
comm = comm,
106+
info = None)
100107

101108
# Define dimensions
102109
dim_z = f.def_dim("Z", NZ*nprocs)
@@ -146,9 +153,9 @@ def pnetcdf_io(filename, file_format):
146153
buf_zy.fill(-1)
147154

148155
# read using flexible API
149-
var_zy.get_var_all(buf_zy, start = starts, \
150-
count = counts, \
151-
bufcount = 1, \
156+
var_zy.get_var_all(buf_zy, start = starts,
157+
count = counts,
158+
bufcount = 1,
152159
buftype = subarray)
153160

154161
# check contents of the get buffer
@@ -171,7 +178,10 @@ def pnetcdf_io(filename, file_format):
171178
array_of_sizes = np.array([NY + 2 * ghost_len, NX + 2 * ghost_len])
172179
array_of_subsizes = np.array([NY, NX])
173180
array_of_starts = np.array([ghost_len, ghost_len])
174-
subarray = MPI.DOUBLE.Create_subarray(array_of_sizes, array_of_subsizes, array_of_starts, order=MPI.ORDER_C)
181+
subarray = MPI.DOUBLE.Create_subarray(array_of_sizes,
182+
array_of_subsizes,
183+
array_of_starts,
184+
order=MPI.ORDER_C)
175185
subarray.Commit()
176186

177187
# initialize write user buffer
@@ -181,9 +191,9 @@ def pnetcdf_io(filename, file_format):
181191
counts = np.array([NY, NX])
182192

183193
# calling a blocking flexible write API
184-
req_id = var_yx.iput_var(buf_yx, start = starts, \
185-
count = counts, \
186-
bufcount = 1, \
194+
req_id = var_yx.iput_var(buf_yx, start = starts,
195+
count = counts,
196+
bufcount = 1,
187197
buftype = subarray)
188198

189199
# commit posted pending nonblocking requests
@@ -198,9 +208,9 @@ def pnetcdf_io(filename, file_format):
198208
buf_yx.fill(-1)
199209

200210
# calling a blocking flexible read API
201-
req_id = var_yx.iget_var(buf_yx, start = starts, \
202-
count = counts, \
203-
bufcount = 1, \
211+
req_id = var_yx.iget_var(buf_yx, start = starts,
212+
count = counts,
213+
bufcount = 1,
204214
buftype=subarray)
205215

206216
# commit posted pending nonblocking requests
@@ -227,7 +237,6 @@ def pnetcdf_io(filename, file_format):
227237

228238

229239
if __name__ == "__main__":
230-
verbose = True
231240
comm = MPI.COMM_WORLD
232241
rank = comm.Get_rank()
233242
nprocs = comm.Get_size()
@@ -245,16 +254,18 @@ def pnetcdf_io(filename, file_format):
245254
parser.add_argument("-k", help="File format: 1 for CDF-1, 2 for CDF-2, 5 for CDF-5")
246255
args = parser.parse_args()
247256

248-
file_format = None
249-
250-
if args.q: verbose = False
257+
verbose = False if args.q else True
251258

259+
file_format = None
252260
if args.k:
253-
kind_dict = {'1':None, '2':"NETCDF3_64BIT_OFFSET", '5':"NETCDF3_64BIT_DATA"}
261+
kind_dict = {'1':None, '2':"NC_64BIT_OFFSET", '5':"NC_64BIT_DATA"}
254262
file_format = kind_dict[args.k]
255263

256264
filename = args.dir
257265

266+
if verbose and rank == 0:
267+
print("{}: example of using flexible APIs".format(os.path.basename(__file__)))
268+
258269
try:
259270
pnetcdf_io(filename, file_format)
260271
except BaseException as err:

‎examples/get_info.py‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ def parse_help():
5050
print(help_text)
5151
return help_flag
5252

53+
5354
def print_info(info_used):
5455
nkeys = info_used.Get_nkeys()
5556
print("MPI File Info: nkeys =", nkeys)
@@ -60,11 +61,13 @@ def print_info(info_used):
6061

6162

6263
def pnetcdf_io(filename):
63-
if verbose and rank == 0:
64-
print("{}: example of getting MPI-IO hints".format(os.path.basename(__file__)))
6564

6665
# create a new file using clobber "w" mode
67-
f = pnetcdf.File(filename=filename, mode = 'w', file_format = "NETCDF3_64BIT_DATA", comm=comm, info=None)
66+
f = pnetcdf.File(filename=filename,
67+
mode = 'w',
68+
file_format = "NC_64BIT_DATA",
69+
comm=comm,
70+
info=None)
6871

6972
# exit the define mode
7073
f.enddef()
@@ -82,7 +85,6 @@ def pnetcdf_io(filename):
8285

8386

8487
if __name__ == "__main__":
85-
verbose = True
8688
comm = MPI.COMM_WORLD
8789
rank = comm.Get_rank()
8890
nprocs = comm.Get_size()
@@ -99,10 +101,13 @@ def pnetcdf_io(filename):
99101
parser.add_argument("-q", help="Quiet mode (reports when fail)", action="store_true")
100102
args = parser.parse_args()
101103

102-
if args.q: verbose = False
104+
verbose = False if args.q else True
103105

104106
filename = args.dir
105107

108+
if verbose and rank == 0:
109+
print("{}: example of getting MPI-IO hints".format(os.path.basename(__file__)))
110+
106111
try:
107112
pnetcdf_io(filename)
108113
except BaseException as err:

0 commit comments

Comments
 (0)