Skip to content

Commit b5f4245

Browse files
wkliaowkliao
committed
add note about possible high cost of ncmpi_close
1 parent caf30ce commit b5f4245

3 files changed

Lines changed: 71 additions & 11 deletions

File tree

‎test/largefile/tst_hash_large_ndims.c‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,10 @@ int main(int argc, char** argv)
2727
{
2828
char filename[256];
2929
int i, rank, nprocs, err, nerrs=0, ncid, cmode, dimid, verbose=1;
30-
double timing[2], max_timing[2];
30+
double timing[3], max_timing[3];
31+
#ifdef PNC_MALLOC_TRACE
3132
MPI_Offset malloc_size[2], sum_size, max_size[2];
33+
#endif
3234
MPI_Info info;
3335

3436
MPI_Init(&argc, &argv);
@@ -65,6 +67,7 @@ int main(int argc, char** argv)
6567

6668
MPI_Info_free(&info);
6769

70+
#ifdef PNC_MALLOC_TRACE
6871
err = ncmpi_inq_malloc_size(&malloc_size[0]); CHECK_ERR
6972
err = ncmpi_inq_malloc_max_size(&malloc_size[1]); CHECK_ERR
7073
MPI_Reduce(&malloc_size, &max_size, 2, MPI_OFFSET, MPI_MAX, 0, MPI_COMM_WORLD);
@@ -75,6 +78,7 @@ int main(int argc, char** argv)
7578
(float)max_size[0]/1048576);
7679
}
7780
fflush(stdout);
81+
#endif
7882

7983
MPI_Barrier(MPI_COMM_WORLD);
8084
timing[0] = MPI_Wtime();
@@ -86,6 +90,7 @@ int main(int argc, char** argv)
8690
}
8791
timing[0] = MPI_Wtime() - timing[0];
8892

93+
#ifdef PNC_MALLOC_TRACE
8994
err = ncmpi_inq_malloc_size(&malloc_size[0]); CHECK_ERR
9095
err = ncmpi_inq_malloc_max_size(&malloc_size[1]); CHECK_ERR
9196
MPI_Reduce(&malloc_size, &max_size, 2, MPI_OFFSET, MPI_MAX, 0, MPI_COMM_WORLD);
@@ -96,13 +101,15 @@ int main(int argc, char** argv)
96101
(float)max_size[0]/1048576);
97102
}
98103
fflush(stdout);
104+
#endif
99105

100106
MPI_Barrier(MPI_COMM_WORLD);
101107
timing[1] = MPI_Wtime();
102108
err = ncmpi_enddef(ncid);
103109
CHECK_ERR
104110
timing[1] = MPI_Wtime() - timing[1];
105111

112+
#ifdef PNC_MALLOC_TRACE
106113
err = ncmpi_inq_malloc_size(&malloc_size[0]); CHECK_ERR
107114
err = ncmpi_inq_malloc_max_size(&malloc_size[1]); CHECK_ERR
108115
MPI_Reduce(&malloc_size, &max_size, 2, MPI_OFFSET, MPI_MAX, 0, MPI_COMM_WORLD);
@@ -117,10 +124,20 @@ int main(int argc, char** argv)
117124
printf("NetCDF file header size %lld extent %lld\n",header_size,header_extent);
118125
}
119126
fflush(stdout);
127+
#endif
120128

129+
/* Note the cost of ncmpi_close can be larger than expected when configured
130+
* with --enable-debug or --enable-profiling. This is because tracing
131+
* malloc builds a database and freeing a large number of memory pointers
132+
* involves searching and can be expensive.
133+
*/
134+
MPI_Barrier(MPI_COMM_WORLD);
135+
timing[2] = MPI_Wtime();
121136
err = ncmpi_close(ncid);
137+
timing[2] = MPI_Wtime() - timing[2];
122138
CHECK_ERR
123139

140+
#ifdef PNC_MALLOC_TRACE
124141
err = ncmpi_inq_malloc_size(&malloc_size[0]); CHECK_ERR
125142
err = ncmpi_inq_malloc_max_size(&malloc_size[1]); CHECK_ERR
126143
MPI_Reduce(&malloc_size, &max_size, 2, MPI_OFFSET, MPI_MAX, 0, MPI_COMM_WORLD);
@@ -140,10 +157,11 @@ int main(int argc, char** argv)
140157
ncmpi_inq_malloc_list();
141158
}
142159
}
160+
#endif
143161

144-
MPI_Reduce(&timing, &max_timing, 2, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
162+
MPI_Reduce(&timing, &max_timing, 3, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
145163
if (verbose && rank == 0)
146-
printf("Time ncmpi_def_dim = %.4f ncmpi_enddef = %.4f\n", max_timing[0],max_timing[1]);
164+
printf("Time ncmpi_def_dim = %.4f ncmpi_enddef = %.4f ncmpi_close = %.4f\n", max_timing[0],max_timing[1],max_timing[2]);
147165

148166
MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
149167
if (rank == 0) {

‎test/largefile/tst_hash_large_ngattrs.c‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,10 @@ int main(int argc, char** argv)
2727
{
2828
char filename[256];
2929
int i, rank, nprocs, err, nerrs=0, ncid, cmode, verbose=1;
30-
double timing[2], max_timing[2];
30+
double timing[3], max_timing[3];
31+
#ifdef PNC_MALLOC_TRACE
3132
MPI_Offset malloc_size[2], sum_size, max_size[2];
33+
#endif
3234
MPI_Info info;
3335

3436
MPI_Init(&argc, &argv);
@@ -65,6 +67,7 @@ int main(int argc, char** argv)
6567

6668
MPI_Info_free(&info);
6769

70+
#ifdef PNC_MALLOC_TRACE
6871
err = ncmpi_inq_malloc_size(&malloc_size[0]); CHECK_ERR
6972
err = ncmpi_inq_malloc_max_size(&malloc_size[1]); CHECK_ERR
7073
MPI_Reduce(&malloc_size, &max_size, 2, MPI_OFFSET, MPI_MAX, 0, MPI_COMM_WORLD);
@@ -75,6 +78,7 @@ int main(int argc, char** argv)
7578
(float)max_size[0]/1048576);
7679
}
7780
fflush(stdout);
81+
#endif
7882

7983
MPI_Barrier(MPI_COMM_WORLD);
8084
timing[0] = MPI_Wtime();
@@ -86,6 +90,7 @@ int main(int argc, char** argv)
8690
}
8791
timing[0] = MPI_Wtime() - timing[0];
8892

93+
#ifdef PNC_MALLOC_TRACE
8994
err = ncmpi_inq_malloc_size(&malloc_size[0]); CHECK_ERR
9095
err = ncmpi_inq_malloc_max_size(&malloc_size[1]); CHECK_ERR
9196
MPI_Reduce(&malloc_size, &max_size, 2, MPI_OFFSET, MPI_MAX, 0, MPI_COMM_WORLD);
@@ -96,13 +101,15 @@ int main(int argc, char** argv)
96101
(float)max_size[0]/1048576);
97102
}
98103
fflush(stdout);
104+
#endif
99105

100106
MPI_Barrier(MPI_COMM_WORLD);
101107
timing[1] = MPI_Wtime();
102108
err = ncmpi_enddef(ncid);
103109
CHECK_ERR
104110
timing[1] = MPI_Wtime() - timing[1];
105111

112+
#ifdef PNC_MALLOC_TRACE
106113
err = ncmpi_inq_malloc_size(&malloc_size[0]); CHECK_ERR
107114
err = ncmpi_inq_malloc_max_size(&malloc_size[1]); CHECK_ERR
108115
MPI_Reduce(&malloc_size, &max_size, 2, MPI_OFFSET, MPI_MAX, 0, MPI_COMM_WORLD);
@@ -117,10 +124,20 @@ int main(int argc, char** argv)
117124
printf("NetCDF file header size %lld extent %lld\n",header_size,header_extent);
118125
}
119126
fflush(stdout);
127+
#endif
120128

129+
/* Note the cost of ncmpi_close can be larger than expected when configured
130+
* with --enable-debug or --enable-profiling. This is because tracing
131+
* malloc builds a database and freeing a large number of memory pointers
132+
* involves searching and can be expensive.
133+
*/
134+
MPI_Barrier(MPI_COMM_WORLD);
135+
timing[2] = MPI_Wtime();
121136
err = ncmpi_close(ncid);
137+
timing[2] = MPI_Wtime() - timing[2];
122138
CHECK_ERR
123139

140+
#ifdef PNC_MALLOC_TRACE
124141
err = ncmpi_inq_malloc_size(&malloc_size[0]); CHECK_ERR
125142
err = ncmpi_inq_malloc_max_size(&malloc_size[1]); CHECK_ERR
126143
MPI_Reduce(&malloc_size, &max_size, 2, MPI_OFFSET, MPI_MAX, 0, MPI_COMM_WORLD);
@@ -140,10 +157,11 @@ int main(int argc, char** argv)
140157
ncmpi_inq_malloc_list();
141158
}
142159
}
160+
#endif
143161

144-
MPI_Reduce(&timing, &max_timing, 2, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
162+
MPI_Reduce(&timing, &max_timing, 3, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
145163
if (verbose && rank == 0)
146-
printf("Time ncmpi_put_att = %.4f ncmpi_enddef = %.4f\n", max_timing[0],max_timing[1]);
164+
printf("Time ncmpi_put_att = %.4f ncmpi_enddef = %.4f ncmpi_close = %.4f\n", max_timing[0],max_timing[1],max_timing[2]);
147165

148166
MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
149167
if (rank == 0) {

‎test/largefile/tst_hash_large_nvars.c‎

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,10 @@ int main(int argc, char** argv)
2727
{
2828
char filename[256];
2929
int i, rank, nprocs, err, nerrs=0, ncid, cmode, dimid, *varid, verbose=1;
30-
double timing[3], max_timing[3];
30+
double timing[4], max_timing[4];
31+
#ifdef PNC_MALLOC_TRACE
3132
MPI_Offset malloc_size[2], sum_size, max_size[2];
33+
#endif
3234
MPI_Info info;
3335

3436
MPI_Init(&argc, &argv);
@@ -59,6 +61,7 @@ int main(int argc, char** argv)
5961

6062
MPI_Info_create(&info);
6163
MPI_Info_set(info, "nc_hash_size_var", "2048");
64+
MPI_Info_set(info, "nc_hash_size_vattr", "2");
6265

6366
/* create a new file for writing ----------------------------------------*/
6467
cmode = NC_CLOBBER | NC_64BIT_DATA;
@@ -67,6 +70,7 @@ int main(int argc, char** argv)
6770

6871
MPI_Info_free(&info);
6972

73+
#ifdef PNC_MALLOC_TRACE
7074
err = ncmpi_inq_malloc_size(&malloc_size[0]); CHECK_ERR
7175
err = ncmpi_inq_malloc_max_size(&malloc_size[1]); CHECK_ERR
7276
MPI_Reduce(&malloc_size, &max_size, 2, MPI_OFFSET, MPI_MAX, 0, MPI_COMM_WORLD);
@@ -77,6 +81,7 @@ int main(int argc, char** argv)
7781
(float)max_size[0]/1048576);
7882
}
7983
fflush(stdout);
84+
#endif
8085

8186
err = ncmpi_def_dim(ncid, "dim", nprocs, &dimid);
8287
CHECK_ERR
@@ -85,12 +90,16 @@ int main(int argc, char** argv)
8590
timing[0] = MPI_Wtime();
8691
for (i=0; i<NVARS; i++) {
8792
char name[64];
88-
sprintf(name, "v%d.x%d", (i*1747)%8642+100000, (i*8313)%97531+100000);
93+
long prefix, suffix;
94+
prefix = ((long)i*1747)%8642+100000;
95+
suffix = ((long)i*8313)%97531+100000;
96+
sprintf(name, "v%ld.x%ld", prefix, suffix);
8997
err = ncmpi_def_var(ncid, name, NC_INT, 1, &dimid, &varid[i]);
9098
CHECK_ERR
9199
}
92100
timing[0] = MPI_Wtime() - timing[0];
93101

102+
#ifdef PNC_MALLOC_TRACE
94103
err = ncmpi_inq_malloc_size(&malloc_size[0]); CHECK_ERR
95104
err = ncmpi_inq_malloc_max_size(&malloc_size[1]); CHECK_ERR
96105
MPI_Reduce(&malloc_size, &max_size, 2, MPI_OFFSET, MPI_MAX, 0, MPI_COMM_WORLD);
@@ -101,6 +110,7 @@ int main(int argc, char** argv)
101110
(float)max_size[0]/1048576);
102111
}
103112
fflush(stdout);
113+
#endif
104114

105115
MPI_Barrier(MPI_COMM_WORLD);
106116
timing[1] = MPI_Wtime();
@@ -112,6 +122,7 @@ int main(int argc, char** argv)
112122
}
113123
timing[1] = MPI_Wtime() - timing[1];
114124

125+
#ifdef PNC_MALLOC_TRACE
115126
err = ncmpi_inq_malloc_size(&malloc_size[0]); CHECK_ERR
116127
err = ncmpi_inq_malloc_max_size(&malloc_size[1]); CHECK_ERR
117128
MPI_Reduce(&malloc_size, &max_size, 2, MPI_OFFSET, MPI_MAX, 0, MPI_COMM_WORLD);
@@ -122,13 +133,15 @@ int main(int argc, char** argv)
122133
(float)max_size[0]/1048576);
123134
}
124135
fflush(stdout);
136+
#endif
125137

126138
MPI_Barrier(MPI_COMM_WORLD);
127139
timing[2] = MPI_Wtime();
128140
err = ncmpi_enddef(ncid);
129141
CHECK_ERR
130142
timing[2] = MPI_Wtime() - timing[2];
131143

144+
#ifdef PNC_MALLOC_TRACE
132145
err = ncmpi_inq_malloc_size(&malloc_size[0]); CHECK_ERR
133146
err = ncmpi_inq_malloc_max_size(&malloc_size[1]); CHECK_ERR
134147
MPI_Reduce(&malloc_size, &max_size, 2, MPI_OFFSET, MPI_MAX, 0, MPI_COMM_WORLD);
@@ -143,12 +156,22 @@ int main(int argc, char** argv)
143156
printf("NetCDF file header size %lld extent %lld\n",header_size,header_extent);
144157
}
145158
fflush(stdout);
159+
#endif
146160

161+
/* Note the cost of ncmpi_close can be larger than expected when configured
162+
* with --enable-debug or --enable-profiling. This is because tracing
163+
* malloc builds a database and freeing a large number of memory pointers
164+
* involves searching and can be expensive.
165+
*/
166+
MPI_Barrier(MPI_COMM_WORLD);
167+
timing[3] = MPI_Wtime();
147168
err = ncmpi_close(ncid);
169+
timing[3] = MPI_Wtime() - timing[3];
148170
CHECK_ERR
149171

150172
free(varid);
151173

174+
#ifdef PNC_MALLOC_TRACE
152175
err = ncmpi_inq_malloc_size(&malloc_size[0]); CHECK_ERR
153176
err = ncmpi_inq_malloc_max_size(&malloc_size[1]); CHECK_ERR
154177
MPI_Reduce(&malloc_size, &max_size, 2, MPI_OFFSET, MPI_MAX, 0, MPI_COMM_WORLD);
@@ -168,11 +191,12 @@ int main(int argc, char** argv)
168191
ncmpi_inq_malloc_list();
169192
}
170193
}
194+
#endif
171195

172-
MPI_Reduce(&timing, &max_timing, 3, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
196+
MPI_Reduce(&timing, &max_timing, 4, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
173197
if (verbose && rank == 0)
174-
printf("Time ncmpi_def_var = %.4f ncmpi_put_att = %.4f ncmpi_enddef = %.4f\n",
175-
max_timing[0],max_timing[1],max_timing[2]);
198+
printf("Time ncmpi_def_var = %.4f ncmpi_put_att = %.4f ncmpi_enddef = %.4f ncmpi_close = %.4f\n",
199+
max_timing[0],max_timing[1],max_timing[2],max_timing[3]);
176200

177201
MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
178202
if (rank == 0) {

0 commit comments

Comments
 (0)