bes  Updated for version 3.20.8
HDF5CF.cc
Go to the documentation of this file.
1 // This file is part of the hdf5_handler implementing for the CF-compliant
2 // Copyright (c) 2011-2016 The HDF Group, Inc. and OPeNDAP, Inc.
3 //
4 // This is free software; you can redistribute it and/or modify it under the
5 // terms of the GNU Lesser General Public License as published by the Free
6 // Software Foundation; either version 2.1 of the License, or (at your
7 // option) any later version.
8 //
9 // This software is distributed in the hope that it will be useful, but
10 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
11 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
12 // License for more details.
13 //
14 // You should have received a copy of the GNU Lesser General Public
15 // License along with this library; if not, write to the Free Software
16 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 //
18 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
19 // You can contact The HDF Group, Inc. at 1800 South Oak Street,
20 // Suite 203, Champaign, IL 61820
21 
36 
37 #include <sstream>
38 #include <algorithm>
39 #include <functional>
40 #include <climits>
41 #include "HDF5CF.h"
42 #include "h5cfdaputil.h"
43 #include "HDF5RequestHandler.h"
44 #include "BESDebug.h"
45 
46 using namespace HDF5CF;
47 
48 Var::Var(Var *var)
49 {
50 
51  newname = var->newname;
52  name = var->name;
53  fullpath = var->fullpath;
54  rank = var->rank;
55  total_elems = var->total_elems;
56  zero_storage_size = var->zero_storage_size;
57  dtype = var->dtype;
58  comp_ratio = var->comp_ratio;
59  unsupported_attr_dtype = var->unsupported_attr_dtype;
60  unsupported_attr_dspace = var->unsupported_attr_dspace;
61  unsupported_dspace = var->unsupported_dspace;
62  unsupported_attr_dspace = var->unsupported_attr_dspace;
63  dimnameflag = var->dimnameflag;
64  coord_attr_add_path = var->coord_attr_add_path;
65 
66  for (vector<Attribute*>::iterator ira = var->attrs.begin(); ira != var->attrs.end(); ++ira) {
67  Attribute* attr = new Attribute();
68  attr->name = (*ira)->name;
69  attr->newname = (*ira)->newname;
70  attr->dtype = (*ira)->dtype;
71  attr->count = (*ira)->count;
72  attr->strsize = (*ira)->strsize;
73  attr->fstrsize = (*ira)->fstrsize;
74  attr->value = (*ira)->value;
75  attrs.push_back(attr);
76  }
77 
78  for (vector<Dimension*>::iterator ird = var->dims.begin(); ird != var->dims.end(); ++ird) {
79  Dimension *dim = new Dimension((*ird)->size);
80  dim->name = (*ird)->name;
81  dim->newname = (*ird)->newname;
82  dim->unlimited_dim = (*ird)->unlimited_dim;
83  dims.push_back(dim);
84  }
85 
86 }
87 
88 bool CVar::isLatLon() const
89 {
90 
91  bool ret_value = false;
92  if (CV_EXIST == this->cvartype || CV_MODIFY == this->cvartype || CV_SPECIAL == this->cvartype) {
93  string attr_name = "units";
94  string lat_unit_value = "degrees_north";
95  string lon_unit_value = "degrees_east";
96 
97  for (vector<Attribute *>::const_iterator ira = this->attrs.begin(); ira != this->attrs.end(); ira++) {
98 
99  if ((H5FSTRING == (*ira)->getType()) || (H5VSTRING == (*ira)->getType())) {
100  if (attr_name == (*ira)->newname) {
101  string attr_value1((*ira)->getValue().begin(), (*ira)->getValue().end());
102 
103  if ((*ira)->getCount() == 1) {
104  string attr_value((*ira)->getValue().begin(), (*ira)->getValue().end());
105  if (attr_value.compare(0, lat_unit_value.size(), lat_unit_value) == 0) {
106  if (attr_value.size() == lat_unit_value.size()) {
107  ret_value = true;
108  break;
109  }
110  else if (attr_value.size() == (lat_unit_value.size() + 1)) {
111  if (attr_value[attr_value.size() - 1] == '\0'
112  || attr_value[attr_value.size() - 1] == ' ') {
113  ret_value = true;
114  break;
115  }
116  }
117  }
118  else if (attr_value.compare(0, lon_unit_value.size(), lon_unit_value) == 0) {
119  if (attr_value.size() == lon_unit_value.size()) {
120  ret_value = true;
121  break;
122  }
123  else if (attr_value.size() == (lon_unit_value.size() + 1)) {
124  if (attr_value[attr_value.size() - 1] == '\0'
125  || attr_value[attr_value.size() - 1] == ' ') {
126  ret_value = true;
127  break;
128  }
129  }
130 
131  }
132 
133  }
134  }
135  }
136  }
137  }
138  else if (this->cvartype == CV_LAT_MISS || this->cvartype == CV_LON_MISS) ret_value = true;
139  return ret_value;
140 
141 }
142 File::~File()
143 {
144 
145  if (this->fileid >= 0) {
146  if (this->rootid >= 0) {
147  for_each(this->groups.begin(), this->groups.end(), delete_elem());
148  for_each(this->vars.begin(), this->vars.end(), delete_elem());
149  for_each(this->root_attrs.begin(), this->root_attrs.end(), delete_elem());
150  H5Gclose(rootid);
151  }
152  }
153 }
154 
155 Group::~Group()
156 {
157  for_each(this->attrs.begin(), this->attrs.end(), delete_elem());
158 }
159 
160 Var::~Var()
161 {
162  for_each(this->dims.begin(), this->dims.end(), delete_elem());
163  for_each(this->attrs.begin(), this->attrs.end(), delete_elem());
164 }
165 
166 Attribute::~Attribute()
167 {
168 }
169 
170 void File::Retrieve_H5_Info(const char * /*path*/, hid_t file_id, bool include_attr)
171 {
172 
173  BESDEBUG("h5", "coming to Retrieve_H5_Info" <<endl);
174 
175  if (true == include_attr) {
176 
177  // Obtain the BES key to check the ignored objects
178  // We will only use DAS to output these information.
179  this->check_ignored = HDF5RequestHandler::get_check_ignore_obj();
180  if (true == this->check_ignored) this->add_ignored_info_page_header();
181 
182  }
183 
184  hid_t root_id;
185  if ((root_id = H5Gopen(file_id, "/", H5P_DEFAULT)) < 0) {
186  throw1("Cannot open the HDF5 root group ");
187  }
188  this->rootid = root_id;
189  try {
190  this->Retrieve_H5_Obj(root_id, "/", include_attr);
191  }
192  catch (...) {
193  throw;
194  }
195 
196  // Obtain attributes only necessary
197  if (true == include_attr) {
198 
199  // Find the file(root group) attribute
200 
201  // Obtain the object type, such as group or dataset.
202  H5O_info_t oinfo;
203  int num_attrs = 0;
204 
205  if (H5Oget_info(root_id, &oinfo) < 0)
206  throw1("Error obtaining the info for the root group");
207 
208  num_attrs = oinfo.num_attrs;
209  bool temp_unsup_attr_atype = false;
210  bool temp_unsup_attr_dspace = false;
211 
212  for (int j = 0; j < num_attrs; j++) {
213  Attribute * attr = new Attribute();
214  try {
215  this->Retrieve_H5_Attr_Info(attr, root_id, j, temp_unsup_attr_atype, temp_unsup_attr_dspace);
216  }
217  catch (...) {
218  delete attr;
219  throw;
220 
221  }
222  this->root_attrs.push_back(attr);
223  }
224 
225  this->unsupported_attr_dtype = temp_unsup_attr_atype;
226  this->unsupported_attr_dspace = temp_unsup_attr_dspace;
227  }
228 }
229 
230 void File::Retrieve_H5_Obj(hid_t grp_id, const char*gname, bool include_attr)
231 {
232 
233  // Iterate through the file to see the members of the group from the root.
234  H5G_info_t g_info;
235  hsize_t nelems = 0;
236 
237  if (H5Gget_info(grp_id, &g_info) < 0)
238  throw2("Counting hdf5 group elements error for ", gname);
239  nelems = g_info.nlinks;
240 
241  ssize_t oname_size = 0;
242  for (hsize_t i = 0; i < nelems; i++) {
243 
244  hid_t cgroup = -1;
245  hid_t cdset = -1;
246  Group *group = NULL;
247  Var *var = NULL;
248  Attribute *attr = NULL;
249 
250  try {
251 
252  size_t dummy_name_len = 1;
253 
254  // Query the length of object name.
255  oname_size = H5Lget_name_by_idx(grp_id, ".", H5_INDEX_NAME, H5_ITER_NATIVE, i, NULL, dummy_name_len,
256  H5P_DEFAULT);
257  if (oname_size <= 0)
258  throw2("Error getting the size of the hdf5 object from the group: ", gname);
259 
260  // Obtain the name of the object
261  vector<char> oname;
262  oname.resize((size_t) oname_size + 1);
263 
264  if (H5Lget_name_by_idx(grp_id, ".", H5_INDEX_NAME, H5_ITER_NATIVE, i, &oname[0], (size_t) (oname_size + 1),
265  H5P_DEFAULT) < 0)
266  throw2("Error getting the hdf5 object name from the group: ", gname);
267 
268  // Check if it is a hard link or a soft link
269  H5L_info_t linfo;
270  if (H5Lget_info(grp_id, &oname[0], &linfo, H5P_DEFAULT) < 0)
271  throw2("HDF5 link name error from ", gname);
272 
273  // We ignore soft links and external links for the CF options
274  if (H5L_TYPE_SOFT == linfo.type || H5L_TYPE_EXTERNAL == linfo.type) {
275  if (true == include_attr && true == check_ignored) {
276  this->add_ignored_info_links_header();
277  string full_path_name;
278  string temp_oname(oname.begin(), oname.end());
279  full_path_name = (
280  (string(gname) != "/") ?
281  (string(gname) + "/" + temp_oname.substr(0, temp_oname.size() - 1)) :
282  ("/" + temp_oname.substr(0, temp_oname.size() - 1)));
283  this->add_ignored_info_links(full_path_name);
284 
285  }
286  continue;
287  }
288 
289  // Obtain the object type, such as group or dataset.
290  H5O_info_t oinfo;
291 
292  if (H5Oget_info_by_idx(grp_id, ".", H5_INDEX_NAME, H5_ITER_NATIVE, i, &oinfo, H5P_DEFAULT) < 0)
293  throw2("Error obtaining the info for the object ", string(oname.begin(), oname.end()));
294 
295  H5O_type_t obj_type = oinfo.type;
296 
297  switch (obj_type) {
298 
299  case H5O_TYPE_GROUP: {
300 
301  // Obtain the full path name
302  string full_path_name;
303  string temp_oname(oname.begin(), oname.end());
304 
305  full_path_name = (
306  (string(gname) != "/") ?
307  (string(gname) + "/" + temp_oname.substr(0, temp_oname.size() - 1)) :
308  ("/" + temp_oname.substr(0, temp_oname.size() - 1)));
309 
310  cgroup = H5Gopen(grp_id, full_path_name.c_str(), H5P_DEFAULT);
311  if (cgroup < 0)
312  throw2("Error opening the group ", full_path_name);
313 
314  group = new Group();
315  group->path = full_path_name;
316  group->newname = full_path_name;
317 
318  // Retrieve group attribute if the attribute flag is true
319  if (true == include_attr) {
320 
321  int num_attrs = oinfo.num_attrs;
322  bool temp_unsup_attr_dtype = false;
323  bool temp_unsup_attr_dspace = false;
324 
325  for (int j = 0; j < num_attrs; j++) {
326 
327  attr = new Attribute();
328  Retrieve_H5_Attr_Info(attr, cgroup, j, temp_unsup_attr_dtype, temp_unsup_attr_dspace);
329  group->attrs.push_back(attr);
330  attr = NULL;
331  }
332 
333  group->unsupported_attr_dtype = temp_unsup_attr_dtype;
334  group->unsupported_attr_dspace = temp_unsup_attr_dspace;
335  }
336  this->groups.push_back(group);
337  Retrieve_H5_Obj(cgroup, full_path_name.c_str(), include_attr);
338  if (H5Gclose(cgroup) < 0)
339  throw2("Error closing the group ", full_path_name);
340  }
341  break;
342  case H5O_TYPE_DATASET: {
343 
344  // Obtain the absolute path of the HDF5 dataset
345  string temp_oname(oname.begin(), oname.end());
346  string full_path_name = (
347  (string(gname) != "/") ?
348  (string(gname) + "/" + temp_oname.substr(0, temp_oname.size() - 1)) :
349  ("/" + temp_oname.substr(0, temp_oname.size() - 1)));
350 
351  var = new Var();
352  var->name = temp_oname.substr(0, temp_oname.size() - 1);
353  var->fullpath = full_path_name;
354 
355  // newname is for the final CF name
356  var->newname = full_path_name;
357 
358  cdset = H5Dopen(grp_id, full_path_name.c_str(), H5P_DEFAULT);
359  if (cdset < 0)
360  throw2("Error opening the HDF5 dataset ", full_path_name);
361 
362  // Retrieve the HDF5 dataset datatype, return the flag for unsupported types.
363  bool temp_unsup_var_dtype = false;
364  Retrieve_H5_VarType(var, cdset, full_path_name, temp_unsup_var_dtype);
365 
366  // Update the unsupported datatype flag
367  if (!this->unsupported_var_dtype && temp_unsup_var_dtype) this->unsupported_var_dtype = true;
368 
369  // Retrieve the HDF5 dataset data space, return the flag for unsupported dataspaces.
370  bool temp_unsup_var_dspace = false;
371  Retrieve_H5_VarDim(var, cdset, full_path_name, temp_unsup_var_dspace);
372 
373  // Update the unsupported data space flag
374  if (!this->unsupported_var_dspace && temp_unsup_var_dspace) this->unsupported_var_dspace = true;
375 
376  hsize_t d_storage_size = H5Dget_storage_size(cdset);
377  var->zero_storage_size =(d_storage_size ==0);
378  var->comp_ratio = Retrieve_H5_VarCompRatio(var, cdset);
379 
380  // Retrieve the attribute info. if asked
381  if (true == include_attr) {
382 
383  int num_attrs = oinfo.num_attrs;
384  bool temp_unsup_attr_dtype = false;
385  bool temp_unsup_attr_dspace = false;
386 
387  for (int j = 0; j < num_attrs; j++) {
388 
389  attr = new Attribute();
390 
391  Retrieve_H5_Attr_Info(attr, cdset, j, temp_unsup_attr_dtype, temp_unsup_attr_dspace);
392  var->attrs.push_back(attr);
393  attr = NULL;
394  }
395 
396  var->unsupported_attr_dtype = temp_unsup_attr_dtype;
397  var->unsupported_attr_dspace = temp_unsup_attr_dspace;
398 
399  if (!this->unsupported_var_attr_dspace && temp_unsup_attr_dspace)
400  this->unsupported_var_attr_dspace = true;
401  }
402 
403  this->vars.push_back(var);
404  if (H5Dclose(cdset) < 0)
405  throw2("Error closing the HDF5 dataset ", full_path_name);
406  }
407  break;
408 
409  case H5O_TYPE_NAMED_DATATYPE: {
410  // ignore the named datatype
411  if (true == include_attr && true == check_ignored) {
412  this->add_ignored_info_namedtypes(string(gname), string(oname.begin(), oname.end()));
413  }
414  }
415  break;
416  default:
417  break;
418  } // "switch (obj_type)"
419  } // try
420  catch (...) {
421 
422  if (attr != NULL) {
423  delete attr;
424  attr = NULL;
425  }
426 
427  if (var != NULL) {
428  delete var;
429  var = NULL;
430  }
431 
432  if (group != NULL) {
433  delete group;
434  group = NULL;
435  }
436 
437  if (cgroup != -1) H5Gclose(cgroup);
438 
439  if (cdset != -1) H5Dclose(cdset);
440  throw;
441 
442  } // catch
443  } // "for (hsize_t i = 0; i < nelems; i++)"
444 
445 }
446 
447 // Retrieve HDF5 dataset datatype
448 float File::Retrieve_H5_VarCompRatio(Var *var, hid_t dset_id)
449 {
450 
451  float comp_ratio = 1.0;
452  // Obtain the data type of the variable.
453  hid_t dset_create_plist = H5Dget_create_plist(dset_id);
454  if (dset_create_plist < 0)
455  throw1("unable to obtain hdf5 dataset creation property list ");
456  H5D_layout_t dset_layout = H5Pget_layout(dset_create_plist);
457  if (dset_layout < 0) {
458  H5Pclose(dset_create_plist);
459  throw1("unable to obtain hdf5 dataset creation property list storage layout");
460  }
461 
462  if (dset_layout == H5D_CHUNKED) {
463 
464  hsize_t dstorage_size = H5Dget_storage_size(dset_id);
465  if (dstorage_size > 0 && var->total_elems > 0) {
466  hid_t ty_id = -1;
467 
468  // Obtain the data type of the variable.
469  if ((ty_id = H5Dget_type(dset_id)) < 0)
470  throw1("unable to obtain hdf5 datatype for the dataset ");
471  size_t type_size = H5Tget_size(ty_id);
472  comp_ratio = ((float) (var->total_elems) * type_size) / dstorage_size;
473  H5Tclose(ty_id);
474  }
475 
476  }
477  H5Pclose(dset_create_plist);
478  return comp_ratio;
479 
480 }
481 // Retrieve HDF5 dataset datatype
482 void File::Retrieve_H5_VarType(Var *var, hid_t dset_id, const string & varname, bool &unsup_var_dtype)
483 {
484 
485  hid_t ty_id = -1;
486 
487  // Obtain the data type of the variable.
488  if ((ty_id = H5Dget_type(dset_id)) < 0)
489  throw2("unable to obtain hdf5 datatype for the dataset ", varname);
490 
491  // The following datatype class and datatype will not be supported for the CF option.
492  // H5T_TIME, H5T_BITFIELD
493  // H5T_OPAQUE, H5T_ENUM
494  // H5T_REFERENCE, H5T_COMPOUND
495  // H5T_VLEN,H5T_ARRAY
496  // 64-bit integer
497 
498  // Note: H5T_REFERENCE H5T_COMPOUND and H5T_ARRAY can be mapped to DAP2 DDS for the default option.
499  // H5T_COMPOUND, H5T_ARRAY can be mapped to DAP2 DAS for the default option.
500  // 1-D variable length of string can also be mapped for the CF option..
501  // The variable length string class is H5T_STRING rather than H5T_VLEN,
502  // We also ignore the mapping of integer 64 bit since DAP2 doesn't
503  // support 64-bit integer. In theory, DAP2 doesn't support long double
504  // (128-bit or 92-bit floating point type).
505  //
506 
507  var->dtype = HDF5CFUtil::H5type_to_H5DAPtype(ty_id);
508  if (false == HDF5CFUtil::cf_strict_support_type(var->dtype)) unsup_var_dtype = true;
509 
510  if (H5Tclose(ty_id) < 0)
511  throw1("Unable to close the HDF5 datatype ");;
512 }
513 
514 // Retrieve the HDF5 dataset dimension information
515 void File::Retrieve_H5_VarDim(Var *var, hid_t dset_id, const string & varname, bool &unsup_var_dspace)
516 {
517 
518  vector<hsize_t> dsize;
519  vector<hsize_t> maxsize;
520 
521  hid_t dspace_id = -1;
522  hid_t ty_id = -1;
523 
524  try {
525  if ((dspace_id = H5Dget_space(dset_id)) < 0)
526  throw2("Cannot get hdf5 dataspace id for the variable ", varname);
527 
528  H5S_class_t space_class = H5S_NO_CLASS;
529  if ((space_class = H5Sget_simple_extent_type(dspace_id)) < 0)
530  throw2("Cannot obtain the HDF5 dataspace class for the variable ", varname);
531 
532  if (H5S_NULL == space_class)
533  unsup_var_dspace = true;
534  else {
535  if (false == unsup_var_dspace) {
536 
537  hssize_t h5_total_elms = H5Sget_simple_extent_npoints(dspace_id);
538  if (h5_total_elms < 0)
539  throw2("Cannot get the total number of elements of HDF5 dataset ", varname);
540  else
541  var->total_elems = (size_t) h5_total_elms;
542  int ndims = H5Sget_simple_extent_ndims(dspace_id);
543  if (ndims < 0)
544  throw2("Cannot get the hdf5 dataspace number of dimension for the variable ", varname);
545 
546  var->rank = ndims;
547  if (ndims != 0) {
548  dsize.resize(ndims);
549  maxsize.resize(ndims);
550  }
551 
552  // The netcdf DAP client supports the representation of the unlimited dimension.
553  // So we need to check.
554  if (H5Sget_simple_extent_dims(dspace_id, &dsize[0], &maxsize[0]) < 0)
555  throw2("Cannot obtain the dim. info for the variable ", varname);
556 
557  for (int i = 0; i < ndims; i++) {
558  Dimension * dim = new Dimension(dsize[i]);
559  if (maxsize[i] == H5S_UNLIMITED) {
560  dim->unlimited_dim = true;
561  if (false == have_udim) have_udim = true;
562  }
563  var->dims.push_back(dim);
564  }
565  }
566  }
567 
568  var->unsupported_dspace = unsup_var_dspace;
569 
570  if (H5Sclose(dspace_id) < 0)
571  throw1("Cannot close the HDF5 dataspace .");
572 
573  }
574 
575  catch (...) {
576 
577  if (dspace_id != -1) H5Sclose(dspace_id);
578 
579  if (ty_id != -1) H5Tclose(ty_id);
580  throw;
581  }
582 
583 }
584 
585 // Retrieve the HDF5 attribute information.
586 void File::Retrieve_H5_Attr_Info(Attribute * attr, hid_t obj_id, const int j, bool &unsup_attr_dtype,
587  bool &unsup_attr_dspace)
588 
589 {
590 
591  hid_t attrid = -1;
592  hid_t ty_id = -1;
593  hid_t aspace_id = -1;
594  hid_t memtype = -1;
595 
596  try {
597 
598  // Obtain the attribute ID.
599  if ((attrid = H5Aopen_by_idx(obj_id, ".", H5_INDEX_CRT_ORDER, H5_ITER_INC, (hsize_t) j, H5P_DEFAULT,
600  H5P_DEFAULT)) < 0)
601  throw1("Unable to open attribute by index ");
602 
603  // Obtain the size of attribute name.
604  ssize_t name_size = H5Aget_name(attrid, 0, NULL);
605  if (name_size < 0)
606  throw1("Unable to obtain the size of the hdf5 attribute name ");
607 
608  string attr_name;
609  attr_name.resize(name_size + 1);
610 
611  // Obtain the attribute name.
612  if ((H5Aget_name(attrid, name_size + 1, &attr_name[0])) < 0)
613  throw1("unable to obtain the hdf5 attribute name ");
614 
615  // Obtain the type of the attribute.
616  if ((ty_id = H5Aget_type(attrid)) < 0)
617  throw2("unable to obtain hdf5 datatype for the attribute ", attr_name);
618 
619  // The following datatype class and datatype will not be supported for the CF option.
620  // H5T_TIME, H5T_BITFIELD
621  // H5T_OPAQUE, H5T_ENUM
622  // H5T_REFERENCE, H5T_COMPOUND
623  // H5T_VLEN,H5T_ARRAY
624  // 64-bit integer
625 
626  // Note: H5T_REFERENCE H5T_COMPOUND and H5T_ARRAY can be mapped to DAP2 DDS for the default option.
627  // H5T_COMPOUND, H5T_ARRAY can be mapped to DAP2 DAS for the default option.
628  // 1-D variable length of string can also be mapped for the CF option..
629  // The variable length string class is H5T_STRING rather than H5T_VLEN,
630  // We also ignore the mapping of integer 64 bit since DAP2 doesn't
631  // support 64-bit integer. In theory, DAP2 doesn't support long double
632  // (128-bit or 92-bit floating point type).
633  //
634  attr->dtype = HDF5CFUtil::H5type_to_H5DAPtype(ty_id);
635  if (false == HDF5CFUtil::cf_strict_support_type(attr->dtype)) unsup_attr_dtype = true;
636 
637  if(H5VSTRING == attr->dtype || H5FSTRING == attr->dtype) {
638  H5T_cset_t c_set_type = H5Tget_cset(ty_id);
639  if(c_set_type <0)
640  throw2("Cannot get hdf5 character set type for the attribute ", attr_name);
641  // This is a UTF-8 string
642  if(c_set_type == 1)
643  attr->is_cset_ascii = false;
644  }
645 
646  if ((aspace_id = H5Aget_space(attrid)) < 0)
647  throw2("Cannot get hdf5 dataspace id for the attribute ", attr_name);
648 
649  int ndims = H5Sget_simple_extent_ndims(aspace_id);
650  if (ndims < 0)
651  throw2("Cannot get the hdf5 dataspace number of dimension for attribute ", attr_name);
652 
653  hsize_t nelmts = 1;
654 
655  // if it is a scalar attribute, just define number of elements to be 1.
656  if (ndims != 0) {
657 
658  vector<hsize_t> asize;
659  vector<hsize_t> maxsize;
660  asize.resize(ndims);
661  maxsize.resize(ndims);
662 
663  // Obtain the attribute data space information.
664  if (H5Sget_simple_extent_dims(aspace_id, &asize[0], &maxsize[0]) < 0)
665  throw2("Cannot obtain the dim. info for the attribute ", attr_name);
666 
667  // Here we need to take care of 0-length attribute. This is legal in HDF5.
668  for (int dim_count = 0;dim_count < ndims; dim_count ++) {
669  // STOP adding unsupported_attr_dspace!
670  if (asize[dim_count] == 0) {
671  unsup_attr_dspace = true;
672  break;
673  }
674  }
675 
676  if (false == unsup_attr_dspace) {
677  // Return ndims and size[ndims].
678  for (int dim_count = 0; dim_count< ndims; dim_count++)
679  nelmts *= asize[dim_count];
680  }
681  else
682  nelmts = 0;
683  } // "if(ndims != 0)"
684 
685  size_t ty_size = H5Tget_size(ty_id);
686  if (0 == ty_size)
687  throw2("Cannot obtain the dtype size for the attribute ", attr_name);
688 
689  memtype = H5Tget_native_type(ty_id, H5T_DIR_ASCEND);
690  if (memtype < 0)
691  throw2("Cannot obtain the memory datatype for the attribute ", attr_name);
692 
693  // Store the name and the count
694  string temp_aname(attr_name.begin(), attr_name.end());
695  attr->name = temp_aname.substr(0, temp_aname.size() - 1);
696  attr->newname = attr->name;
697  attr->count = nelmts;
698 
699  // Release HDF5 resources.
700  if (H5Tclose(ty_id) < 0)
701  throw1("Cannot successfully close the attribute datatype.");
702  if (H5Tclose(memtype) < 0)
703  throw1("Cannot successfully close the attribute memory datatype.");
704  if (H5Sclose(aspace_id) < 0)
705  throw1("Cannot successfully close the HDF5 dataspace.");
706  if (H5Aclose(attrid) < 0)
707  throw1("Cannot successfully close the HDF5 attribute.");
708 
709  } // try
710  catch (...) {
711 
712  if (ty_id != -1) H5Tclose(ty_id);
713 
714  if (memtype != -1) H5Tclose(memtype);
715 
716  if (aspace_id != -1) H5Sclose(aspace_id);
717 
718  if (attrid != -1) H5Aclose(attrid);
719 
720  throw;
721  }
722 
723 }
724 
725 // Retrieve all HDF5 supported attribute values.
727 {
728 
729  for (vector<Attribute *>::iterator ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira)
730  Retrieve_H5_Attr_Value(*ira, "/");
731 
732  for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
733  for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end(); ++ira) {
734  Retrieve_H5_Attr_Value(*ira, (*irg)->path);
735  }
736  }
737 
738  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
739  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
740  Retrieve_H5_Attr_Value(*ira, (*irv)->fullpath);
741  }
742  }
743 }
744 
746 {
747  for (vector<Attribute *>::iterator ira = var->attrs.begin(); ira != var->attrs.end(); ++ira) {
748  Retrieve_H5_Attr_Value(*ira, var->fullpath);
749  }
750 }
751 
752 // Retrieve the values of a specific HDF5 attribute.
753 void File::Retrieve_H5_Attr_Value(Attribute *attr, string obj_name)
754 {
755 
756  // Define HDF5 object Ids.
757  hid_t obj_id = -1;
758  hid_t attr_id = -1;
759  hid_t ty_id = -1;
760  hid_t memtype_id = -1;
761  hid_t aspace_id = -1;
762 
763  try {
764 
765  // Open the object that hold this attribute
766  obj_id = H5Oopen(this->fileid, obj_name.c_str(), H5P_DEFAULT);
767  if (obj_id < 0)
768  throw2("Cannot open the object ", obj_name);
769 
770  attr_id = H5Aopen(obj_id, (attr->name).c_str(), H5P_DEFAULT);
771  if (attr_id < 0)
772  throw4("Cannot open the attribute ", attr->name, " of object ", obj_name);
773 
774  ty_id = H5Aget_type(attr_id);
775  if (ty_id < 0)
776  throw4("Cannot obtain the datatype of the attribute ", attr->name, " of object ", obj_name);
777 
778  memtype_id = H5Tget_native_type(ty_id, H5T_DIR_ASCEND);
779  if (memtype_id < 0)
780  throw2("Cannot obtain the memory datatype for the attribute ", attr->name);
781 
782  size_t ty_size = H5Tget_size(memtype_id);
783  if (0 == ty_size)
784  throw4("Cannot obtain the dtype size for the attribute ", attr->name, " of object ", obj_name);
785 
786  size_t total_bytes = attr->count * ty_size;
787 
788  // We have to handle variable length string differently.
789  if (H5VSTRING == attr->dtype) {
790 
791  // Variable length string attribute values only store pointers of the actual string value.
792  vector<char> temp_buf;
793  temp_buf.resize(total_bytes);
794 
795  if (H5Aread(attr_id, memtype_id, &temp_buf[0]) < 0)
796  throw4("Cannot obtain the value of the attribute ", attr->name, " of object ", obj_name);
797 
798  char *temp_bp = NULL;
799  char *ptr_1stvlen_ptr = &temp_buf[0];
800  temp_bp = &temp_buf[0];
801  char* onestring = NULL;
802  string total_vstring = "";
803 
804  attr->strsize.resize(attr->count);
805 
806  for (unsigned int temp_i = 0; temp_i < attr->count; temp_i++) {
807 
808  // This line will assure that we get the real variable length string value.
809  onestring = *(char **) temp_bp;
810  if (onestring != NULL) {
811  total_vstring += string(onestring);
812  attr->strsize[temp_i] = (string(onestring)).size();
813  }
814  else
815  attr->strsize[temp_i] = 0;
816 
817  // going to the next value.
818  temp_bp += ty_size;
819  }
820 
821  if (ptr_1stvlen_ptr != NULL) {
822  aspace_id = H5Aget_space(attr_id);
823  if (aspace_id < 0)
824  throw4("Cannot obtain space id for ", attr->name, " of object ", obj_name);
825 
826  // Reclaim any VL memory if necessary.
827  if (H5Dvlen_reclaim(memtype_id, aspace_id, H5P_DEFAULT, &temp_buf[0]) < 0)
828  throw4("Cannot reclaim VL memory for ", attr->name, " of object ", obj_name);
829 
830  H5Sclose(aspace_id);
831  }
832 
833  if (HDF5CFUtil::H5type_to_H5DAPtype(ty_id) != H5VSTRING)
834  throw4("Error to obtain the VL string type for attribute ", attr->name, " of object ", obj_name);
835 
836  attr->value.resize(total_vstring.size());
837 
838  copy(total_vstring.begin(), total_vstring.end(), attr->value.begin());
839 
840  }
841  else {
842 
843  if (attr->dtype == H5FSTRING) {
844  attr->fstrsize = ty_size;
845  }
846 
847  attr->value.resize(total_bytes);
848 
849  // Read HDF5 attribute data.
850  if (H5Aread(attr_id, memtype_id, (void *) &attr->value[0]) < 0)
851  throw4("Cannot obtain the dtype size for the attribute ", attr->name, " of object ", obj_name);
852 
853  if (attr->dtype == H5FSTRING) {
854 
855  size_t sect_size = ty_size;
856  int num_sect = 1;
857  if (sect_size > 0)
858  num_sect =
859  (total_bytes % sect_size == 0) ? (total_bytes / sect_size) : (total_bytes / sect_size + 1);
860  else
861  throw4("The attribute datatype size is not a positive integer ", attr->name, " of object ",
862  obj_name);
863 
864  vector<size_t> sect_newsize;
865  sect_newsize.resize(num_sect);
866 
867  string total_fstring = string(attr->value.begin(), attr->value.end());
868 
869  string new_total_fstring = HDF5CFUtil::trim_string(memtype_id, total_fstring, num_sect, sect_size,
870  sect_newsize);
871  attr->value.resize(new_total_fstring.size());
872  copy(new_total_fstring.begin(), new_total_fstring.end(), attr->value.begin());
873  attr->strsize.resize(num_sect);
874  for (int temp_i = 0; temp_i < num_sect; temp_i++)
875  attr->strsize[temp_i] = sect_newsize[temp_i];
876 
877 #if 0
878  // "h5","new string value " <<string(attr->value.begin(), attr->value.end()) <<endl;
879  for (int temp_i = 0; temp_i <num_sect; temp_i ++)
880  "h5","string new section size = " << attr->strsize[temp_i] <<endl;
881 #endif
882  }
883  }
884 
885  if (H5Tclose(memtype_id) < 0)
886  throw1("Fail to close the HDF5 memory datatype ID.");
887  if (H5Tclose(ty_id) < 0)
888  throw1("Fail to close the HDF5 datatype ID.");
889  if (H5Aclose(attr_id) < 0)
890  throw1("Fail to close the HDF5 attribute ID.");
891  if (H5Oclose(obj_id) < 0)
892  throw1("Fail to close the HDF5 object ID.");
893 
894  }
895 
896  catch (...) {
897 
898  if (memtype_id != -1) H5Tclose(memtype_id);
899 
900  if (ty_id != -1) H5Tclose(ty_id);
901 
902  if (aspace_id != -1) H5Sclose(aspace_id);
903 
904  if (attr_id != -1) H5Aclose(attr_id);
905 
906  if (obj_id != -1) H5Oclose(obj_id);
907 
908  throw;
909  }
910 
911 }
912 
913 // Handle the unsupported datatype
914 void File::Handle_Unsupported_Dtype(bool include_attr)
915 {
916 
917  if (true == include_attr) {
918  Handle_Group_Unsupported_Dtype();
919  Handle_VarAttr_Unsupported_Dtype();
920  }
921 
922  Handle_Var_Unsupported_Dtype();
923 }
924 
925 //Leave this code here.
926 #if 0
927 // First the root attributes
928 if (true == include_attr) {
929  if (true == this->unsupported_attr_dtype) {
930  for (vector<Attribute *>::iterator ira = this->root_attrs.begin();
931  ira != this->root_attrs.end(); ) {
932  H5DataType temp_dtype = (*ira)->getType();
933  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
934  delete (*ira);
935  ira = this->root_attrs.erase(ira);
936 
937  }
938  else {
939  ++ira;
940  }
941  }
942  }
943 }
944 
945 // Then the group attributes
946 if (false == this->groups.empty()) {
947  for (vector<Group *>::iterator irg = this->groups.begin();
948  irg != this->groups.end(); ++irg) {
949  if (false == (*irg)->attrs.empty()) {
950  if (true == (*irg)->unsupported_attr_dtype) {
951  for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin();
952  ira != (*irg)->attrs.end(); ) {
953  H5DataType temp_dtype = (*ira)->getType();
954  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
955  delete (*ira);
956  ira = (*irg)->attrs.erase(ira);
957  }
958  else {
959  ++ira;
960  }
961  }
962  }
963  }
964  }
965 }
966 }
967 
968  // Then the variable(HDF5 dataset) and the correponding attributes.
969 if (false == this->vars.empty()) {
970 if (true == include_attr) {
971  for (vector<Var *>::iterator irv = this->vars.begin();
972  irv != this->vars.end();++irv ) {
973  if (false == (*irv)->attrs.empty()) {
974  if (true == (*irv)->unsupported_attr_dtype) {
975  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin();
976  ira != (*irv)->attrs.end(); ) {
977  H5DataType temp_dtype = (*ira)->getType();
978  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
979  delete (*ira);
980  ira = (*irv)->attrs.erase(ira);
981  //ira--;
982  }
983  else {
984  ++ira;
985  }
986  }
987  }
988  }
989  }
990 }
991 if (true == this->unsupported_var_dtype) {
992  // "h5","having unsupported variable datatype" <<endl;
993  for (vector<Var *>::iterator irv = this->vars.begin();
994  irv != this->vars.end(); ) {
995  H5DataType temp_dtype = (*irv)->getType();
996  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
997  delete (*irv);
998  irv = this->vars.erase(irv);
999  //irv--;
1000  }
1001  else
1002  ++irv;
1003  }
1004 }
1005 }
1006 #endif
1007 
1008 void File::Handle_Group_Unsupported_Dtype()
1009 {
1010 
1011  // First root
1012  if (false == this->root_attrs.empty()) {
1013  if (true == this->unsupported_attr_dtype) {
1014  for (vector<Attribute *>::iterator ira = this->root_attrs.begin(); ira != this->root_attrs.end();) {
1015  H5DataType temp_dtype = (*ira)->getType();
1016  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
1017  delete (*ira);
1018  ira = this->root_attrs.erase(ira);
1019  }
1020  else {
1021  ++ira;
1022  }
1023  }
1024  }
1025  }
1026 
1027  // Then the group attributes
1028  if (false == this->groups.empty()) {
1029  for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1030  if (false == (*irg)->attrs.empty()) {
1031  if (true == (*irg)->unsupported_attr_dtype) {
1032  for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end();) {
1033  H5DataType temp_dtype = (*ira)->getType();
1034  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
1035  delete (*ira);
1036  ira = (*irg)->attrs.erase(ira);
1037  }
1038  else {
1039  ++ira;
1040  }
1041  }
1042  }
1043  }
1044  }
1045  }
1046 }
1047 
1048 // Generate group unsupported datatype Information, this is for the BES ignored object key
1049 void File::Gen_Group_Unsupported_Dtype_Info()
1050 {
1051 
1052  // First root
1053  if (false == this->root_attrs.empty()) {
1054  //if (true == this->unsupported_attr_dtype) {
1055  for (vector<Attribute *>::iterator ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira) {
1056  H5DataType temp_dtype = (*ira)->getType();
1057  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype) || temp_dtype == H5INT64 || temp_dtype == H5UINT64) {
1058  this->add_ignored_info_attrs(true, "/", (*ira)->name);
1059  }
1060  }
1061  //}
1062  }
1063 
1064  // Then the group attributes
1065  if (false == this->groups.empty()) {
1066  for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1067  if (false == (*irg)->attrs.empty()) {
1068  //if (true == (*irg)->unsupported_attr_dtype) {
1069  for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end(); ++ira) {
1070  H5DataType temp_dtype = (*ira)->getType();
1071  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype) || temp_dtype == H5INT64 || temp_dtype==H5UINT64 ) {
1072  this->add_ignored_info_attrs(true, (*irg)->path, (*ira)->name);
1073  }
1074  }
1075  //}
1076  }
1077  }
1078  }
1079 }
1080 
1081 // Handler unsupported variable datatype
1082 void File::Handle_Var_Unsupported_Dtype()
1083 {
1084  if (false == this->vars.empty()) {
1085  if (true == this->unsupported_var_dtype) {
1086  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end();) {
1087  H5DataType temp_dtype = (*irv)->getType();
1088  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
1089  delete (*irv);
1090  irv = this->vars.erase(irv);
1091  }
1092  else {
1093  ++irv;
1094 
1095  }
1096  }
1097  }
1098  }
1099 }
1100 
1101 // Generate unsupported variable type info. This is for the ignored objects.
1102 void File::Gen_Var_Unsupported_Dtype_Info()
1103 {
1104 
1105  if (false == this->vars.empty()) {
1106  //if (true == this->unsupported_var_dtype) {
1107  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1108  H5DataType temp_dtype = (*irv)->getType();
1109  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)||(H5INT64 == temp_dtype) ||(H5UINT64 == temp_dtype)) {
1110  this->add_ignored_info_objs(false, (*irv)->fullpath);
1111  }
1112  }
1113  //}
1114  }
1115 
1116 }
1117 
1118 // Handling unsupported datatypes for variable(HDF5 dataset) and the correponding attributes.
1119 void File::Handle_VarAttr_Unsupported_Dtype()
1120 {
1121  if (false == this->vars.empty()) {
1122  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1123  if (false == (*irv)->attrs.empty()) {
1124  if (true == (*irv)->unsupported_attr_dtype) {
1125  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end();) {
1126  H5DataType temp_dtype = (*ira)->getType();
1127  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
1128  delete (*ira);
1129  ira = (*irv)->attrs.erase(ira);
1130  }
1131  else {
1132  ++ira;
1133  }
1134  }
1135  }
1136  }
1137  }
1138  }
1139 }
1140 
1141 // Generated unsupported var/attribute unsupported datatype Info when the BES ignored object key is on.
1142 void File::Gen_VarAttr_Unsupported_Dtype_Info()
1143 {
1144 
1145  if (false == this->vars.empty()) {
1146  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1147  if (false == (*irv)->attrs.empty()) {
1148  //if (true == (*irv)->unsupported_attr_dtype) {
1149  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
1150  H5DataType temp_dtype = (*ira)->getType();
1151  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype) || (temp_dtype==H5INT64) || (temp_dtype == H5UINT64)) {
1152  this->add_ignored_info_attrs(false, (*irv)->fullpath, (*ira)->name);
1153  }
1154  }
1155  //}
1156  }
1157  }
1158  }
1159 }
1160 
1161 // Generated unsupported datatype information for HDF5 dimension scales.
1162 // The datatypes of HDF5 dimension scales("DIMENSION_LIST" and "REFERENCE_LIST")
1163 // are not supported. However, the information
1164 // are retrieved by the handlers so we don't want to report them as ignored objects.
1165 void File::Gen_DimScale_VarAttr_Unsupported_Dtype_Info()
1166 {
1167 
1168  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1169 
1170  // If the attribute REFERENCE_LIST comes with the attribut CLASS, the
1171  // attribute REFERENCE_LIST is okay to ignore. No need to report.
1172  bool is_ignored = ignored_dimscale_ref_list((*irv));
1173  if (false == (*irv)->attrs.empty()) {
1174  //if (true == (*irv)->unsupported_attr_dtype) {
1175  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
1176  H5DataType temp_dtype = (*ira)->getType();
1177  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype) || (temp_dtype == H5INT64) || (temp_dtype == H5UINT64)) {
1178  // "DIMENSION_LIST" is okay to ignore and "REFERENCE_LIST"
1179  // is okay to ignore if the variable has another attribute
1180  // CLASS="DIMENSION_SCALE"
1181  if (("DIMENSION_LIST" != (*ira)->name)
1182  && ("REFERENCE_LIST" != (*ira)->name || true == is_ignored))
1183  this->add_ignored_info_attrs(false, (*irv)->fullpath, (*ira)->name);
1184  }
1185  }
1186  //}
1187  }
1188  }
1189 }
1190 
1191 // Handle unsupported dataspace for group attributes.
1192 void File::Handle_GroupAttr_Unsupported_Dspace()
1193 {
1194 
1195  // First root
1196  if (false == this->root_attrs.empty()) {
1197  if (true == this->unsupported_attr_dspace) {
1198  for (vector<Attribute *>::iterator ira = this->root_attrs.begin(); ira != this->root_attrs.end();) {
1199  // Remove 0-size attribute
1200  if ((*ira)->count == 0) {
1201  delete (*ira);
1202  ira = this->root_attrs.erase(ira);
1203  }
1204  else {
1205  ++ira;
1206  }
1207  }
1208  }
1209  }
1210 
1211  // Then the group attributes
1212  if (false == this->groups.empty()) {
1213  for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1214  if (false == (*irg)->attrs.empty()) {
1215  if (true == (*irg)->unsupported_attr_dspace) {
1216  for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end();) {
1217  if ((*ira)->count == 0) {
1218  delete (*ira);
1219  ira = (*irg)->attrs.erase(ira);
1220  }
1221  else {
1222  ++ira;
1223  }
1224  }
1225  }
1226  }
1227  }
1228  }
1229 }
1230 
1231 // Handle unsupported data space information for variable and attribute
1232 void File::Handle_VarAttr_Unsupported_Dspace()
1233 {
1234 
1235  if (false == this->vars.empty()) {
1236  if (true == this->unsupported_var_attr_dspace) {
1237  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1238  if (false == (*irv)->attrs.empty()) {
1239  if (true == (*irv)->unsupported_attr_dspace) {
1240  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end();) {
1241  if (0 == (*ira)->count) {
1242  delete (*ira);
1243  ira = (*irv)->attrs.erase(ira);
1244  }
1245  else {
1246  ++ira;
1247  }
1248  }
1249  }
1250  }
1251  }
1252  }
1253  }
1254 }
1255 
1256 // Handle unsupported data space.
1257 void File::Handle_Unsupported_Dspace(bool include_attr)
1258 {
1259 
1260  // The unsupported data space
1261  if (false == this->vars.empty()) {
1262  if (true == this->unsupported_var_dspace) {
1263  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end();) {
1264  if (true == (*irv)->unsupported_dspace) {
1265  delete (*irv);
1266  irv = this->vars.erase(irv);
1267  }
1268  else {
1269  ++irv;
1270 
1271  }
1272  }
1273  }
1274  }
1275 
1276  if (true == include_attr) {
1277  Handle_GroupAttr_Unsupported_Dspace();
1278  Handle_VarAttr_Unsupported_Dspace();
1279  }
1280 }
1281 
1282 // Generated unsupported dataspace Info when the BES ignored object key is on.
1283 void File::Gen_Unsupported_Dspace_Info()
1284 {
1285 
1286  // Notice in this function, we deliberately don't put the case when an attribute dimension has 0 length.
1287  // Since doing this requires non-trivial change of the source code and the 0-size attribute case is really, really rare,
1288  // so we just "ignore" this case in the "ignored" information.
1289  // In fact, the zero size variable is allowed in both HDF5 and DAP2. So we don't ignore 0-size HDF5 dataset. So
1290  // the only case this function checks is the H5S_NULL case.
1291  if (false == this->vars.empty()) {
1292  if (true == this->unsupported_var_dspace) {
1293  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1294  if (true == (*irv)->unsupported_dspace) {
1295  this->add_ignored_info_objs(true, (*irv)->fullpath);
1296  }
1297  }
1298  }
1299  }
1300 
1301 }
1302 
1303 // Handle other unsupported information.
1304 void File::Handle_Unsupported_Others(bool include_attr)
1305 {
1306 
1307  if (true == this->check_ignored && true == include_attr) {
1308 
1309  if (true == HDF5RequestHandler::get_drop_long_string()) {
1310 
1311  // netCDF java doesn't have limitation for attributes
1312 #if 0
1313  for (vector<Attribute *>::iterator ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira) {
1314  if (H5FSTRING == (*ira)->dtype || H5VSTRING == (*ira)->dtype) {
1315  if ((*ira)->getBufSize() > NC_JAVA_STR_SIZE_LIMIT) {
1316  this->add_ignored_droplongstr_hdr();
1317  this->add_ignored_grp_longstr_info("/", (*ira)->name);
1318  }
1319  }
1320  }
1321 
1322  for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1323  for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end(); ++ira) {
1324  if (H5FSTRING == (*ira)->dtype || H5VSTRING == (*ira)->dtype) {
1325  if ((*ira)->getBufSize() > NC_JAVA_STR_SIZE_LIMIT) {
1326  this->add_ignored_droplongstr_hdr();
1327  this->add_ignored_grp_longstr_info((*irg)->path, (*ira)->name);
1328  }
1329  }
1330  }
1331  }
1332 #endif
1333  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1334  if (true == Check_DropLongStr((*irv), NULL)) {
1335  this->add_ignored_droplongstr_hdr();
1336  this->add_ignored_var_longstr_info((*irv), NULL);
1337  }
1338  // netCDF java doesn't have limitation for attributes
1339 #if 0
1340  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
1341  if (true == Check_DropLongStr((*irv), (*ira))) {
1342  this->add_ignored_droplongstr_hdr();
1343  this->add_ignored_var_longstr_info((*irv), (*ira));
1344  }
1345  }
1346 #endif
1347  }
1348  }
1349  }
1350 
1351 }
1352 
1353 // Flatten the object name, mainly call get_CF_string.
1354 void File::Flatten_Obj_Name(bool include_attr)
1355 {
1356 
1357  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1358  (*irv)->newname = get_CF_string((*irv)->newname);
1359 
1360  for (vector<Dimension *>::iterator ird = (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
1361  (*ird)->newname = get_CF_string((*ird)->newname);
1362  }
1363  }
1364 
1365  if (true == include_attr) {
1366 
1367  for (vector<Attribute *>::iterator ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira) {
1368  (*ira)->newname = get_CF_string((*ira)->newname);
1369  }
1370 
1371  for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1372  (*irg)->newname = get_CF_string((*irg)->newname);
1373  for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end(); ++ira) {
1374  (*ira)->newname = get_CF_string((*ira)->newname);
1375  }
1376  }
1377 
1378  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1379  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
1380  (*ira)->newname = get_CF_string((*ira)->newname);
1381  }
1382  }
1383  } // "if (true == include_attr)"
1384 }
1385 
1386 // Variable name clashing
1387 void File::Handle_Var_NameClashing(set<string>&objnameset)
1388 {
1389 
1390  Handle_General_NameClashing(objnameset, this->vars);
1391 }
1392 
1393 // Group name clashing
1394 void File::Handle_Group_NameClashing(set<string> &objnameset)
1395 {
1396 
1397  pair<set<string>::iterator, bool> setret;
1398 
1399  // Now for DAS, we need to handle name clashings for
1400  // DAS tables. Namely we need to make sure the global attribute
1401  // table(HDF5_GLOBAL) and the attribute tables mapped from
1402  // HDF5 groups will not have name clashings with the variable name
1403  // lists. If having the name clashings, the global attribute table and the
1404  // the attribute tables generated from the groups will be changed.
1405  // The file attribute name clashing
1406 
1407  setret = objnameset.insert(FILE_ATTR_TABLE_NAME);
1408  if (false == setret.second) {
1409 
1410  int clash_index = 1;
1411  string fa_clash_name = FILE_ATTR_TABLE_NAME;
1412  HDF5CFUtil::gen_unique_name(fa_clash_name, objnameset, clash_index);
1413  FILE_ATTR_TABLE_NAME = fa_clash_name;
1414  }
1415 
1416  // The group attribute name clashing
1417  Handle_General_NameClashing(objnameset, this->groups);
1418 
1419 }
1420 
1421 //Object attribute name clashing
1422 void File::Handle_Obj_AttrNameClashing()
1423 {
1424 
1425  // Now handling the possible name clashings for attributes
1426  // For attribute clashings, we only need to resolve the name clashings
1427  // for attributes within each variable, file attributes and attributes
1428  // within each group. The name clashings for attributes should be very rare.
1429  // Potentially the checking and the correcting may be costly.
1430  // This is another reason for special products, we may not even need to check
1431  // the name clashings. KY 2011-12-24
1432 
1433  set<string> objnameset;
1434 
1435  // For root attributes
1436  Handle_General_NameClashing(objnameset, this->root_attrs);
1437 
1438  // For group attributes
1439  for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1440  objnameset.clear();
1441  Handle_General_NameClashing(objnameset, (*irg)->attrs);
1442  }
1443 
1444  // For variable attributes
1445  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1446  objnameset.clear();
1447  Handle_General_NameClashing(objnameset, (*irv)->attrs);
1448  }
1449 }
1450 
1451 // Handle General name clashing
1452 //class T must have member string newname. In our case, T is either groups, attributes or vars.
1453 template<class T> void File::Handle_General_NameClashing(set<string>&objnameset, vector<T*>& objvec)
1454 {
1455 
1456 // set<string> objnameset;
1457  pair<set<string>::iterator, bool> setret;
1458  set<string>::iterator iss;
1459 
1460  vector<string> clashnamelist;
1461  vector<string>::iterator ivs;
1462 
1463  map<int, int> cl_to_ol;
1464  int ol_index = 0;
1465  int cl_index = 0;
1466 
1467  /*class*/
1468  typename vector<T*>::iterator irv;
1469 
1470  for (irv = objvec.begin(); irv != objvec.end(); ++irv) {
1471  setret = objnameset.insert((*irv)->newname);
1472  if (false == setret.second) {
1473  clashnamelist.insert(clashnamelist.end(), (*irv)->newname);
1474  cl_to_ol[cl_index] = ol_index;
1475  cl_index++;
1476  }
1477  ol_index++;
1478  }
1479 
1480  // Now change the clashed elements to unique elements,
1481  // Generate the set which has the same size as the original vector.
1482  for (ivs = clashnamelist.begin(); ivs != clashnamelist.end(); ivs++) {
1483  int clash_index = 1;
1484  string temp_clashname = *ivs + '_';
1485  HDF5CFUtil::gen_unique_name(temp_clashname, objnameset, clash_index);
1486  *ivs = temp_clashname;
1487  }
1488 
1489  // Now go back to the original vector, make it unique.
1490  for (unsigned int i = 0; i < clashnamelist.size(); i++)
1491  objvec[cl_to_ol[i]]->newname = clashnamelist[i];
1492 
1493 }
1494 
1495 // Handle General object name clashing
1496 void File::Handle_GeneralObj_NameClashing(bool include_attr, set<string>& objnameset)
1497 {
1498 
1499  Handle_Var_NameClashing(objnameset);
1500  if (true == include_attr) {
1501  Handle_Group_NameClashing(objnameset);
1502  Handle_Obj_AttrNameClashing();
1503  }
1504 }
1505 
1506 // Get CF name, flatten the path, change the non-alphanumeric letters to underscore.
1507 string File::get_CF_string(string s)
1508 {
1509 
1510  if ("" == s) return s;
1511  string insertString(1, '_');
1512 
1513  // Always start with _ if the first character is not a letter
1514  if (true == isdigit(s[0])) s.insert(0, insertString);
1515 
1516  for (unsigned int i = 0; i < s.length(); i++)
1517  if ((false == isalnum(s[i])) && (s[i] != '_')) s[i] = '_';
1518 
1519  return s;
1520 
1521 }
1522 
1523 // For the connection of variable dimensions. Build dimname to dimsize(unlimited) maps
1524 void File::Insert_One_NameSizeMap_Element(string name, hsize_t size, bool unlimited)
1525 {
1526  pair<map<string, hsize_t>::iterator, bool> mapret;
1527  mapret = dimname_to_dimsize.insert(pair<string, hsize_t>(name, size));
1528  if (false == mapret.second)
1529  throw4("The dimension name ", name, " should map to ", size);
1530 
1531  pair<map<string, bool>::iterator, bool> mapret2;
1532  mapret2 = dimname_to_unlimited.insert(pair<string, bool>(name, unlimited));
1533  if (false == mapret2.second)
1534  throw3("The dimension name ", name, " unlimited dimension info. should be provided.");
1535 
1536 }
1537 
1538 // Similar to Inset_One_NameSizeMap_Element but the maps are provided as parameters.
1539 void File::Insert_One_NameSizeMap_Element2(map<string, hsize_t>& name_to_size, map<string, bool>& name_to_unlimited,
1540  string name, hsize_t size, bool unlimited)
1541 {
1542  pair<map<string, hsize_t>::iterator, bool> mapret;
1543  mapret = name_to_size.insert(pair<string, hsize_t>(name, size));
1544  if (false == mapret.second)
1545  throw4("The dimension name ", name, " should map to ", size);
1546 
1547  pair<map<string, bool>::iterator, bool> mapret2;
1548  mapret2 = name_to_unlimited.insert(pair<string, bool>(name, unlimited));
1549  if (false == mapret2.second)
1550  throw3("The dimension name ", name, " unlimited dimension info. should be provided.");
1551 
1552 }
1553 
1554 // For dimension names added by the handlers, by default,
1555 // Each dimension will have a unique dimension name. For example,
1556 // Int foo[100][200] will be Int foo[Fakedim1][Fakedim2]
1557 // If you have many variables, the dimension names may be too many.
1558 // To reduce numbers, we ASSUME that the dimension having the same
1559 // size shares the same dimension. In this way, the number of dimension names
1560 // will be reduced.
1561 // For example, Int foo2[100][300] will be Int foo2[Fakedim1][Fakedim3]
1562 // instead of foo2[Fakedim3][Fakedim4]. However, that may impose
1563 // another problem. Suppose Int Foosame[100][100] becomes
1564 // Int Foosame[FakeDim1][FakeDim1]. This doesn't make sense for some
1565 // applications. The fuction Adjust_Duplicate_FakeDim_Name will make sure
1566 // this case will not happen.
1567 void File::Add_One_FakeDim_Name(Dimension *dim)
1568 {
1569 
1570  stringstream sfakedimindex;
1571  string fakedimstr = "FakeDim";
1572  pair<set<string>::iterator, bool> setret;
1573  map<hsize_t, string>::iterator im;
1574  pair<map<hsize_t, string>::iterator, bool> mapret;
1575 
1576  sfakedimindex << addeddimindex;
1577  string added_dimname = fakedimstr + sfakedimindex.str();
1578 
1579  // Build up the size to fakedim map.
1580  mapret = dimsize_to_fakedimname.insert(pair<hsize_t, string>(dim->size, added_dimname));
1581  if (false == mapret.second) { //The dim size exists, use the corresponding name.
1582  dim->name = dimsize_to_fakedimname[dim->size];
1583  dim->newname = dim->name;
1584  }
1585  else { // Insert this (dimsize,dimname) pair to dimsize_to_fakedimname map successfully.
1586  //First make sure this new dim name doesn't have name clashing
1587  // with previous dim names, after the checking, inserting to the
1588  // dimname list set.
1589  // dimnamelist is a private memeber of File.
1590  setret = dimnamelist.insert(added_dimname);
1591  if (false == setret.second) {
1592  int clash_index = 1;
1593  string temp_clashname = added_dimname + '_';
1594  HDF5CFUtil::gen_unique_name(temp_clashname, dimnamelist, clash_index);
1595  dim->name = temp_clashname;
1596  dim->newname = dim->name;
1597  setret = dimnamelist.insert(dim->name);
1598  if (false == setret.second)
1599  throw2("Fail to insert the unique dimsizede name ", dim->name);
1600 
1601  // We have to adjust the dim. name of the dimsize_to_fakedimname map, since the
1602  // dimname has been updated for this size.
1603  dimsize_to_fakedimname.erase(dim->size);
1604  mapret = dimsize_to_fakedimname.insert(pair<hsize_t, string>(dim->size, dim->name));
1605  if (false == mapret.second)
1606  throw4("The dimension size ", dim->size, " should map to ", dim->name);
1607  } // "if(false == setret.second)"
1608 
1609  // New dim name is inserted successfully, update the dimname_to_dimsize map.
1610  dim->name = added_dimname;
1611  dim->newname = dim->name;
1612  Insert_One_NameSizeMap_Element(dim->name, dim->size, dim->unlimited_dim);
1613 
1614  // Increase the dimindex since the new dimname has been inserted.
1615  addeddimindex++;
1616  } // else
1617 }
1618 
1619 // See the function comments of Add_One_FakeDim_Name
1620 void File::Adjust_Duplicate_FakeDim_Name(Dimension * dim)
1621 {
1622 
1623  // No need to adjust the dimsize_to_fakedimname map, only create a new Fakedim
1624  // The simplest way is to increase the dim index and resolve any name clashings with other dim names.
1625  // Note: No need to update the dimsize_to_dimname map since the original "FakeDim??" of this size
1626  // can be used as a dimension name of other variables. But we need to update the dimname_to_dimsize map
1627  // since this is a new dim name.
1628  stringstream sfakedimindex;
1629  pair<set<string>::iterator, bool> setret;
1630 
1631  addeddimindex++;
1632  sfakedimindex << addeddimindex;
1633  string added_dimname = "FakeDim" + sfakedimindex.str();
1634  setret = dimnamelist.insert(added_dimname);
1635  if (false == setret.second) {
1636  int clash_index = 1;
1637  string temp_clashname = added_dimname + '_';
1638  HDF5CFUtil::gen_unique_name(temp_clashname, dimnamelist, clash_index);
1639  dim->name = temp_clashname;
1640  dim->newname = dim->name;
1641  setret = dimnamelist.insert(dim->name);
1642  if (false == setret.second)
1643  throw2("Fail to insert the unique dimsizede name ", dim->name);
1644  }
1645  dim->name = added_dimname;
1646  dim->newname = dim->name;
1647  Insert_One_NameSizeMap_Element(dim->name, dim->size, dim->unlimited_dim);
1648 
1649  // Need to prepare for the next unique FakeDim.
1650  addeddimindex++;
1651 }
1652 
1653 // Replace all dimension names, this function is currently not used. So comment out. May delete it in the future.
1654 #if 0
1655 void File::Replace_Dim_Name_All(const string orig_dim_name, const string new_dim_name) {
1656 
1657  // The newname of the original dimension should also be replaced by new_dim_name
1658  for (vector<Var *>::iterator irv = this->vars.begin();
1659  irv != this->vars.end(); ++irv) {
1660  for (vector<Dimension *>::iterator ird= (*irv)->dims.begin();
1661  ird != (*irv)->dims.end(); ++ird) {
1662  if((*ird)->name == orig_dim_name) {
1663  (*ird)->name = new_dim_name;
1664  (*ird)->newname = new_dim_name;
1665  }
1666 
1667  }
1668  }
1669 }
1670 #endif
1671 
1672 #if 0
1673 void File::Use_Dim_Name_With_Size_All(const string dim_name, const size_t dim_size) {
1674 
1675  // The newname of the original dimension should also be replaced by new_dim_name
1676  for (vector<Var *>::iterator irv = this->vars.begin();
1677  irv != this->vars.end(); ++irv) {
1678  for (vector<Dimension *>::iterator ird= (*irv)->dims.begin();
1679  ird != (*irv)->dims.end(); ++ird) {
1680  if((*ird)->size == orig_dim_name) {
1681  (*ird)->name = new_dim_name;
1682  (*ird)->newname = new_dim_name;
1683  }
1684 
1685  }
1686  }
1687 }
1688 #endif
1689 
1690 // Often times we need to add a CF attribute with string datatype because some products don't provide them
1691 // Examples are units, comment etc.
1692 void File::Add_Str_Attr(Attribute* attr, const string &attrname, const string& strvalue)
1693 {
1694 
1695  attr->name = attrname;
1696  attr->newname = attr->name;
1697  attr->dtype = H5FSTRING;
1698  attr->count = 1;
1699  attr->fstrsize = strvalue.size();
1700  attr->strsize.resize(1);
1701  attr->strsize[0] = attr->fstrsize;
1702  attr->value.resize(strvalue.size());
1703  copy(strvalue.begin(), strvalue.end(), attr->value.begin());
1704 }
1705 
1706 #if 0
1707 bool
1708 File:: Var_Has_Attr(Var*var,const string &attrname) {
1709 
1710  for (vector<Attribute *>:: iterator ira =var->attrs.begin(); ira !=var->attrs.end(); ++ira) {
1711 
1712  // We only check the original attribute name
1713  // Remove the original "coordinates" attribute.
1714  if((*ira)->name == attrname || (*ira)->newname == attrname) {
1715  return true;
1716  }
1717  }
1718  return false;
1719 }
1720 #endif
1721 
1722 // Rretrieve the variable attribute in string.var_path is the variable path.
1723 string File::Retrieve_Str_Attr_Value(Attribute *attr, const string var_path)
1724 {
1725 
1726  if (attr != NULL && var_path != "") {
1727  Retrieve_H5_Attr_Value(attr, var_path);
1728  string orig_attr_value(attr->value.begin(), attr->value.end());
1729  return orig_attr_value;
1730  }
1731  return "";
1732 
1733 }
1734 
1735 //Check if the attribute value of this variable is the input value.
1736 bool File::Is_Str_Attr(Attribute* attr, string varfullpath, const string &attrname, const string& strvalue)
1737 {
1738  bool ret_value = false;
1739  if (attrname == get_CF_string(attr->newname)) {
1740  Retrieve_H5_Attr_Value(attr, varfullpath);
1741  string attr_value(attr->value.begin(), attr->value.end());
1742  if (attr_value == strvalue) ret_value = true;
1743  }
1744  return ret_value;
1745 }
1746 
1747 // If this latitude or longitude units follows the CF
1748 bool File::has_latlon_cf_units(Attribute *attr, const string &varfullpath, bool is_lat)
1749 {
1750  string attr_name = "units";
1751  if (true == is_lat) {
1752  string lat_unit_value = "degrees_north";
1753  return Is_Str_Attr(attr, varfullpath, attr_name, lat_unit_value);
1754  }
1755  else {
1756  string lon_unit_value = "degrees_east";
1757  return Is_Str_Attr(attr, varfullpath, attr_name, lon_unit_value);
1758  }
1759 }
1760 
1761 // This function is mainly to add _FillValue.
1762 void File::Add_One_Float_Attr(Attribute* attr, const string &attrname, float float_value)
1763 {
1764  attr->name = attrname;
1765  attr->newname = attr->name;
1766  attr->dtype = H5FLOAT32;
1767  attr->count = 1;
1768  attr->value.resize(sizeof(float));
1769  memcpy(&(attr->value[0]), (void*) (&float_value), sizeof(float));
1770 }
1771 
1772 // Products like GPM use string type for MissingValue, we need to change them to the corresponding variable datatype and
1773 // get the value corrected.
1774 void File::Change_Attr_One_Str_to_Others(Attribute* attr, Var*var)
1775 {
1776 
1777  char *pEnd;
1778  // string to long int number.
1779  long int num_sli = 0;
1780  if (attr->dtype != H5FSTRING)
1781  throw2("Currently we only convert fixed-size string to other datatypes. ", attr->name);
1782  if (attr->count != 1)
1783  throw4("The fixed-size string count must be 1 and the current count is ", attr->count, " for the attribute ",
1784  attr->name);
1785 
1786  Retrieve_H5_Attr_Value(attr, var->fullpath);
1787  string attr_value;
1788  attr_value.resize(attr->value.size());
1789  copy(attr->value.begin(), attr->value.end(), attr_value.begin());
1790 
1791  switch (var->dtype) {
1792 
1793  case H5UCHAR: {
1794  num_sli = strtol(&(attr->value[0]), &pEnd, 10);
1795  if (num_sli < 0 || num_sli > UCHAR_MAX)
1796  throw5("Attribute type is unsigned char, the current attribute ", attr->name, " has the value ", num_sli,
1797  ". It is overflowed. ");
1798  else {
1799  unsigned char num_suc = (unsigned char) num_sli;
1800  attr->dtype = H5UCHAR;
1801  attr->value.resize(sizeof(unsigned char));
1802  memcpy(&(attr->value[0]), (void*) (&num_suc), sizeof(unsigned char));
1803  }
1804 
1805  }
1806  break;
1807  case H5CHAR: {
1808  num_sli = strtol(&(attr->value[0]), &pEnd, 10);
1809  if (num_sli < SCHAR_MIN || num_sli > SCHAR_MAX)
1810  throw5("Attribute type is signed char, the current attribute ", attr->name, " has the value ", num_sli,
1811  ". It is overflowed. ");
1812  else {
1813  char num_sc = (char) num_sli;
1814  attr->dtype = H5CHAR;
1815  attr->value.resize(sizeof(char));
1816  memcpy(&(attr->value[0]), (void*) (&num_sc), sizeof(char));
1817  }
1818 
1819  }
1820  break;
1821  case H5INT16: {
1822  num_sli = strtol(&(attr->value[0]), &pEnd, 10);
1823  if (num_sli < SHRT_MIN || num_sli > SHRT_MAX)
1824  throw5("Attribute type is 16-bit integer, the current attribute ", attr->name, " has the value ", num_sli,
1825  ". It is overflowed. ");
1826  else {
1827  short num_ss = (short) num_sli;
1828  attr->dtype = H5INT16;
1829  attr->value.resize(sizeof(short));
1830  memcpy(&(attr->value[0]), (void*) (&num_ss), sizeof(short));
1831  }
1832 
1833  }
1834  break;
1835  case H5UINT16: {
1836  num_sli = strtol(&(attr->value[0]), &pEnd, 10);
1837  if (num_sli < 0 || num_sli > USHRT_MAX)
1838  throw5("Attribute type is unsigned 16-bit integer, the current attribute ", attr->name, " has the value ",
1839  num_sli, ". It is overflowed. ");
1840  else {
1841  unsigned short num_uss = (unsigned short) num_sli;
1842  attr->dtype = H5UINT16;
1843  attr->value.resize(sizeof(unsigned short));
1844  memcpy(&(attr->value[0]), (void*) (&num_uss), sizeof(unsigned short));
1845  }
1846  }
1847  break;
1848  case H5INT32: {
1849  num_sli = strtol(&(attr->value[0]), &pEnd, 10);
1850  //No need to check overflow, the number will always be in the range.
1851 #if 0
1852  //if(num_sli <LONG_MIN || num_sli >LONG_MAX)
1853  // throw5("Attribute type is 32-bit integer, the current attribute ",attr->name, " has the value ",num_sli, ". It is overflowed. ");
1854 #endif
1855  attr->dtype = H5INT32;
1856  attr->value.resize(sizeof(long int));
1857  memcpy(&(attr->value[0]), (void*) (&num_sli), sizeof(long int));
1858 
1859  }
1860  break;
1861  case H5UINT32: {
1862  unsigned long int num_suli = strtoul(&(attr->value[0]), &pEnd, 10);
1863  // No need to check since num_suli will not be bigger than ULONG_MAX.
1864  attr->dtype = H5UINT32;
1865  attr->value.resize(sizeof(unsigned long int));
1866  memcpy(&(attr->value[0]), (void*) (&num_suli), sizeof(unsigned long int));
1867  }
1868  break;
1869  case H5FLOAT32: {
1870  float num_sf = strtof(&(attr->value[0]), NULL);
1871  // Don't think it is necessary to check if floating-point is oveflowed for this routine. ignore it now. KY 2014-09-22
1872  attr->dtype = H5FLOAT32;
1873  attr->value.resize(sizeof(float));
1874  memcpy(&(attr->value[0]), (void*) (&num_sf), sizeof(float));
1875  }
1876  break;
1877  case H5FLOAT64: {
1878  double num_sd = strtod(&(attr->value[0]), NULL);
1879  // Don't think it is necessary to check if floating-point is oveflowed for this routine. ignore it now. KY 2014-09-22
1880  attr->dtype = H5FLOAT64;
1881  attr->value.resize(sizeof(double));
1882  memcpy(&(attr->value[0]), (void*) (&num_sd), sizeof(double));
1883  }
1884  break;
1885 
1886  default:
1887  throw4("Unsupported HDF5 datatype that the string is converted to for the attribute ", attr->name,
1888  " of the variable ", var->fullpath);
1889  } // "switch(var->dtype)"
1890 
1891 }
1892 
1893 // Change a string type attribute to the input value
1894 void File::Replace_Var_Str_Attr(Var* var, const string &attr_name, const string& strvalue)
1895 {
1896 
1897  bool rep_attr = true;
1898  bool rem_attr = false;
1899  for (vector<Attribute *>::iterator ira = var->attrs.begin(); ira != var->attrs.end(); ira++) {
1900  if ((*ira)->name == attr_name) {
1901  if (true == Is_Str_Attr(*ira, var->fullpath, attr_name, strvalue))
1902  rep_attr = false;
1903  else
1904  rem_attr = true;
1905  break;
1906  }
1907  }
1908 
1909  // Remove the attribute if the attribute value is not strvalue
1910  if (true == rem_attr) {
1911  for (vector<Attribute *>::iterator ira = var->attrs.begin(); ira != var->attrs.end(); ira++) {
1912  if ((*ira)->name == attr_name) {
1913  delete (*ira);
1914  var->attrs.erase(ira);
1915  break;
1916  }
1917  }
1918  }
1919 
1920  // Add the attribute with strvalue
1921  if (true == rep_attr) {
1922  Attribute * attr = new Attribute();
1923  Add_Str_Attr(attr, attr_name, strvalue);
1924  var->attrs.push_back(attr);
1925  }
1926 }
1927 
1928 // Check if this variable if latitude,longitude.We check the three name pairs(lat,lon),(latitude,longitude),(Latitude,Longitude)
1929 bool File::Is_geolatlon(const string & var_name, bool is_lat)
1930 {
1931 
1932  bool ret_value = false;
1933  if (true == is_lat) {
1934  string lat1 = "lat";
1935  string lat2 = "latitude";
1936  string lat3 = "Latitude";
1937 
1938  if (var_name.compare(lat1) == 0 || var_name.compare(lat2) == 0 || var_name.compare(lat3) == 0) ret_value = true;
1939  }
1940 
1941  else {
1942  string lon1 = "lon";
1943  string lon2 = "longitude";
1944  string lon3 = "Longitude";
1945  if (var_name.compare(lon1) == 0 || var_name.compare(lon2) == 0 || var_name.compare(lon3) == 0) ret_value = true;
1946 
1947  }
1948  return ret_value;
1949 }
1950 
1951 // Add supplementary attributes.
1952 void File::Add_Supplement_Attrs(bool add_path)
1953 {
1954 
1955  if (false == add_path) return;
1956 
1957  // Adding variable original name(origname) and full path(fullpath)
1958  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1959  Attribute * attr = new Attribute();
1960  const string varname = (*irv)->name;
1961  const string attrname = "origname";
1962  Add_Str_Attr(attr, attrname, varname);
1963  (*irv)->attrs.push_back(attr);
1964  }
1965 
1966  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1967  // Turn off the fullnamepath attribute when zero_storage_size is 0.
1968  // Use the BES key since quite a few testing cases will be affected.
1969  // KY 2020-03-23
1970  if((*irv)->zero_storage_size==false
1971  || HDF5RequestHandler::get_no_zero_size_fullnameattr() == false) {
1972  Attribute * attr = new Attribute();
1973  const string varname = (*irv)->fullpath;
1974  const string attrname = "fullnamepath";
1975  Add_Str_Attr(attr, attrname, varname);
1976  (*irv)->attrs.push_back(attr);
1977  }
1978  }
1979 
1980  // Adding group path
1981  for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1982  // Only when this group has attributes, the original path of the group has some values. So add it.
1983  if (false == (*irg)->attrs.empty()) {
1984 
1985  Attribute * attr = new Attribute();
1986  const string varname = (*irg)->path;
1987  const string attrname = "fullnamepath";
1988  Add_Str_Attr(attr, attrname, varname);
1989  (*irg)->attrs.push_back(attr);
1990  }
1991  }
1992 
1993 }
1994 
1995 // Variable target will not be deleted, but rather its contents are replaced.
1996 // We may make this as an operator = in the future.
1997 // Note: the attributes can not be replaced.
1998 void File::Replace_Var_Info(Var *src, Var *target)
1999 {
2000 
2001 #if 0
2002  for_each (target->dims.begin (), target->dims.end (),
2003  delete_elem ());
2004  for_each (target->attrs.begin (), target->attrs.end (),
2005  delete_elem ());
2006 #endif
2007 
2008  target->newname = src->newname;
2009  target->name = src->name;
2010  target->fullpath = src->fullpath;
2011  target->rank = src->rank;
2012  target->dtype = src->dtype;
2013  target->unsupported_attr_dtype = src->unsupported_attr_dtype;
2014  target->unsupported_dspace = src->unsupported_dspace;
2015 #if 0
2016  for (vector<Attribute*>::iterator ira = target->attrs.begin();
2017  ira!=target->attrs.end(); ++ira) {
2018  delete (*ira);
2019  target->attrs.erase(ira);
2020  ira--;
2021  }
2022 #endif
2023  for (vector<Dimension*>::iterator ird = target->dims.begin(); ird != target->dims.end();) {
2024  delete (*ird);
2025  ird = target->dims.erase(ird);
2026  }
2027 
2028  // Somehow attributes cannot be replaced.
2029 #if 0
2030  for (vector<Attribute*>::iterator ira = src->attrs.begin();
2031  ira!=src->attrs.end(); ++ira) {
2032  Attribute* attr= new Attribute();
2033  attr->name = (*ira)->name;
2034  attr->newname = (*ira)->newname;
2035  attr->dtype =(*ira)->dtype;
2036  attr->count =(*ira)->count;
2037  attr->strsize = (*ira)->strsize;
2038  attr->fstrsize = (*ira)->fstrsize;
2039  attr->value =(*ira)->value;
2040  target->attrs.push_back(attr);
2041  }
2042 #endif
2043 
2044  for (vector<Dimension*>::iterator ird = src->dims.begin(); ird != src->dims.end(); ++ird) {
2045  Dimension *dim = new Dimension((*ird)->size);
2046  dim->name = (*ird)->name;
2047  dim->newname = (*ird)->newname;
2048  target->dims.push_back(dim);
2049  }
2050 
2051 }
2052 
2053 // Replace the attributes of target with src.
2054 void File::Replace_Var_Attrs(Var *src, Var *target)
2055 {
2056 
2057 #if 0
2058  for_each (target->dims.begin (), target->dims.end (),
2059  delete_elem ());
2060  for_each (target->attrs.begin (), target->attrs.end (),
2061  delete_elem ());
2062 #endif
2063 
2064  for (vector<Attribute*>::iterator ira = target->attrs.begin(); ira != target->attrs.end();) {
2065  delete (*ira);
2066  ira = target->attrs.erase(ira);
2067  }
2068  for (vector<Attribute*>::iterator ira = src->attrs.begin(); ira != src->attrs.end(); ++ira) {
2069  Attribute* attr = new Attribute();
2070  attr->name = (*ira)->name;
2071  attr->newname = (*ira)->newname;
2072  attr->dtype = (*ira)->dtype;
2073  attr->count = (*ira)->count;
2074  attr->strsize = (*ira)->strsize;
2075  attr->fstrsize = (*ira)->fstrsize;
2076  attr->value = (*ira)->value;
2077  target->attrs.push_back(attr);
2078  }
2079 
2080 }
2081 
2082 // Check if a variable with a var name is under a specific group with groupname
2083 // note: the variable's size at each dimension is also returned. The user must allocate the
2084 // memory for the dimension sizes(an array(vector is perferred).
2085 bool File::is_var_under_group(const string &varname, const string &grpname, const int var_rank,
2086  vector<size_t> & var_size)
2087 {
2088 
2089  bool ret_value = false;
2090  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2091 
2092  if ((*irv)->rank == var_rank) {
2093  if ((*irv)->name == varname) {
2094 
2095  // Obtain the variable path
2096  string var_path = HDF5CFUtil::obtain_string_before_lastslash((*irv)->fullpath);
2097 
2098  // Check if we find the variable under this group
2099  if (grpname == var_path) {
2100  ret_value = true;
2101  for (int i = 0; i < var_rank; i++)
2102  var_size[i] = (*irv)->getDimensions()[i]->size;
2103  break;
2104  }
2105  }
2106  } // "if((*irv)->rank == var_rank)"
2107  } // "for (vector<Var *>::iterator irv = this->vars.begin()"
2108 
2109 
2110  return ret_value;
2111 
2112 }
2114 
2115  bool ret_value = false;
2116  for (vector<Var *>::iterator irv = this->vars.begin();
2117  irv != this->vars.end(); ++irv) {
2118  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin();
2119  ira != (*irv)->attrs.end(); ++ira) {
2120  if((*ira)->name =="grid_mapping") {
2121  ret_value = true;
2122  break;
2123  }
2124  }
2125  if(true == ret_value)
2126  break;
2127  }
2128 
2129  return ret_value;
2130 
2131 }
2132 
2134 
2135  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2136  string attr_value;
2137  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
2138  if((*ira)->name =="grid_mapping") {
2139  Retrieve_H5_Attr_Value(*ira, (*irv)->fullpath);
2140  attr_value.resize((*ira)->value.size());
2141  copy((*ira)->value.begin(), (*ira)->value.end(), attr_value.begin());
2142  break;
2143  }
2144 
2145  }
2146  if(attr_value.find('/') ==string::npos){
2147  string new_name = Check_Grid_Mapping_VarName(attr_value,(*irv)->fullpath);
2148  if(new_name != "")
2149  Replace_Var_Str_Attr((*irv),"grid_mapping",new_name);
2150 
2151  }
2152  else {
2153  string new_name = Check_Grid_Mapping_FullPath(attr_value);
2154  //Using new_name as the attribute value
2155  if(new_name != "")
2156  Replace_Var_Str_Attr((*irv),"grid_mapping",new_name);
2157  }
2158  }
2159 
2160 }
2161 
2162 string File::Check_Grid_Mapping_VarName(const string & a_value,const string & var_fpath) {
2163 
2164  string var_path = HDF5CFUtil::obtain_string_before_lastslash(var_fpath);
2165  string gmap_new_name;
2166  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2167  if((*irv)->name == a_value){
2168  if(var_path == HDF5CFUtil::obtain_string_before_lastslash((*irv)->fullpath)) {
2169  gmap_new_name = (*irv)->newname;
2170  break;
2171  }
2172  }
2173  }
2174  return gmap_new_name;
2175 }
2176 
2177 
2178 string File::Check_Grid_Mapping_FullPath(const string & a_value) {
2179 
2180  string gmap_new_name;
2181  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2182  if((*irv)->fullpath == a_value){
2183  gmap_new_name = (*irv)->newname;
2184  break;
2185  }
2186  }
2187 
2188  return gmap_new_name;
2189 }
2190 
2191 void File::remove_netCDF_internal_attributes(bool include_attr) {
2192 
2193  if(true == include_attr) {
2194  for (vector<Var *>::iterator irv = this->vars.begin();
2195  irv != this->vars.end(); ++irv) {
2196  bool var_has_dimscale = false;
2197 
2198  for(vector<Attribute *>::iterator ira = (*irv)->attrs.begin();
2199  ira != (*irv)->attrs.end();) {
2200  if((*ira)->name == "CLASS") {
2201  string class_value = Retrieve_Str_Attr_Value(*ira,(*irv)->fullpath);
2202 
2203  // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
2204  // "DIMENSION_SCALE", which is 15.
2205  if (0 == class_value.compare(0,15,"DIMENSION_SCALE")) {
2206  delete(*ira);
2207  ira = (*irv)->attrs.erase(ira);
2208  var_has_dimscale = true;
2209 
2210  }
2211 #if 0
2212  else if(1) {// Add a BES key,also delete
2213 
2214  }
2215 #endif
2216  else {
2217  ++ira;
2218  }
2219  }
2220 #if 0
2221  else if((*ira)->name == "NAME") {// Add a BES Key
2222  string name_value = Retrieve_Str_Attr_Value(*ira,(*irv)->fullpath);
2223  if( 0 == name_value.compare(0,(*irv)->name.size(),(*irv)->name)) {
2224  delete(*ira);
2225  ira =(*irv)->attrs.erase(ira);
2226  }
2227  else {
2228  string netcdf_dim_mark= "This is a netCDF dimension but not a netCDF variable";
2229  if( 0 == name_value.compare(0,netcdf_dim_mark.size(),netcdf_dim_mark)) {
2230  delete(*ira);
2231  ira =(*irv)->attrs.erase(ira);
2232  }
2233  else {
2234  ++ira;
2235  }
2236  }
2237 
2238  }
2239 #endif
2240  else if((*ira)->name == "_Netcdf4Dimid") {
2241  delete(*ira);
2242  ira =(*irv)->attrs.erase(ira);
2243  }
2244  else if((*ira)->name == "_Netcdf4Coordinates") {
2245  delete(*ira);
2246  ira =(*irv)->attrs.erase(ira);
2247  }
2248 #if 0
2249  else if((*ira)->name == "_nc3_strict") {
2250  delete((*ira));
2251  ira =(*irv)->attrs.erase(ira);
2252  }
2253 #endif
2254  else {
2255  ++ira;
2256  }
2257  }
2258 
2259  if(true == var_has_dimscale) {
2260  for(vector<Attribute *>::iterator ira = (*irv)->attrs.begin();
2261  ira != (*irv)->attrs.end();++ira) {
2262  if((*ira)->name == "NAME") {// Add a BES Key
2263  delete(*ira);
2264  ira =(*irv)->attrs.erase(ira);
2265  break;
2266  }
2267  }
2268  }
2269  }
2270  }
2271 
2272 }
2273 // Add ignored page header info. Mainly a helper message.
2274 void File::add_ignored_info_page_header()
2275 {
2276  ignored_msg =
2277  " \n This page is for HDF5 CF hyrax data providers or distributors to check if any HDF5 object or attribute information are ignored during the mapping. \n\n";
2278 }
2279 
2280 // Add ignored object header info. Mainly a helper message.
2281 void File::add_ignored_info_obj_header()
2282 {
2283 
2284  ignored_msg += " Some HDF5 objects or the object information are ignored when mapping to DAP2 by the HDF5 OPeNDAP";
2285  ignored_msg += " handler due to the restrictions of DAP2, CF conventions or CF tools.";
2286  ignored_msg += " Please use HDF5 tools(h5dump or HDFView) to check carefully and make sure that these objects";
2287  ignored_msg +=
2288  " are OK to ignore for your service. For questions or requests to find a way to handle the ignored objects, please";
2289  ignored_msg += " contact the HDF5 OPeNDAP handler developer or send an email to help@hdfgroup.org.\n";
2290 
2291  ignored_msg += " \n In general, ignored HDF5 objects include HDF5 soft links, external links and named datatype.\n";
2292  ignored_msg +=
2293  " \n The HDF5 datasets(variables in the CF term) and attributes that have the following datatypes are ignored: \n";
2294  ignored_msg +=
2295  " Signed and unsigned 64-bit integers, HDF5 compound, HDF5 variable length(excluding variable length string),";
2296  ignored_msg += " HDF5 reference, HDF5 enum, HDF5 opaque , HDF5 bitfield, HDF5 Array and HDF5 Time datatypes.\n";
2297 
2298  ignored_msg +=
2299  " \n The HDF5 datasets(variables in the CF term) and attributes associated with the following dimensions are ignored: \n";
2300  ignored_msg += " 1) variables that have HDF5 NULL dataspace(H5S_NULL)(rarely occurred)\n";
2301  ignored_msg += " 2) attributes that have any zero size dimensions(not reported due to extreme rarity and non-trivial coding)\n\n";
2302 
2303 }
2304 
2305 // Add the ignored links information.Mainly a helper message.
2306 void File::add_ignored_info_links_header()
2307 {
2308 
2309  if (false == this->have_ignored) {
2310  add_ignored_info_obj_header();
2311  have_ignored = true;
2312  }
2313  // Add ignored datatype header.
2314  string lh_msg = "******WARNING******\n";
2315  lh_msg += "IGNORED soft links or external links are: ";
2316  if (ignored_msg.rfind(lh_msg) == string::npos) ignored_msg += lh_msg + "\n";
2317 
2318 }
2319 
2320 // Leave the code for the time being.
2321 #if 0
2322 void
2323 File:: add_ignored_info_obj_dtype_header() {
2324 
2325  // Add ignored datatype header.
2326  ignored_msg += " \n Variables and attributes ignored due to the unsupported datatypes. \n";
2327  ignored_msg += " In general, the unsupported datatypes include: \n";
2328  ignored_msg += " Signed and unsigned 64-bit integers, HDF5 compound, HDF5 variable length(excluding variable length string),";
2329  ignored_msg += " HDF5 reference, HDF5 enum, HDF5 opaque , HDF5 bitfield, HDF5 Array and HDF5 Time datatypes.\n";
2330 
2331 }
2332 
2333 void
2334 File:: add_ignored_info_obj_dspace_header() {
2335 
2336  // Add ignored dataspace header.
2337  ignored_msg += " \n Variables and attributes ignored due to the unsupported dimensions. \n";
2338  ignored_msg += " In general, the unsupported dimensions include: \n";
2339  ignored_msg += " 1) variables that have HDF5 NULL dataspace(H5S_NULL)(rarely occurred)\n";
2340  ignored_msg += " 2) variables that have any zero size dimensions\n";
2341 
2342 }
2343 #endif
2344 
2345 // Add the ignored link info.
2346 void File::add_ignored_info_links(const string & link_path)
2347 {
2348  if (ignored_msg.find("Link paths: ") == string::npos)
2349  ignored_msg += " Link paths: " + link_path;
2350  else
2351  ignored_msg += " " + link_path;
2352 }
2353 
2354 // Add the ignored name datatype info.
2355 void File::add_ignored_info_namedtypes(const string& grp_name, const string& named_dtype_name)
2356 {
2357 
2358  if (false == this->have_ignored) {
2359  add_ignored_info_obj_header();
2360  have_ignored = true;
2361  }
2362 
2363  string ignored_HDF5_named_dtype_hdr = "\n******WARNING******";
2364  ignored_HDF5_named_dtype_hdr += "\n IGNORED HDF5 named datatype objects:\n";
2365  string ignored_HDF5_named_dtype_msg = " Group name: " + grp_name + " HDF5 named datatype name: " + named_dtype_name.substr(0,named_dtype_name.size()-1)
2366  + "\n";
2367  if (ignored_msg.find(ignored_HDF5_named_dtype_hdr) == string::npos)
2368  ignored_msg += ignored_HDF5_named_dtype_hdr + ignored_HDF5_named_dtype_msg;
2369  else
2370  ignored_msg += ignored_HDF5_named_dtype_msg;
2371 
2372 }
2373 
2374 // Add the ignored attribute information. When is_grp is true, the ignored group attribute names are added.
2375 // Otherwise, the ignored dataset attribute names are added.
2376 void File::add_ignored_info_attrs(bool is_grp, const string & obj_path, const string & attr_name)
2377 {
2378 
2379  if (false == this->have_ignored) {
2380  add_ignored_info_obj_header();
2381  have_ignored = true;
2382  }
2383 
2384 
2385  string ignored_warning_str = "\n******WARNING******";
2386  string ignored_HDF5_grp_hdr = ignored_warning_str + "\n Ignored attributes under root and groups:\n";
2387  string ignored_HDF5_grp_msg = " Group path: " + obj_path + " Attribute names: " + attr_name + "\n";
2388  string ignored_HDF5_var_hdr = ignored_warning_str + "\n Ignored attributes for variables:\n";
2389  string ignored_HDF5_var_msg = " Variable path: " + obj_path + " Attribute names: " + attr_name + "\n";
2390 
2391 
2392  if (true == is_grp) {
2393  if (ignored_msg.find(ignored_HDF5_grp_hdr) == string::npos)
2394  ignored_msg += ignored_HDF5_grp_hdr + ignored_HDF5_grp_msg;
2395  else
2396  ignored_msg += ignored_HDF5_grp_msg;
2397  }
2398  else {
2399  if (ignored_msg.find(ignored_HDF5_var_hdr) == string::npos)
2400  ignored_msg += ignored_HDF5_var_hdr + ignored_HDF5_var_msg;
2401  else
2402  ignored_msg += ignored_HDF5_var_msg;
2403  }
2404 
2405 }
2406 
2407 //Ignored object information. When is_dim_related is true, ignored data space info. is present.
2408 //When is_dim_related is false, ignored data type info. is present.
2409 void File::add_ignored_info_objs(bool is_dim_related, const string & obj_path)
2410 {
2411 
2412  if (false == this->have_ignored) {
2413  add_ignored_info_obj_header();
2414  have_ignored = true;
2415  }
2416 
2417  string ignored_warning_str = "\n******WARNING******";
2418  string ignored_HDF5_dtype_var_hdr = ignored_warning_str + "\n IGNORED variables due to unsupported datatypes:\n";
2419  string ignored_HDF5_dspace_var_hdr = ignored_warning_str + "\n IGNORED variables due to unsupported dimensions:\n";
2420  string ignored_HDF5_var_msg = " Variable path: " + obj_path + "\n";
2421 
2422  if (true == is_dim_related) {
2423  if (ignored_msg.find(ignored_HDF5_dspace_var_hdr) == string::npos)
2424  ignored_msg += ignored_HDF5_dspace_var_hdr + ignored_HDF5_var_msg;
2425  else
2426  ignored_msg += ignored_HDF5_var_msg;
2427 
2428  }
2429  else {
2430  if (ignored_msg.find(ignored_HDF5_dtype_var_hdr) == string::npos)
2431  ignored_msg += ignored_HDF5_dtype_var_hdr + ignored_HDF5_var_msg;
2432  else
2433  ignored_msg += ignored_HDF5_var_msg;
2434  }
2435 
2436 }
2437 
2438 // No ignored info.
2439 void File::add_no_ignored_info()
2440 {
2441 
2442  ignored_msg += "There are no ignored HDF5 objects or attributes.";
2443 
2444 }
2445 
2446 // This function should only be used when the HDF5 file is following the netCDF data model.
2447 // Check if we should not report the Dimension scale related attributes as ignored.
2448 bool File::ignored_dimscale_ref_list(Var *var)
2449 {
2450 
2451  bool ignored_dimscale = true;
2452  // Only when "General_Product == this->product_type && GENERAL_DIMSCALE== this->gproduct_pattern)"
2453 
2454  bool has_dimscale = false;
2455  bool has_reference_list = false;
2456  for (vector<Attribute *>::iterator ira = var->attrs.begin(); ira != var->attrs.end(); ira++) {
2457  if ((*ira)->name == "REFERENCE_LIST" && false == HDF5CFUtil::cf_strict_support_type((*ira)->getType()))
2458  has_reference_list = true;
2459  if ((*ira)->name == "CLASS") {
2460  Retrieve_H5_Attr_Value(*ira, var->fullpath);
2461  string class_value;
2462  class_value.resize((*ira)->value.size());
2463  copy((*ira)->value.begin(), (*ira)->value.end(), class_value.begin());
2464 
2465  // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
2466  // "DIMENSION_SCALE", which is 15.
2467  if (0 == class_value.compare(0, 15, "DIMENSION_SCALE")) {
2468  has_dimscale = true;
2469  }
2470  }
2471 
2472  if (true == has_dimscale && true == has_reference_list) {
2473  ignored_dimscale = false;
2474  break;
2475  }
2476 
2477  }
2478  //}
2479  return ignored_dimscale;
2480 }
2481 
2482 // Check if the long string can should be dropped from a dataset or an attribute. Users can set up a BES key to turn it off or on.
2483 bool File::Check_DropLongStr(Var *var, Attribute * attr)
2484 {
2485 
2486  bool drop_longstr = false;
2487  if (NULL == attr) {
2488  if (H5FSTRING == var->dtype || H5VSTRING == var->dtype) {
2489  try {
2490  drop_longstr = Check_VarDropLongStr(var->fullpath, var->dims, var->dtype);
2491  }
2492  catch (...) {
2493  throw1("Check_VarDropLongStr fails ");
2494  }
2495  }
2496  }
2497  // No limitation for the attributes. KY 2018-02-26
2498 #if 0
2499  else {
2500  if (H5FSTRING == attr->dtype || H5VSTRING == attr->dtype) {
2501  if (attr->getBufSize() > NC_JAVA_STR_SIZE_LIMIT) {
2502  drop_longstr = true;
2503  }
2504  }
2505 
2506  }
2507 #endif
2508  return drop_longstr;
2509 }
2510 
2511 // Check if a long string dataset should be dropped. Users can turn on a BES key not to drop the long string.
2512 // However, the Java clients may not access.
2513 //
2514 bool File::Check_VarDropLongStr(const string & varpath, const vector<Dimension *>& dims, H5DataType dtype) const
2515 
2516 {
2517 
2518  bool drop_longstr = false;
2519 
2520  hid_t dset_id = H5Dopen2(this->fileid, varpath.c_str(), H5P_DEFAULT);
2521  if (dset_id < 0)
2522  throw2("Cannot open the dataset ", varpath);
2523 
2524  hid_t dtype_id = -1;
2525  if ((dtype_id = H5Dget_type(dset_id)) < 0) {
2526  H5Dclose(dset_id);
2527  throw2("Cannot obtain the datatype of the dataset ", varpath);
2528  }
2529 
2530  size_t ty_size = H5Tget_size(dtype_id);
2531  if (ty_size == 0) {
2532  H5Tclose(dtype_id);
2533  H5Dclose(dset_id);
2534  throw2("Cannot obtain the datatype size of the dataset ", varpath);
2535  }
2536 
2537  if (H5FSTRING == dtype) { // Fixed-size, just check the number of elements.
2538  if (ty_size > NC_JAVA_STR_SIZE_LIMIT) drop_longstr = true;
2539  }
2540  else if (H5VSTRING == dtype) {
2541 
2542  unsigned long long total_elms = 1;
2543  if (dims.size() != 0) {
2544  for (unsigned int i = 0; i < dims.size(); i++)
2545  total_elms = total_elms * ((dims[i])->size);
2546  }
2547  vector<char> strval;
2548  strval.resize(total_elms * ty_size);
2549  hid_t read_ret = H5Dread(dset_id, dtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, (void*) &strval[0]);
2550  if (read_ret < 0) {
2551  H5Tclose(dtype_id);
2552  H5Dclose(dset_id);
2553  throw2("Cannot read the data of the dataset ", varpath);
2554  }
2555 
2556  vector<string> finstrval;
2557  finstrval.resize(total_elms);
2558  char*temp_bp = &strval[0];
2559  char*onestring = NULL;
2560  for (unsigned long long i = 0; i < total_elms; i++) {
2561  onestring = *(char**) temp_bp;
2562  if (onestring != NULL) {
2563  finstrval[i] = string(onestring);
2564  if(finstrval[i].size()>NC_JAVA_STR_SIZE_LIMIT) {
2565  drop_longstr = true;
2566  break;
2567  }
2568  }
2569  temp_bp += ty_size;
2570  }
2571 
2572  if (false == strval.empty()) {
2573  herr_t ret_vlen_claim;
2574  hid_t dspace_id = H5Dget_space(dset_id);
2575  if (dspace_id < 0) {
2576  H5Tclose(dtype_id);
2577  H5Dclose(dset_id);
2578  throw2("Cannot obtain the dataspace id.", varpath);
2579  }
2580  ret_vlen_claim = H5Dvlen_reclaim(dtype_id, dspace_id, H5P_DEFAULT, (void*) &strval[0]);
2581  if (ret_vlen_claim < 0) {
2582  H5Tclose(dtype_id);
2583  H5Sclose(dspace_id);
2584  H5Dclose(dset_id);
2585  throw2("Cannot reclaim the vlen space ", varpath);
2586  }
2587  if (H5Sclose(dspace_id) < 0) {
2588  H5Tclose(dtype_id);
2589  H5Dclose(dset_id);
2590  throw2("Cannot close the HDF5 data space.", varpath);
2591  }
2592  }
2593  }
2594  if (H5Tclose(dtype_id) < 0) {
2595  H5Dclose(dset_id);
2596  throw2("Cannot close the HDF5 data type.", varpath);
2597  }
2598  if (H5Dclose(dset_id) < 0)
2599  throw2("Cannot close the HDF5 data type.", varpath);
2600 
2601  return drop_longstr;
2602 }
2603 #if 0
2604 bool File::Check_VarDropLongStr(const string & varpath, const vector<Dimension *>& dims, H5DataType dtype)
2605 
2606 {
2607 
2608  bool drop_longstr = false;
2609 
2610  unsigned long long total_elms = 1;
2611  if (dims.size() != 0) {
2612  for (unsigned int i = 0; i < dims.size(); i++)
2613  total_elms = total_elms * ((dims[i])->size);
2614  }
2615 
2616  if (total_elms > NC_JAVA_STR_SIZE_LIMIT)
2617  drop_longstr = true;
2618 
2619  else { // We need to check both fixed-size and variable-length strings.
2620 
2621  hid_t dset_id = H5Dopen2(this->fileid, varpath.c_str(), H5P_DEFAULT);
2622  if (dset_id < 0)
2623  throw2("Cannot open the dataset ", varpath);
2624 
2625  hid_t dtype_id = -1;
2626  if ((dtype_id = H5Dget_type(dset_id)) < 0) {
2627  H5Dclose(dset_id);
2628  throw2("Cannot obtain the datatype of the dataset ", varpath);
2629  }
2630 
2631  size_t ty_size = H5Tget_size(dtype_id);
2632  if (ty_size == 0) {
2633  H5Tclose(dtype_id);
2634  H5Dclose(dset_id);
2635  throw2("Cannot obtain the datatype size of the dataset ", varpath);
2636  }
2637 
2638  if (H5FSTRING == dtype) { // Fixed-size, just check the number of elements.
2639  if ((ty_size * total_elms) > NC_JAVA_STR_SIZE_LIMIT) drop_longstr = true;
2640  }
2641  else if (H5VSTRING == dtype) {
2642 
2643  vector<char> strval;
2644  strval.resize(total_elms * ty_size);
2645  hid_t read_ret = H5Dread(dset_id, dtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, (void*) &strval[0]);
2646  if (read_ret < 0) {
2647  H5Tclose(dtype_id);
2648  H5Dclose(dset_id);
2649  throw2("Cannot read the data of the dataset ", varpath);
2650  }
2651 
2652  vector<string> finstrval;
2653  finstrval.resize(total_elms);
2654  char*temp_bp = &strval[0];
2655  char*onestring = NULL;
2656  for (unsigned long long i = 0; i < total_elms; i++) {
2657  onestring = *(char**) temp_bp;
2658  if (onestring != NULL)
2659  finstrval[i] = string(onestring);
2660  else
2661  // We will add a NULL if onestring is NULL.
2662  finstrval[i] = "";
2663  temp_bp += ty_size;
2664  }
2665 
2666  if (false == strval.empty()) {
2667  herr_t ret_vlen_claim;
2668  hid_t dspace_id = H5Dget_space(dset_id);
2669  if (dspace_id < 0) {
2670  H5Tclose(dtype_id);
2671  H5Dclose(dset_id);
2672  throw2("Cannot obtain the dataspace id.", varpath);
2673  }
2674  ret_vlen_claim = H5Dvlen_reclaim(dtype_id, dspace_id, H5P_DEFAULT, (void*) &strval[0]);
2675  if (ret_vlen_claim < 0) {
2676  H5Tclose(dtype_id);
2677  H5Sclose(dspace_id);
2678  H5Dclose(dset_id);
2679  throw2("Cannot reclaim the vlen space ", varpath);
2680  }
2681  if (H5Sclose(dspace_id) < 0) {
2682  H5Tclose(dtype_id);
2683  H5Dclose(dset_id);
2684  throw2("Cannot close the HDF5 data space.", varpath);
2685  }
2686  }
2687  unsigned long long total_str_size = 0;
2688  for (unsigned long long i = 0; i < total_elms; i++) {
2689  total_str_size += finstrval[i].size();
2690  if (total_str_size > NC_JAVA_STR_SIZE_LIMIT) {
2691  drop_longstr = true;
2692  break;
2693  }
2694  }
2695  }
2696  if (H5Tclose(dtype_id) < 0) {
2697  H5Dclose(dset_id);
2698  throw2("Cannot close the HDF5 data type.", varpath);
2699  }
2700  if (H5Dclose(dset_id) < 0)
2701  throw2("Cannot close the HDF5 data type.", varpath);
2702  }
2703  return drop_longstr;
2704 }
2705 #endif
2706 
2707 
2708 // Provide if the long string is dropped.
2709 void File::add_ignored_grp_longstr_info(const string& grp_path, const string & attr_name)
2710 {
2711 
2712  ignored_msg += "The HDF5 group: " + grp_path + " has an empty-set string attribute: " + attr_name + "\n";
2713 
2714  return;
2715 }
2716 
2717 // Provide if the long variable string is dropped.
2718 void File::add_ignored_var_longstr_info(Var *var, Attribute *attr)
2719 {
2720 
2721  if (NULL == attr)
2722  ignored_msg += "String variable: " + var->fullpath + " value is set to empty.\n";
2723  else {
2724  ignored_msg += "The variable: " + var->fullpath + " has an empty-set string attribute: " + attr->name + "\n";
2725 
2726  }
2727  return;
2728 }
2729 
2730 // The warnings of the drop of the long string header
2731 void File::add_ignored_droplongstr_hdr()
2732 {
2733 
2734  if (false == this->have_ignored) this->have_ignored = true;
2735  string hdr = "\n\n The values of the following string variables ";
2736  hdr += " are set to empty because at least one string size in this variable exceeds netCDF Java string limit(32767 bytes).\n";
2737  hdr += "To obtain the values, change the BES key H5.EnableDropLongString=true at the handler BES";
2738  hdr += " configuration file(h5.conf)\nto H5.EnableDropLongString=false.\n\n";
2739 
2740  if (ignored_msg.rfind(hdr) == string::npos) ignored_msg += hdr;
2741 
2742 }
2743 
2744 // Sometimes, we need to release the temporary added resources.
2745 void File::release_standalone_var_vector(vector<Var*>&temp_vars)
2746 {
2747 
2748  for (vector<Var *>::iterator i = temp_vars.begin(); i != temp_vars.end();) {
2749  delete (*i);
2750  i = temp_vars.erase(i);
2751  }
2752 
2753 }
This class specifies the core engineering of mapping HDF5 to DAP by following CF.
#define throw1(a1)
The followings are convenient functions to throw exceptions with different.
Definition: HDF5CF.h:128
include the entry functions to execute the handlers
This class represents one attribute.
Definition: HDF5CF.h:189
This class repersents one dimension of an HDF5 dataset(variable).
Definition: HDF5CF.h:145
std::vector< Group * > groups
Non-root group vectors.
Definition: HDF5CF.h:805
virtual void Retrieve_H5_Var_Attr_Values(Var *var)
Retrieve attribute values for a variable.
Definition: HDF5CF.cc:745
virtual void Handle_Unsupported_Dspace(bool)
Handle unsupported HDF5 dataspaces for datasets.
Definition: HDF5CF.cc:1257
std::map< hsize_t, std::string > dimsize_to_fakedimname
Handle added dimension names.
Definition: HDF5CF.h:822
virtual void Handle_Grid_Mapping_Vars()
Handle Grid Mapping Vars.
Definition: HDF5CF.cc:2133
virtual void Handle_Unsupported_Others(bool)
Handle other unmapped objects/attributes.
Definition: HDF5CF.cc:1304
virtual void Retrieve_H5_Supported_Attr_Values()
Retrieve attribute values for the supported HDF5 datatypes.
Definition: HDF5CF.cc:726
std::vector< Var * > vars
Var vectors.
Definition: HDF5CF.h:799
virtual void Add_Supplement_Attrs(bool)
Add supplemental attributes such as fullpath and original name.
Definition: HDF5CF.cc:1952
virtual void Retrieve_H5_Info(const char *path, hid_t file_id, bool)
Definition: HDF5CF.cc:170
std::vector< Attribute * > root_attrs
Root attribute vectors.
Definition: HDF5CF.h:802
virtual void Handle_Unsupported_Dtype(bool)
Handle unsupported HDF5 datatypes.
Definition: HDF5CF.cc:914
virtual void Flatten_Obj_Name(bool)
Flatten the object name.
Definition: HDF5CF.cc:1354
virtual bool Have_Grid_Mapping_Attrs()
Check if having Grid Mapping Attrs.
Definition: HDF5CF.cc:2113
This class represents an HDF5 group. The group will be flattened according to the CF conventions.
Definition: HDF5CF.h:552
This class represents one HDF5 dataset(CF variable)
Definition: HDF5CF.h:259
Helper functions for generating DAS attributes and a function to check BES Key.
static H5DataType H5type_to_H5DAPtype(hid_t h5_type_id)
Map HDF5 Datatype to the intermediate H5DAPtype for the future use.
Definition: HDF5CFUtil.cc:54
static std::string trim_string(hid_t dtypeid, const std::string s, int num_sect, size_t section_size, std::vector< size_t > &sect_newsize)
Definition: HDF5CFUtil.cc:229