bes  Updated for version 3.20.8
HDF5RequestHandler.cc
Go to the documentation of this file.
1 
2 // -*- mode: c++; c-basic-offset:4 -*-
3 
4 // This file is part of hdf5_handler, a data handler for the OPeNDAP data
5 // server.
6 
7 // Copyright (c) 2002,2003 OPeNDAP, Inc.
8 // Copyright (c) 2007-2016 The HDF Group, Inc. and OPeNDAP, Inc.
9 // Author: James Gallagher <jgallagher@opendap.org>
10 //
11 // This is free software; you can redistribute it and/or modify it under the
12 // terms of the GNU Lesser General Public License as published by the Free
13 // Software Foundation; either version 2.1 of the License, or (at your
14 // option) any later version.
15 //
16 // This software is distributed in the hope that it will be useful, but
17 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
18 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
19 // License for more details.
20 //
21 // You should have received a copy of the GNU Lesser General Public
22 // License along with this library; if not, write to the Free Software
23 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24 //
25 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
26 
27 // HDF5RequestHandler.cc
33 
34 
35 #include <iostream>
36 #include <fstream>
37 #include <sstream>
38 #include <string>
39 #include <sys/types.h>
40 #include <sys/stat.h>
41 #include <fcntl.h>
42 #include <unistd.h>
43 
44 #include <DMR.h>
45 #include <D4BaseTypeFactory.h>
46 #include <BESDMRResponse.h>
47 #include <ObjMemCache.h>
48 #include "HDF5_DMR.h"
49 
50 #include <mime_util.h>
51 #include "hdf5_handler.h"
52 #include "HDF5RequestHandler.h"
53 #include "HDF5_DDS.h"
54 
55 #include <BESDASResponse.h>
56 #include <BESDDSResponse.h>
57 #include <BESDataDDSResponse.h>
58 #include <Ancillary.h>
59 #include <BESInfo.h>
60 #include <BESDapNames.h>
61 #include <BESResponseNames.h>
62 #include <BESContainer.h>
63 #include <BESResponseHandler.h>
64 #include <BESVersionInfo.h>
65 #include <BESServiceRegistry.h>
66 #include <BESUtil.h>
67 #include <BESDapError.h>
68 #include <BESInternalFatalError.h>
69 #include <TheBESKeys.h>
70 #include <BESDebug.h>
71 #include <BESStopWatch.h>
72 #include "h5get.h"
73 #include "config_hdf5.h"
74 
75 #define HDF5_NAME "h5"
76 #include "h5cfdaputil.h"
77 
78 using namespace std;
79 using namespace libdap;
80 
81 #define prolog std::string("HDF5RequestHandler::").append(__func__).append("() - ")
82 
83 // The debug function to dump all the contents of a DAS table.
84 void get_attr_contents(AttrTable* temp_table);
85 
86 // Five functions to generate a DAS cache file.
87 // 1. The wrapper function to call write_das_table_to_file to generate the cache.
88 void write_das_to_file(DAS*das_ptr,FILE* das_file);
89 
90 // 2. The main function to generate the DAS cache.
91 void write_das_table_to_file(AttrTable*temp_table,FILE* das_file);
92 
93 // 3. The function to write the DAS container/table name to the cache
94 void write_container_name_to_file(const string&,FILE* das_file);
95 
96 // 4. The function to write the DAS attribute name,type and values to the cache
97 void write_das_attr_info(AttrTable* dtp,const string&,const string&,FILE * das_file);
98 
99 // 5. The function to copy a string to a memory buffer.
100 char* copy_str(char *temp_ptr,const string & str);
101 
102 // These two functions are for reading the DAS from the DAS cache file
103 // 1. Obtain the string from a memory buffer.
104 char* obtain_str(char*temp_ptr,string & str);
105 
106 // 2. The main function to obtain DAS info from the cache.
107 char* get_attr_info_from_dc(char*temp_pointer,DAS *das,AttrTable *at);
108 
109 // Check if the BES key is set.
110 bool check_beskeys(const string);
111 
112 // Obtain the BES key as an integer
113 static unsigned int get_uint_key(const string &key,unsigned int def_val);
114 static unsigned long get_ulong_key(const string &key,unsigned long def_val);
115 
116 // Obtain the BES key as a floating-pointer number.
117 static float get_float_key(const string &key, float def_val);
118 
119 // Obtain the BES key as a string.
120 static string get_beskeys(const string&);
121 
122 // For the CF option
123 extern void read_cfdas(DAS &das, const string & filename,hid_t fileid);
124 extern void read_cfdds(DDS &dds, const string & filename,hid_t fileid);
125 
126 
127 // Check the description of cache_entries and cache_purge_level at h5.conf.in.
128 unsigned int HDF5RequestHandler::_mdcache_entries = 500;
129 unsigned int HDF5RequestHandler::_lrdcache_entries = 0;
130 unsigned int HDF5RequestHandler::_srdcache_entries = 0;
131 float HDF5RequestHandler::_cache_purge_level = 0.2;
132 
133 // Metadata object cache at DAS,DDS and DMR.
134 ObjMemCache *HDF5RequestHandler::das_cache = 0;
135 ObjMemCache *HDF5RequestHandler::dds_cache = 0;
136 ObjMemCache *HDF5RequestHandler::datadds_cache = 0;
137 ObjMemCache *HDF5RequestHandler::dmr_cache = 0;
138 
139 ObjMemCache *HDF5RequestHandler::lrdata_mem_cache = 0;
140 ObjMemCache *HDF5RequestHandler::srdata_mem_cache = 0;
141 
142 // Set default values of all BES keys be false.
143 bool HDF5RequestHandler::_usecf = false;
144 bool HDF5RequestHandler::_pass_fileid = false;
145 bool HDF5RequestHandler::_disable_structmeta = false;
146 bool HDF5RequestHandler::_disable_ecsmeta = false;
147 bool HDF5RequestHandler::_keep_var_leading_underscore = false;
148 bool HDF5RequestHandler::_check_name_clashing = false;
149 bool HDF5RequestHandler::_add_path_attrs = false;
150 bool HDF5RequestHandler::_drop_long_string = false;
151 bool HDF5RequestHandler::_fillvalue_check = false;
152 bool HDF5RequestHandler::_check_ignore_obj = false;
153 bool HDF5RequestHandler::_flatten_coor_attr = false;
154 bool HDF5RequestHandler::_default_handle_dimension = false;
155 bool HDF5RequestHandler::_eos5_rm_convention_attr_path = true;
156 bool HDF5RequestHandler::_dmr_long_int = false;
157 bool HDF5RequestHandler::_no_zero_size_fullnameattr = false;
158 bool HDF5RequestHandler::_enable_coord_attr_add_path = false;
159 
160 bool HDF5RequestHandler::_common_cache_dirs = false;
161 
162 bool HDF5RequestHandler::_use_disk_cache =false;
163 bool HDF5RequestHandler::_use_disk_dds_cache =false;
164 string HDF5RequestHandler::_disk_cache_dir ="";
165 string HDF5RequestHandler::_disk_cachefile_prefix ="";
166 unsigned long long HDF5RequestHandler::_disk_cache_size =0;
167 
168 
169 bool HDF5RequestHandler::_disk_cache_comp_data =false;
170 bool HDF5RequestHandler::_disk_cache_float_only_comp_data =false;
171 float HDF5RequestHandler::_disk_cache_comp_threshold =1.0;
172 unsigned long HDF5RequestHandler::_disk_cache_var_size =0;
173 
174 bool HDF5RequestHandler::_use_disk_meta_cache = false;
175 string HDF5RequestHandler::_disk_meta_cache_path ="";
176 
177 bool HDF5RequestHandler::_use_latlon_disk_cache = false;
178 long HDF5RequestHandler::_latlon_disk_cache_size =0;
179 string HDF5RequestHandler::_latlon_disk_cache_dir ="";
180 string HDF5RequestHandler::_latlon_disk_cachefile_prefix="";
181 
182 DMR* HDF5RequestHandler::dmr_int64 = 0;
183 
184 
185 #if 0
186 //BaseTypeFactory factory;
187 //libdap::DDS HDF5RequestHandler::hd_dds(&factory,"");
188 #endif
189 string HDF5RequestHandler::_stp_east_filename;
190 string HDF5RequestHandler::_stp_north_filename;
191 vector<string> HDF5RequestHandler::lrd_cache_dir_list;
192 vector<string> HDF5RequestHandler::lrd_non_cache_dir_list;
193 vector<string> HDF5RequestHandler::lrd_var_cache_file_list;
194 
195 #if 0
196 //libdap::DDS*cache_dds;
197 #endif
198 
199 HDF5RequestHandler::HDF5RequestHandler(const string & name)
200  :BESRequestHandler(name)
201 {
202 
203  BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
204 
205  add_handler(DAS_RESPONSE, HDF5RequestHandler::hdf5_build_das);
206  add_handler(DDS_RESPONSE, HDF5RequestHandler::hdf5_build_dds);
207  add_handler(DATA_RESPONSE, HDF5RequestHandler::hdf5_build_data);
208  add_handler(DMR_RESPONSE, HDF5RequestHandler::hdf5_build_dmr);
209  add_handler(DAP4DATA_RESPONSE, HDF5RequestHandler::hdf5_build_dmr);
210 
211  add_handler(HELP_RESPONSE, HDF5RequestHandler::hdf5_build_help);
212  add_handler(VERS_RESPONSE, HDF5RequestHandler::hdf5_build_version);
213 
214  load_config();
215 
216  BESDEBUG(HDF5_NAME, prolog << "END" << endl);
217 }
218 
219 HDF5RequestHandler::~HDF5RequestHandler()
220 {
221  // delete the cache.
222  delete das_cache;
223  delete dds_cache;
224  delete datadds_cache;
225  delete dmr_cache;
226  delete lrdata_mem_cache;
227  delete srdata_mem_cache;
228 
229 }
230 
234 void HDF5RequestHandler::load_config()
235 {
236  BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
237  BESStopWatch sw;
238  if (BESISDEBUG(HDF5_NAME)){
239  sw.start(prolog,"ClockTheBESKeys");
240  }
241 
242  // Obtain the metadata cache entries and purge level.
243  HDF5RequestHandler::_mdcache_entries = get_uint_key("H5.MetaDataMemCacheEntries", 0);
244  HDF5RequestHandler::_lrdcache_entries = get_uint_key("H5.LargeDataMemCacheEntries", 0);
245  HDF5RequestHandler::_srdcache_entries = get_uint_key("H5.SmallDataMemCacheEntries", 0);
246  HDF5RequestHandler::_cache_purge_level = get_float_key("H5.CachePurgeLevel", 0.2);
247 
248  if (get_mdcache_entries()) { // else it stays at its default of null
249  das_cache = new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
250  dds_cache = new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
251  datadds_cache = new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
252  dmr_cache = new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
253  }
254 
255  // Check if the EnableCF key is set.
256  _usecf = check_beskeys("H5.EnableCF");
257  BESDEBUG(HDF5_NAME, prolog << "H5.EnableCF: " << (_usecf?"true":"false") << endl);
258 
259  // The following keys are only effective when usecf is true.
260  _pass_fileid = check_beskeys("H5.EnablePassFileID");
261  _disable_structmeta = check_beskeys("H5.DisableStructMetaAttr");
262  _disable_ecsmeta = check_beskeys("H5.DisableECSMetaAttr");
263  _keep_var_leading_underscore = check_beskeys("H5.KeepVarLeadingUnderscore");
264  _check_name_clashing = check_beskeys("H5.EnableCheckNameClashing");
265  _add_path_attrs = check_beskeys("H5.EnableAddPathAttrs");
266  _drop_long_string = check_beskeys("H5.EnableDropLongString");
267  _fillvalue_check = check_beskeys("H5.EnableFillValueCheck");
268  _check_ignore_obj = check_beskeys("H5.CheckIgnoreObj");
269  _flatten_coor_attr = check_beskeys("H5.ForceFlattenNDCoorAttr");
270  _default_handle_dimension = check_beskeys("H5.DefaultHandleDimension");
271  _eos5_rm_convention_attr_path= check_beskeys("H5.RmConventionAttrPath");
272  _dmr_long_int = check_beskeys("H5.EnableDMR64bitInt");
273  _no_zero_size_fullnameattr = check_beskeys("H5.NoZeroSizeFullnameAttr");
274  _enable_coord_attr_add_path = check_beskeys("H5.EnableCoorattrAddPath");
275 
276  _use_disk_cache = check_beskeys("H5.EnableDiskDataCache");
277  _disk_cache_dir = get_beskeys("H5.DiskCacheDataPath");
278  _disk_cachefile_prefix = get_beskeys("H5.DiskCacheFilePrefix");
279  _disk_cache_size = get_ulong_key("H5.DiskCacheSize",0);
280 
281  _disk_cache_comp_data = check_beskeys("H5.DiskCacheComp");
282  _disk_cache_float_only_comp_data = check_beskeys("H5.DiskCacheFloatOnlyComp");
283  _disk_cache_comp_threshold = get_float_key("H5.DiskCacheCompThreshold",1.0);
284  _disk_cache_var_size = 1024*get_uint_key("H5.DiskCacheCompVarSize",0);
285 
286  _use_disk_meta_cache = check_beskeys("H5.EnableDiskMetaDataCache");
287  _use_disk_dds_cache = check_beskeys("H5.EnableDiskDDSCache");
288  _disk_meta_cache_path = get_beskeys("H5.DiskMetaDataCachePath");
289 
290  _use_latlon_disk_cache = check_beskeys("H5.EnableEOSGeoCacheFile");
291  _latlon_disk_cache_size = get_uint_key("H5.Cache.latlon.size",0);
292  _latlon_disk_cache_dir = get_beskeys("H5.Cache.latlon.path");
293  _latlon_disk_cachefile_prefix= get_beskeys("H5.Cache.latlon.prefix");
294 
295  if(get_usecf()) {
296  if(get_lrdcache_entries()) {
297  lrdata_mem_cache = new ObjMemCache(get_lrdcache_entries(), get_cache_purge_level());
298  if(true == check_beskeys("H5.LargeDataMemCacheConfig")) {
299  _common_cache_dirs =obtain_lrd_common_cache_dirs();
300 #if 0
301  if(false == _common_cache_dirs)
302 cerr<<"No specific cache info"<<endl;
303 #endif
304 
305  }
306  }
307  if(get_srdcache_entries()) {
308 
309  BESDEBUG(HDF5_NAME, prolog << "Generate memory cache for smaller coordinate variables" << endl);
310  srdata_mem_cache = new ObjMemCache(get_srdcache_entries(),get_cache_purge_level());
311 
312  }
313 
314  if(_disk_cache_comp_data == true && _use_disk_cache == true) {
315  if(_disk_cache_comp_threshold < 1.0) {
316  ostringstream ss;
317  ss<< _disk_cache_comp_threshold;
318  string _comp_threshold_str(ss.str());
319  string invalid_comp_threshold ="The Compression Threshold is the total size of the variable array";
320  invalid_comp_threshold+=" divided by the storage size of compressed array. It should always be >1";
321  invalid_comp_threshold+=" The current threhold set at h5.conf is ";
322  invalid_comp_threshold+=_comp_threshold_str;
323  invalid_comp_threshold+=" . Go back to h5.conf and change the H5.DiskCacheCompThreshold to a >1.0 number.";
324  throw BESInternalError(invalid_comp_threshold,__FILE__,__LINE__);
325  }
326  }
327  _stp_east_filename = get_beskeys("H5.STPEastFileName");
328  _stp_north_filename = get_beskeys("H5.STPNorthFileName");
329  }
330  BESDEBUG(HDF5_NAME, prolog << "END" << endl);
331 }
332 
333 
334 // Build DAS
335 bool HDF5RequestHandler::hdf5_build_das(BESDataHandlerInterface & dhi)
336 {
337  BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
338 #if DYNAMIC_CONFIG_ENABLED
339  load_config();
340 #endif
341 
342  // For the time being, separate CF file ID from the default file ID(mainly for debugging)
343  hid_t cf_fileid = -1;
344 
345  // Obtain the HDF5 file name.
346  string filename = dhi.container->access();
347 
348  // Obtain the BES object from the client
349  BESResponseObject *response = dhi.response_handler->get_response_object() ;
350 
351  // Convert to the BES DAS response
352  BESDASResponse *bdas = dynamic_cast < BESDASResponse * >(response) ;
353  if( !bdas )
354  throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
355 
356  try {
357  bdas->set_container( dhi.container->get_symbolic_name() ) ;
358  DAS *das = bdas->get_das();
359 
360  // Look inside the memory cache to see if it's initialized
361  DAS *cached_das_ptr = 0;
362  bool use_das_cache = false;
363  if (das_cache)
364  cached_das_ptr = static_cast<DAS*>(das_cache->get(filename));
365  if (cached_das_ptr)
366  use_das_cache = true;
367 
368  if (true == use_das_cache) {
369 
370  // copy the cached DAS into the BES response object
371  BESDEBUG(HDF5_NAME, prolog << "DAS Cached hit for : " << filename << endl);
372  *das = *cached_das_ptr;
373  }
374  else {
375 
376  bool das_from_dc = false;
377  string das_cache_fname;
378 
379  // If the use_disk_meta_cache is set, check if the cache file exists and sets the flag.
380  if(_use_disk_meta_cache == true) {
381 
382  string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
383  das_cache_fname = _disk_meta_cache_path+"/" +base_filename+"_das";
384 
385  if(access(das_cache_fname.c_str(),F_OK) !=-1)
386  das_from_dc = true;
387 
388  }
389 
390  // If reading DAS from the disk cache, call the corresponding function
391  if(true == das_from_dc) {
392  read_das_from_disk_cache(das_cache_fname,das);
393 
394  // If the memory cache is set, adding the DAS copy to the memory cache
395  if (das_cache) {
396  // add a copy
397  BESDEBUG(HDF5_NAME, prolog << "HDF5 DAS reading DAS from the disk cache. For memory cache, DAS added to the cache for : " << filename << endl);
398  das_cache->add(new DAS(*das), filename);
399  }
400  }
401 
402  else {// Need to build from the HDF5 file
403  H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
404  if (true == _usecf) {//CF option
405 
406  cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
407  if (cf_fileid < 0){
408  string invalid_file_msg="Could not open this HDF5 file ";
409  invalid_file_msg +=filename;
410  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
411  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
412  invalid_file_msg +=" distributor.";
413  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
414  }
415  // Need to check if DAP4 DMR CF 64-bit integer mapping is on.
416  if(HDF5RequestHandler::get_dmr_64bit_int()!=NULL)
417  HDF5RequestHandler::set_dmr_64bit_int(NULL);
418  read_cfdas( *das,filename,cf_fileid);
419  H5Fclose(cf_fileid);
420  }
421  else {// Default option
422  hid_t fileid = get_fileid(filename.c_str());
423  if (fileid < 0) {
424  string invalid_file_msg="Could not open this HDF5 file ";
425  invalid_file_msg +=filename;
426  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
427  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
428  invalid_file_msg +=" distributor.";
429  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
430 
431  }
432 
433  find_gloattr(fileid, *das);
434  depth_first(fileid, "/", *das);
435  close_fileid(fileid);
436  }
437 
438  Ancillary::read_ancillary_das( *das, filename ) ;
439 
440 #if 0
441 // Dump all attribute contents
442 AttrTable* top_table = das->get_top_level_attributes();
443 get_attr_contents(top_table);
444 
445 // Dump all variable contents
446 AttrTable::Attr_iter start_aiter=das->var_begin();
447 AttrTable::Attr_iter it = start_aiter;
448 AttrTable::Attr_iter end_aiter = das->var_end();
449 while(it != end_aiter) {
450 AttrTable* temp_table = das->get_table(it);
451 if(temp_table!=0){
452 cerr<<"var_begin"<<endl;
453 temp_table->print(cerr);
454 }
455 ++it;
456 }
457 #endif
458  // If the memory cache is turned on
459  if(das_cache) {
460  // add a copy
461  BESDEBUG(HDF5_NAME, prolog << "DAS added to the cache for : " << filename << endl);
462  das_cache->add(new DAS(*das), filename);
463  }
464 
465  // DAS disk cache fname will be set only when the metadata disk cache is turned on
466  // So if it comes here, the das cache should be generated.
467  if(das_cache_fname!="") {
468  BESDEBUG(HDF5_NAME, prolog << "HDF5 Build DAS: Write DAS to disk cache " << das_cache_fname << endl);
469  write_das_to_disk_cache(das_cache_fname,das);
470  }
471  }
472  }
473 
474  bdas->clear_container() ;
475  }
476  catch(BESError & e) {
477  if(cf_fileid !=-1)
478  H5Fclose(cf_fileid);
479  BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
480  throw;
481  }
482  catch(InternalErr & e) {
483 
484  if(cf_fileid !=-1)
485  H5Fclose(cf_fileid);
486  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
487  __FILE__, __LINE__);
488  }
489  catch(Error & e) {
490 
491  if(cf_fileid !=-1)
492  H5Fclose(cf_fileid);
493  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
494  __FILE__, __LINE__);
495  }
496  catch(...) {
497 
498  if(cf_fileid !=-1)
499  H5Fclose(cf_fileid);
500  string s = "unknown exception caught building HDF5 DAS";
501  throw BESInternalFatalError(s, __FILE__, __LINE__);
502  }
503 
504  BESDEBUG(HDF5_NAME, prolog << "END" << endl);
505  return true;
506 }
507 
508 // Convenient function that helps build DDS and Data
509 // Since this function will be used by both the DDS and Data services, we need to pass both BESDDSResponse and BESDataDDSResponse.
510 // This two parameters are necessary for the future DDS disk cache feature.
511 void HDF5RequestHandler::get_dds_with_attributes( BESDDSResponse*bdds,BESDataDDSResponse*data_bdds,const string &container_name, const string& filename,const string &dds_cache_fname, const string &das_cache_fname,bool dds_from_dc,bool das_from_dc, bool build_data)
512 {
513  DDS *dds;
514  if(true == build_data)
515  dds = data_bdds->get_dds();
516  else dds = bdds->get_dds();
517 
518  // For the time being, separate CF file ID from the default file ID(mainly for debugging)
519  hid_t fileid = -1;
520  hid_t cf_fileid = -1;
521 
522  try {
523 
524  // Look in memory cache to see if it's initialized
525  DDS* cached_dds_ptr = 0;
526  bool use_dds_cache = false;
527  if (dds_cache)
528  cached_dds_ptr = static_cast<DDS*>(dds_cache->get(filename));
529  if (cached_dds_ptr)
530  use_dds_cache = true;
531  if (true == use_dds_cache) {
532  // copy the cached DDS into the BES response object. Assume that any cached DDS
533  // includes the DAS information.
534  BESDEBUG(HDF5_NAME, prolog << "DDS Metadata Cached hit for : " << filename << endl);
535  *dds = *cached_dds_ptr; // Copy the referenced object
536  }
537  else if (true ==dds_from_dc) {//Currently the dds_from_ds is always false by default.
538  read_dds_from_disk_cache(bdds,data_bdds,build_data,container_name,filename,dds_cache_fname,das_cache_fname,-1,das_from_dc);
539  }
540  else {
541  BESDEBUG(HDF5_NAME, prolog << "Build DDS from the HDF5 file. " << filename << endl);
542  H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
543  dds->filename(filename);
544 
545  // For the time being, not mess up CF's fileID with Default's fileID
546  if(true == _usecf) {
547 
548  cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
549  if (cf_fileid < 0){
550  string invalid_file_msg="Could not open this HDF5 file ";
551  invalid_file_msg +=filename;
552  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
553  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
554  invalid_file_msg +=" distributor.";
555  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
556  }
557  // The following is for DAP4 CF(DMR) 64-bit mapping, we need to set the flag
558  // to let the handler map the 64-bit integer.
559  if(HDF5RequestHandler::get_dmr_64bit_int() != NULL)
560  HDF5RequestHandler::set_dmr_64bit_int(NULL);
561  read_cfdds(*dds,filename,cf_fileid);
562  }
563  else {
564 
565  fileid = get_fileid(filename.c_str());
566  if (fileid < 0) {
567  string invalid_file_msg="Could not open this HDF5 file ";
568  invalid_file_msg +=filename;
569  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
570  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
571  invalid_file_msg +=" distributor.";
572  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
573  }
574 
575  depth_first(fileid, (char*)"/", *dds, filename.c_str());
576 
577  }
578  // Check semantics
579  if (!dds->check_semantics()) { // DDS didn't comply with the semantics
580  dds->print(cerr);
581  throw InternalErr(__FILE__, __LINE__,
582  "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
583  }
584 
585  Ancillary::read_ancillary_dds( *dds, filename ) ;
586 
587  // Generate the DDS cached file if needed,currently this is always false by default
588  if(dds_cache_fname!="" && dds_from_dc == false)
589  write_dds_to_disk_cache(dds_cache_fname,dds);
590 
591  // Add attributes
592  {
593  hid_t h5_fd = -1;
594  if(_usecf == true)
595  h5_fd = cf_fileid;
596  else
597  h5_fd = fileid;
598  add_das_to_dds(dds,container_name,filename,das_cache_fname,h5_fd,das_from_dc);
599  }
600 
601  // Add memory cache if possible
602  if (dds_cache) {
603  // add a copy
604  BESDEBUG(HDF5_NAME, prolog << "DDS added to the cache for : " << filename << endl);
605  dds_cache->add(new DDS(*dds), filename);
606  }
607 
608  if(cf_fileid != -1)
609  H5Fclose(cf_fileid);
610  if(fileid != -1)
611  H5Fclose(fileid);
612 
613  }
614 
615  }
616  catch(InternalErr & e) {
617 
618  if(cf_fileid !=-1)
619  H5Fclose(cf_fileid);
620 
621  if(fileid != -1)
622  H5Fclose(fileid);
623 
624  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
625  __FILE__, __LINE__);
626  }
627  catch(Error & e) {
628 
629  if(cf_fileid !=-1)
630  H5Fclose(cf_fileid);
631  if(fileid !=-1)
632  H5Fclose(fileid);
633 
634  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
635  __FILE__, __LINE__);
636  }
637  catch(...) {
638 
639  if(cf_fileid !=-1)
640  H5Fclose(cf_fileid);
641  if(fileid !=-1)
642  H5Fclose(fileid);
643 
644  string s = "unknown exception caught building HDF5 DDS";
645  throw BESInternalFatalError(s, __FILE__, __LINE__);
646  }
647 
648 }
649 
650 void HDF5RequestHandler::get_dds_without_attributes_datadds(BESDataDDSResponse*data_bdds,const string &container_name, const string& filename)
651 {
652  DDS *dds = data_bdds->get_dds();
653 
654  // For the time being, separate CF file ID from the default file ID(mainly for debugging)
655  hid_t fileid = -1;
656  hid_t cf_fileid = -1;
657 
658  try {
659 
660  // Look in memory cache to see if it's initialized
661  DDS* cached_dds_ptr = 0;
662  bool use_datadds_cache = false;
663  if (datadds_cache)
664  cached_dds_ptr = static_cast<DDS*>(datadds_cache->get(filename));
665  if (cached_dds_ptr)
666  use_datadds_cache = true;
667  if (true == use_datadds_cache) {
668  // copy the cached DDS into the BES response object.
669  // The DAS information is not included.
670  BESDEBUG(HDF5_NAME, prolog << "DataDDS Metadata Cached hit for : " << filename << endl);
671  *dds = *cached_dds_ptr; // Copy the referenced object
672  }
673  else {
674  BESDEBUG(HDF5_NAME, prolog << "Build DDS from the HDF5 file. " << filename << endl);
675  H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
676  dds->filename(filename);
677 
678  // For the time being, not mess up CF's fileID with Default's fileID
679  if(true == _usecf) {
680 
681  cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
682  if (cf_fileid < 0){
683  string invalid_file_msg="Could not open this HDF5 file ";
684  invalid_file_msg +=filename;
685  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
686  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
687  invalid_file_msg +=" distributor.";
688  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
689  }
690  // The following is for DAP4 CF(DMR) 64-bit mapping, we need to set the flag
691  // to let the handler map the 64-bit integer.
692  if(HDF5RequestHandler::get_dmr_64bit_int() != NULL)
693  HDF5RequestHandler::set_dmr_64bit_int(NULL);
694  read_cfdds(*dds,filename,cf_fileid);
695  }
696  else {
697 
698  fileid = get_fileid(filename.c_str());
699  if (fileid < 0) {
700  string invalid_file_msg="Could not open this HDF5 file ";
701  invalid_file_msg +=filename;
702  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
703  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
704  invalid_file_msg +=" distributor.";
705  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
706  }
707 
708  depth_first(fileid, (char*)"/", *dds, filename.c_str());
709 
710  }
711  // Check semantics
712  if (!dds->check_semantics()) { // DDS didn't comply with the semantics
713  dds->print(cerr);
714  throw InternalErr(__FILE__, __LINE__,
715  "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
716  }
717 
718  Ancillary::read_ancillary_dds( *dds, filename ) ;
719 
720 #if 0
721  // Add attributes
722  {
723  hid_t h5_fd = -1;
724  if(_usecf == true)
725  h5_fd = cf_fileid;
726  else
727  h5_fd = fileid;
728  add_das_to_dds(dds,container_name,filename,das_cache_fname,h5_fd,das_from_dc);
729  }
730 #endif
731 
732  // Add memory cache if possible
733  if (datadds_cache) {
734  // add a copy
735  BESDEBUG(HDF5_NAME, prolog << "DataDDS added to the cache for : " << filename << endl);
736  datadds_cache->add(new DDS(*dds), filename);
737  }
738 
739  if(cf_fileid != -1)
740  H5Fclose(cf_fileid);
741  if(fileid != -1)
742  H5Fclose(fileid);
743 
744  }
745  BESDEBUG(HDF5_NAME, prolog << "Data ACCESS build_data(): set the including attribute flag to false: "<<filename << endl);
746  data_bdds->set_ia_flag(false);
747 
748  }
749  catch(InternalErr & e) {
750 
751  if(cf_fileid !=-1)
752  H5Fclose(cf_fileid);
753 
754  if(fileid != -1)
755  H5Fclose(fileid);
756 
757  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
758  __FILE__, __LINE__);
759  }
760  catch(Error & e) {
761 
762  if(cf_fileid !=-1)
763  H5Fclose(cf_fileid);
764  if(fileid !=-1)
765  H5Fclose(fileid);
766 
767  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
768  __FILE__, __LINE__);
769  }
770  catch(...) {
771 
772  if(cf_fileid !=-1)
773  H5Fclose(cf_fileid);
774  if(fileid !=-1)
775  H5Fclose(fileid);
776 
777  string s = "unknown exception caught building HDF5 DDS";
778  throw BESInternalFatalError(s, __FILE__, __LINE__);
779  }
780 
781 }
782 
783 
784 #if 0
785 // OLD function: Keep it for a while
786 // Convenient function that helps build DDS and Data
787 void HDF5RequestHandler::get_dds_with_attributes(const string &filename, const string &container_name, DDS*dds) {
788 
789  // For the time being, separate CF file ID from the default file ID(mainly for debugging)
790  hid_t fileid = -1;
791  hid_t cf_fileid = -1;
792 
793  try {
794 
795  // Look in memory cache to see if it's initialized
796  DDS* cached_dds_ptr = 0;
797  if (dds_cache && (cached_dds_ptr = static_cast<DDS*>(dds_cache->get(filename)))) {
798  // copy the cached DDS into the BES response object. Assume that any cached DDS
799  // includes the DAS information.
800  BESDEBUG(HDF5_NAME, prolog << "DDS Cached hit for : " << filename << endl);
801  *dds = *cached_dds_ptr; // Copy the referenced object
802  }
803 
804  else {
805 
806  H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
807  if (!container_name.empty())
808  dds->container_name(container_name);
809  dds->filename(filename);
810 
811  // For the time being, not mess up CF's fileID with Default's fileID
812  if(true == _usecf) {
813 // This block cannot be used to cache the data
814 #if 0
815  string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
816  string dds_filename = "/tmp/"+base_filename+"_dds";
817  FILE *dds_file = fopen(dds_filename.c_str(),"r");
818 cerr<<"before parsing "<<endl;
819 BaseTypeFactory tf;
820 DDS tdds(&tf,name_path(filename),"3.2");
821 tdds.filename(filename);
822  //dds->parse(dds_file);
823  tdds.parse(dds_file);
824  //DDS *cache_dds = new DDS(tdds);
825  cache_dds = new DDS(tdds);
826 if(dds!=NULL)
827  delete dds;
828 dds = cache_dds;
829 tdds.print(cout);
830 dds->print(cout);
831 cerr<<"after parsing "<<endl;
832 //dds->print(cout);
833  // fclose(dds_file);
834 //#endif
835 
836 #endif
837 // end of this block
838 
839 
840  cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
841  if (cf_fileid < 0){
842  string invalid_file_msg="Could not open this HDF5 file ";
843  invalid_file_msg +=filename;
844  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
845  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
846  invalid_file_msg +=" distributor.";
847  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
848  }
849 //#if 0
850  read_cfdds(*dds,filename,cf_fileid);
851 //#endif
852  // Generate the DDS cached file
853  string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
854  string dds_filename = "/tmp/"+base_filename+"_dds";
855  FILE *dds_file = fopen(dds_filename.c_str(),"w");
856  dds->print(dds_file);
857  fclose(dds_file);
858 
859  }
860  else {
861 
862  fileid = get_fileid(filename.c_str());
863  if (fileid < 0) {
864  string invalid_file_msg="Could not open this HDF5 file ";
865  invalid_file_msg +=filename;
866  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
867  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
868  invalid_file_msg +=" distributor.";
869  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
870  }
871 
872  depth_first(fileid, (char*)"/", *dds, filename.c_str());
873 
874  }
875 
876 
877  // Check semantics
878  if (!dds->check_semantics()) { // DDS didn't comply with the semantics
879  dds->print(cerr);
880  throw InternalErr(__FILE__, __LINE__,
881  "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
882  }
883 
884  Ancillary::read_ancillary_dds( *dds, filename ) ;
885 
886 
887  // Check DAS cache
888  DAS *das = 0 ;
889 
890  if (das_cache && (das = static_cast<DAS*>(das_cache->get(filename)))) {
891  BESDEBUG(HDF5_NAME, prolog << "DAS Cached hit for : " << filename << endl);
892  dds->transfer_attributes(das); // no need to copy the cached DAS
893  }
894 
895  else {
896 
897  das = new DAS ;
898 
899  if (!container_name.empty())
900  das->container_name(container_name);
901 
902  if (true == _usecf) {
903 
904  // go to the CF option
905  read_cfdas( *das,filename,cf_fileid);
906 
907  }
908  else {
909 
910  find_gloattr(fileid, *das);
911  depth_first(fileid, "/", *das);
912  close_fileid(fileid);
913  }
914 
915  if(cf_fileid != -1)
916  H5Fclose(cf_fileid);
917 
918  Ancillary::read_ancillary_das( *das, filename ) ;
919 
920  dds->transfer_attributes(das);
921 
922 
923  // Only free the DAS if it's not added to the cache
924  if (das_cache) {
925  // add a copy
926  BESDEBUG(HDF5_NAME, prolog << "DAS added to the cache for : " << filename << endl);
927  //das_cache->add(new DAS(*das), filename);
928  das_cache->add(das, filename);
929  }
930  else {
931  delete das;
932  }
933  }
934 
935  if (dds_cache) {
936  // add a copy
937  BESDEBUG(HDF5_NAME, prolog << "DDS added to the cache for : " << filename << endl);
938  dds_cache->add(new DDS(*dds), filename);
939  }
940 
941  }
942 
943 //dds->print(cout);
944 
945  }
946  catch(InternalErr & e) {
947 
948  if(cf_fileid !=-1)
949  H5Fclose(cf_fileid);
950 
951  if(fileid != -1)
952  H5Fclose(fileid);
953 
954  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
955  __FILE__, __LINE__);
956  }
957  catch(Error & e) {
958 
959  if(cf_fileid !=-1)
960  H5Fclose(cf_fileid);
961  if(fileid !=-1)
962  H5Fclose(fileid);
963 
964  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
965  __FILE__, __LINE__);
966  }
967  catch(...) {
968 
969  if(cf_fileid !=-1)
970  H5Fclose(cf_fileid);
971  if(fileid !=-1)
972  H5Fclose(fileid);
973 
974  string s = "unknown exception caught building HDF5 DDS";
975  throw BESInternalFatalError(s, __FILE__, __LINE__);
976  }
977 
978 }
979 #endif
980 
981 // Build DDS
982 bool HDF5RequestHandler::hdf5_build_dds(BESDataHandlerInterface & dhi)
983 {
984  BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
985 #if DYNAMIC_CONFIG_ENABLED
986  load_config();
987 #endif
988 
989  // Obtain the HDF5 file name.
990  string filename = dhi.container->access();
991 
992  string container_name = dhi.container->get_symbolic_name();
993  BESResponseObject *response = dhi.response_handler->get_response_object();
994  BESDDSResponse *bdds = dynamic_cast < BESDDSResponse * >(response);
995  if( !bdds )
996  throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
997  bdds->set_container(container_name);
998 
999  try {
1000 
1001  bool dds_from_dc = false;
1002  bool das_from_dc = false;
1003  bool build_data = false;
1004  string dds_cache_fname;
1005  string das_cache_fname;
1006 
1007  if(_use_disk_meta_cache) {
1008 
1009  string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
1010 
1011  // The _use_disk_dds_cache is always set to false by default
1012  if(_use_disk_dds_cache) {
1013  dds_cache_fname = _disk_meta_cache_path+"/" +base_filename+"_dds";
1014  if(access(dds_cache_fname.c_str(),F_OK) !=-1)
1015  dds_from_dc = true;
1016  }
1017 
1018  das_cache_fname = _disk_meta_cache_path+"/" +base_filename+"_das";
1019  // Check if das files exist
1020  if(access(das_cache_fname.c_str(),F_OK) !=-1)
1021  das_from_dc = true;
1022 
1023  }
1024 
1025  get_dds_with_attributes(bdds, NULL,container_name,filename, dds_cache_fname,das_cache_fname,dds_from_dc,das_from_dc,build_data);
1026 
1027  // The following block reads dds from a dds cache file.
1028 #if 0
1029  string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
1030  string dds_filename = "/tmp/"+base_filename+"_dds";
1031 
1032  BaseTypeFactory tf;
1033  DDS tdds(&tf,name_path(filename),"3.2");
1034  tdds.filename(filename);
1035 
1036 
1037  FILE *dds_file = fopen(dds_filename.c_str(),"r");
1038  tdds.parse(dds_file);
1039 //cerr<<"before parsing "<<endl;
1040  DDS* cache_dds = new DDS(tdds);
1041  if(dds != NULL)
1042  delete dds;
1043  bdds->set_dds(cache_dds);
1044  fclose(dds_file);
1045 #endif
1046 
1047  bdds->set_constraint( dhi ) ;
1048  bdds->clear_container() ;
1049 
1050  }
1051  catch(BESError & e) {
1052  BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
1053  throw;
1054  }
1055  catch(InternalErr & e) {
1056 
1057  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1058  __FILE__, __LINE__);
1059  }
1060  catch(Error & e) {
1061 
1062  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1063  __FILE__, __LINE__);
1064  }
1065  catch(...) {
1066 
1067  string s = "unknown exception caught building HDF5 DDS";
1068  throw BESInternalFatalError(s, __FILE__, __LINE__);
1069  }
1070 
1071  BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1072  return true;
1073 }
1074 
1075 bool HDF5RequestHandler::hdf5_build_data(BESDataHandlerInterface & dhi)
1076 {
1077  BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
1078 #if DYNAMIC_CONFIG_ENABLED
1079  load_config();
1080 #endif
1081 
1082  if(true ==_usecf) {
1083 
1084  if(true == _pass_fileid)
1085  return hdf5_build_data_with_IDs(dhi);
1086  }
1087 
1088  string filename = dhi.container->access();
1089 
1090  string container_name = dhi.container->get_symbolic_name();
1091  BESResponseObject *response = dhi.response_handler->get_response_object();
1092  BESDataDDSResponse *bdds = dynamic_cast < BESDataDDSResponse * >(response);
1093  if( !bdds )
1094  throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
1095  bdds->set_container(container_name);
1096 
1097  try {
1098 
1099  bool dds_from_dc = false;
1100  bool das_from_dc = false;
1101  bool build_data = true;
1102  string dds_cache_fname;
1103  string das_cache_fname;
1104 
1105 
1106  // Only DAS is read from the cache. dds_from_dc is always false.
1107  if(_use_disk_meta_cache == true) {
1108 
1109  string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
1110  das_cache_fname = _disk_meta_cache_path+"/" +base_filename+"_das";
1111 
1112  if(access(das_cache_fname.c_str(),F_OK) !=-1)
1113  das_from_dc = true;
1114 
1115  }
1116 
1117  //get_dds_with_attributes(NULL,bdds, container_name,filename, dds_cache_fname,das_cache_fname,dds_from_dc,das_from_dc,build_data);
1118  get_dds_without_attributes_datadds(bdds,container_name,filename);
1119 
1120  bdds->set_constraint( dhi ) ;
1121  bdds->clear_container() ;
1122 
1123  }
1124  catch(BESError & e) {
1125  BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
1126  throw;
1127  }
1128  catch(InternalErr & e) {
1129 
1130  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1131  __FILE__, __LINE__);
1132  }
1133  catch(Error & e) {
1134 
1135  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1136  __FILE__, __LINE__);
1137  }
1138  catch(...) {
1139 
1140  string s = "unknown exception caught building HDF5 DDS";
1141  throw BESInternalFatalError(s, __FILE__, __LINE__);
1142  }
1143 
1144  BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1145  return true;
1146 }
1147 
1148 // Obtain data when turning on the pass fileID key.The memory cache is not used.
1149 bool HDF5RequestHandler::hdf5_build_data_with_IDs(BESDataHandlerInterface & dhi)
1150 {
1151  BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
1152 #if DYNAMIC_CONFIG_ENABLED
1153  load_config();
1154 #endif
1155 
1156  BESDEBUG(HDF5_NAME,prolog << "Building DataDDS by passing file IDs. "<<endl);
1157  hid_t cf_fileid = -1;
1158 
1159  string filename = dhi.container->access();
1160 
1161  H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
1162  cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1163  if (cf_fileid < 0){
1164  string invalid_file_msg="Could not open this HDF5 file ";
1165  invalid_file_msg +=filename;
1166  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1167  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1168  invalid_file_msg +=" distributor.";
1169  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1170  }
1171 
1172  BESResponseObject *response = dhi.response_handler->get_response_object();
1173  BESDataDDSResponse *bdds = dynamic_cast < BESDataDDSResponse * >(response);
1174  if( !bdds )
1175  throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
1176 
1177  try {
1178 
1179  bdds->set_container( dhi.container->get_symbolic_name() ) ;
1180 
1181  HDF5DDS *hdds = new HDF5DDS(bdds->get_dds());
1182  delete bdds->get_dds();
1183 
1184  bdds->set_dds(hdds);
1185 
1186  hdds->setHDF5Dataset(cf_fileid);
1187 
1188  read_cfdds( *hdds,filename,cf_fileid);
1189 
1190  if (!hdds->check_semantics()) { // DDS didn't comply with the DAP semantics
1191  hdds->print(cerr);
1192  throw InternalErr(__FILE__, __LINE__,
1193  "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1194  }
1195 
1196  Ancillary::read_ancillary_dds( *hdds, filename ) ;
1197 
1198  DAS *das = new DAS ;
1199  BESDASResponse bdas( das ) ;
1200  bdas.set_container( dhi.container->get_symbolic_name() ) ;
1201  read_cfdas( *das,filename,cf_fileid);
1202  Ancillary::read_ancillary_das( *das, filename ) ;
1203 
1204  hdds->transfer_attributes(das);
1205  bdds->set_constraint( dhi ) ;
1206  bdds->clear_container() ;
1207 
1208  }
1209 
1210  catch(BESError & e) {
1211  if(cf_fileid !=-1)
1212  H5Fclose(cf_fileid);
1213  BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
1214  throw;
1215  }
1216  catch(InternalErr & e) {
1217  if(cf_fileid !=-1)
1218  H5Fclose(cf_fileid);
1219  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1220  __FILE__, __LINE__);
1221  }
1222  catch(Error & e) {
1223  if(cf_fileid !=-1)
1224  H5Fclose(cf_fileid);
1225  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1226  __FILE__, __LINE__);
1227  }
1228  catch(...) {
1229  if(cf_fileid !=-1)
1230  H5Fclose(cf_fileid);
1231  string s = "unknown exception caught building HDF5 DataDDS";
1232  throw BESInternalFatalError(s, __FILE__, __LINE__);
1233  }
1234 
1235  BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1236  return true;
1237 }
1238 
1239 bool HDF5RequestHandler::hdf5_build_dmr(BESDataHandlerInterface & dhi)
1240 {
1241  BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
1242 #if DYNAMIC_CONFIG_ENABLED
1243  load_config();
1244 #endif
1245 
1246  // Extract the DMR Response object - this holds the DMR used by the
1247  // other parts of the framework.
1248  BESResponseObject *response = dhi.response_handler->get_response_object();
1249  BESDMRResponse &bes_dmr_response = dynamic_cast<BESDMRResponse &>(*response);
1250 
1251  string filename = dhi.container->access();
1252 
1253  DMR *dmr = bes_dmr_response.get_dmr();
1254 
1255  // For the time being, separate CF file ID from the default file ID(mainly for debugging)
1256  hid_t fileid = -1;
1257  hid_t cf_fileid = -1;
1258 
1259  try {
1260 
1261  DMR* cached_dmr_ptr = 0;
1262  if (dmr_cache){
1263  BESDEBUG(HDF5_NAME, prolog << "Checking DMR cache for : " << filename << endl);
1264  cached_dmr_ptr = static_cast<DMR*>(dmr_cache->get(filename));
1265  }
1266 
1267  if (cached_dmr_ptr) {
1268  // copy the cached DMR into the BES response object
1269  BESDEBUG(HDF5_NAME, prolog << "DMR cache hit for : " << filename << endl);
1270  *dmr = *cached_dmr_ptr; // Copy the referenced object
1271  dmr->set_request_xml_base(bes_dmr_response.get_request_xml_base());
1272  }
1273  else {// No cache
1274 
1275  H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
1276  D4BaseTypeFactory MyD4TypeFactory;
1277  dmr->set_factory(&MyD4TypeFactory);
1278 
1279  if(true ==_usecf) {// CF option
1280 
1281  if(true == _pass_fileid)
1282  return hdf5_build_dmr_with_IDs(dhi);
1283 
1284  cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1285  if (cf_fileid < 0){
1286  string invalid_file_msg="Could not open this HDF5 file ";
1287  invalid_file_msg +=filename;
1288  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1289  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1290  invalid_file_msg +=" distributor.";
1291  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1292  }
1293 
1294  BaseTypeFactory factory;
1295  DDS dds(&factory, name_path(filename), "3.2");
1296  dds.filename(filename);
1297 
1298  DAS das;
1299 
1300  // For the CF option dmr response, we need to map 64-bit integer separately
1301  // So set the flag to map 64-bit integer.
1302  HDF5RequestHandler::set_dmr_64bit_int(dmr);
1303  read_cfdds( dds,filename,cf_fileid);
1304  if (!dds.check_semantics()) { // DDS didn't comply with the DAP semantics
1305  dds.print(cerr);
1306  throw InternalErr(__FILE__, __LINE__,
1307  "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1308  }
1309 
1310  read_cfdas(das,filename,cf_fileid);
1311  Ancillary::read_ancillary_das( das, filename ) ;
1312 
1313  dds.transfer_attributes(&das);
1314 
1316  if(cf_fileid !=-1)
1317  H5Fclose(cf_fileid);
1318 
1319  dmr->build_using_dds(dds);
1320 
1321  }// "if(true == _usecf)"
1322  else {// default option
1323 
1324  // Obtain the HDF5 file ID.
1325  fileid = get_fileid(filename.c_str());
1326  if (fileid < 0) {
1327  string invalid_file_msg="Could not open this HDF5 file ";
1328  invalid_file_msg +=filename;
1329  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1330  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1331  invalid_file_msg +=" distributor.";
1332  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1333  }
1334 
1335  bool use_dimscale = false;
1336  if(true == _default_handle_dimension)
1337  use_dimscale = check_dimscale(fileid);
1338  dmr->set_name(name_path(filename));
1339  dmr->set_filename(name_path(filename));
1340 
1341  // The breadth_first() function builds the variables and attributes and
1342  // loads them into the root group (building child groups as needed).
1343  // jhrg 4/30/20
1344  D4Group* root_grp = dmr->root();
1345  BESDEBUG("h5", "use_dimscale is "<< use_dimscale <<endl);
1346  breadth_first(fileid,(char*)"/",root_grp,filename.c_str(),use_dimscale);
1347 #if 0
1348  BESDEBUG("h5", "build_dmr - before obtain dimensions"<< endl);
1349  D4Dimensions *root_dims = root_grp->dims();
1350  for(D4Dimensions::D4DimensionsIter di = root_dims->dim_begin(), de = root_dims->dim_end(); di != de; ++di) {
1351  BESDEBUG("fonc", "transform_dap4() - check dimensions"<< endl);
1352  BESDEBUG("fonc", "transform_dap4() - dim name is: "<<(*di)->name()<<endl);
1353  BESDEBUG("fonc", "transform_dap4() - dim size is: "<<(*di)->size()<<endl);
1354  BESDEBUG("fonc", "transform_dap4() - fully_qualfied_dim name is: "<<(*di)->fully_qualified_name()<<endl);
1355  //cout <<"dim size is: "<<(*di)->size()<<endl;
1356  //cout <<"dim fully_qualified_name is: "<<(*di)->fully_qualified_name()<<endl;
1357  }
1358  BESDEBUG("h5", "build_dmr - after obtain dimensions"<< endl);
1359 #endif
1360 
1361 #if 0
1362  if(true == use_dimscale)
1363  //breadth_first(fileid,(char*)"/",*dmr,root_grp,filename.c_str(),true);
1364  breadth_first(fileid,(char*)"/",root_grp,filename.c_str(),true);
1365  else
1366  depth_first(fileid,(char*)"/",root_grp,filename.c_str());
1367  //depth_first(fileid,(char*)"/",*dmr,root_grp,filename.c_str());
1368 #endif
1369 
1370  close_fileid(fileid);
1371 
1372  }// else (default option)
1373 
1374  // If the cache is turned on, add the memory cache.
1375  if (dmr_cache) {
1376  // add a copy
1377  BESDEBUG(HDF5_NAME, prolog << "DMR added to the cache for : " << filename << endl);
1378  dmr_cache->add(new DMR(*dmr), filename);
1379  }
1380  }// else no cache
1381  }// try
1382  catch(BESError & e) {
1383  if(cf_fileid !=-1)
1384  H5Fclose(cf_fileid);
1385  if(fileid !=-1)
1386  H5Fclose(fileid);
1387  BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
1388  throw;
1389  }
1390  catch(InternalErr & e) {
1391 
1392  if(cf_fileid !=-1)
1393  H5Fclose(cf_fileid);
1394  if(fileid !=-1)
1395  H5Fclose(fileid);
1396 
1397  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1398  __FILE__, __LINE__);
1399  }
1400  catch(Error & e) {
1401 
1402  if(cf_fileid !=-1)
1403  H5Fclose(cf_fileid);
1404  if(fileid !=-1)
1405  H5Fclose(fileid);
1406  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1407  __FILE__, __LINE__);
1408  }
1409  catch(...) {
1410 
1411  if(cf_fileid !=-1)
1412  H5Fclose(cf_fileid);
1413  if(fileid !=-1)
1414  H5Fclose(fileid);
1415  string s = "unknown exception caught building HDF5 DMR";
1416  throw BESInternalFatalError(s, __FILE__, __LINE__);
1417  }
1418 
1419  //dmr->print(cout);
1420 
1421  // Instead of fiddling with the internal storage of the DHI object,
1422  // (by setting dhi.data[DAP4_CONSTRAINT], etc., directly) use these
1423  // methods to set the constraints. But, why? Ans: from Patrick is that
1424  // in the 'container' mode of BES each container can have a different
1425  // CE.
1426  bes_dmr_response.set_dap4_constraint(dhi);
1427  bes_dmr_response.set_dap4_function(dhi);
1428  dmr->set_factory(0);
1429 
1430  BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1431  return true;
1432 }
1433 
1434 // This function is only used when EnableCF is true.
1435 bool HDF5RequestHandler::hdf5_build_dmr_with_IDs(BESDataHandlerInterface & dhi)
1436 {
1437  BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
1438 #if DYNAMIC_CONFIG_ENABLED
1439  load_config();
1440 #endif
1441 
1442  BESDEBUG("h5","Building DMR with passing file IDs. "<<endl);
1443  string filename = dhi.container->access();
1444  hid_t cf_fileid = -1;
1445 
1446  H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
1447  cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1448  if (cf_fileid < 0){
1449  string invalid_file_msg="Could not open this HDF5 file ";
1450  invalid_file_msg +=filename;
1451  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1452  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1453  invalid_file_msg +=" distributor.";
1454  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1455  }
1456 
1457  BaseTypeFactory factory;
1458  DDS dds(&factory, name_path(filename), "3.2");
1459  dds.filename(filename);
1460 
1461  DAS das;
1462 
1463  try {
1464 
1465 
1466  // This is the CF option
1467  read_cfdds( dds,filename,cf_fileid);
1468 
1469  if (!dds.check_semantics()) { // DDS didn't comply with the DAP semantics
1470  dds.print(cerr);
1471  throw InternalErr(__FILE__, __LINE__,
1472  "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1473  }
1474 
1475  Ancillary::read_ancillary_dds( dds, filename ) ;
1476 
1477 
1478  read_cfdas(das,filename,cf_fileid);
1479 
1480  Ancillary::read_ancillary_das( das, filename ) ;
1481 
1482  dds.transfer_attributes(&das);
1483 
1485  //if(cf_fileid !=-1)
1486  // H5Fclose(cf_fileid);
1487 
1488  }
1489  catch(BESError & e) {
1490  if(cf_fileid !=-1)
1491  H5Fclose(cf_fileid);
1492  BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
1493  throw;
1494  }
1495  catch(InternalErr & e) {
1496 
1497  if(cf_fileid !=-1)
1498  H5Fclose(cf_fileid);
1499 
1500  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1501  __FILE__, __LINE__);
1502  }
1503  catch(Error & e) {
1504 
1505  if(cf_fileid !=-1)
1506  H5Fclose(cf_fileid);
1507 
1508  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1509  __FILE__, __LINE__);
1510  }
1511  catch(...) {
1512 
1513  if(cf_fileid !=-1)
1514  H5Fclose(cf_fileid);
1515 
1516  string s = "unknown exception caught building HDF5 DataDDS";
1517  throw BESInternalFatalError(s, __FILE__, __LINE__);
1518  }
1519 
1520  // Extract the DMR Response object - this holds the DMR used by the
1521  // other parts of the framework.
1522  BESResponseObject *response = dhi.response_handler->get_response_object();
1523  BESDMRResponse &bes_dmr = dynamic_cast<BESDMRResponse &>(*response);
1524 
1525  // In this handler we use a different pattern since the handler specializes the DDS/DMR.
1526  // First, build the DMR adding the open handle to the HDF4 dataset, then free the DMR
1527  // the BES built and add this one. The HDF4DMR object will close the open dataset when
1528  // the BES runs the DMR's destructor.
1529 
1530  DMR *dmr = bes_dmr.get_dmr();
1531  D4BaseTypeFactory MyD4TypeFactory;
1532  dmr->set_factory(&MyD4TypeFactory);
1533  dmr->build_using_dds(dds);
1534 
1535  HDF5DMR *hdf5_dmr = new HDF5DMR(dmr);
1536  hdf5_dmr->setHDF5Dataset(cf_fileid);
1537  delete dmr; // The call below will make 'dmr' unreachable; delete it now to avoid a leak.
1538  bes_dmr.set_dmr(hdf5_dmr); // BESDMRResponse will delete hdf5_dmr
1539 
1540  // Instead of fiddling with the internal storage of the DHI object,
1541  // (by setting dhi.data[DAP4_CONSTRAINT], etc., directly) use these
1542  // methods to set the constraints. But, why? Ans: from Patrick is that
1543  // in the 'container' mode of BES each container can have a different
1544  // CE.
1545  bes_dmr.set_dap4_constraint(dhi);
1546  bes_dmr.set_dap4_function(dhi);
1547  hdf5_dmr->set_factory(0);
1548 
1549  BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1550  return true;
1551 }
1552 
1553 bool HDF5RequestHandler::hdf5_build_help(BESDataHandlerInterface & dhi)
1554 {
1555  BESResponseObject *response = dhi.response_handler->get_response_object();
1556  BESInfo *info = dynamic_cast<BESInfo *>(response);
1557  if( !info )
1558  throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
1559 
1560  string add_info="Just for Test";
1561 
1562  map<string,string> attrs ;
1563  attrs["name"] = MODULE_NAME ;
1564  attrs["version"] = MODULE_VERSION ;
1565  list<string> services ;
1566  BESServiceRegistry::TheRegistry()->services_handled( HDF5_NAME, services );
1567  if( services.size() > 0 )
1568  {
1569  string handles = BESUtil::implode( services, ',' ) ;
1570  attrs["handles"] = handles ;
1571  }
1572  info->begin_tag( "module", &attrs ) ;
1573  info->end_tag( "module" ) ;
1574  info->add_data(add_info);
1575 
1576  return true;
1577 }
1578 
1579 bool HDF5RequestHandler::hdf5_build_version(BESDataHandlerInterface & dhi)
1580 {
1581  BESResponseObject *response = dhi.response_handler->get_response_object();
1582  BESVersionInfo *info = dynamic_cast < BESVersionInfo * >(response);
1583  if( !info )
1584  throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
1585 
1586  info->add_module( MODULE_NAME, MODULE_VERSION ) ;
1587 
1588  return true;
1589 }
1590 
1591 
1592 bool HDF5RequestHandler::obtain_lrd_common_cache_dirs()
1593 {
1594  string lrd_config_fpath;
1595  string lrd_config_fname;
1596 
1597  // Obtain DataCache path
1598  lrd_config_fpath = get_beskeys("H5.DataCachePath");
1599 
1600  // Obtain the configure file name that specifics the large file configuration
1601  lrd_config_fname = get_beskeys("H5.LargeDataMemCacheFileName");
1602 
1603  // If either the configure file path or fname is missing, won't add specific mem. cache dirs.
1604  if(lrd_config_fpath=="" || lrd_config_fname=="")
1605  return false;
1606 
1607  // temp_line for storing info of one line in the config. file
1608  string temp_line;
1609 
1610  // The full path of the configure file
1611  string mcache_config_fname = lrd_config_fpath+"/"+lrd_config_fname;
1612 
1613  //ifstream mcache_config_file("example.txt");
1614  // Open the configure file
1615  ifstream mcache_config_file(mcache_config_fname.c_str());
1616 
1617  // If the configuration file is not open, return false.
1618  if(mcache_config_file.is_open()==false){
1619  BESDEBUG(HDF5_NAME, prolog << "The large data memory cache configure file "<<mcache_config_fname );
1620  BESDEBUG(HDF5_NAME, prolog << " cannot be opened."<<endl);
1621  return false;
1622  }
1623 
1624  // Read the configuration file line by line
1625  while(getline(mcache_config_file,temp_line)) {
1626 
1627  // Only consider lines that is no less than 2 characters and the 2nd character is space.
1628  if(temp_line.size()>1 && temp_line.at(1)==' ') {
1629  char sep=' ';
1630  string subline = temp_line.substr(2);
1631  vector<string> temp_name_list;
1632 
1633  // Include directories to store common latitude and longitude values
1634  if(temp_line.at(0)=='1') {
1635  HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1636  //lrd_cache_dir_list +=temp_name_list;
1637  lrd_cache_dir_list.insert(lrd_cache_dir_list.end(),temp_name_list.begin(),temp_name_list.end());
1638  }
1639  // Include directories not to store common latitude and longitude values
1640  else if(temp_line.at(0)=='0'){
1641  HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1642  //lrd_non_cache_dir_list +=temp_name_list;
1643  lrd_non_cache_dir_list.insert(lrd_non_cache_dir_list.end(),temp_name_list.begin(),temp_name_list.end());
1644  }
1645  // Include variable names that the server would like to store in the memory cache
1646  else if(temp_line.at(0)=='2') {
1647 
1648  // We need to handle the space case inside a variable path
1649  // either "" or '' needs to be used to identify a var path
1650  vector<int>dq_pos;
1651  vector<int>sq_pos;
1652  for(unsigned int i = 0; i<subline.size();i++){
1653  if(subline[i]=='"') {
1654  dq_pos.push_back(i);
1655  }
1656  else if(subline[i]=='\'')
1657  sq_pos.push_back(i);
1658  }
1659  if(dq_pos.size()==0 && sq_pos.size()==0)
1660  HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1661  else if((dq_pos.size()!=0) &&(dq_pos.size()%2==0)&& sq_pos.size()==0) {
1662  unsigned int dq_index= 0;
1663  while(dq_index < dq_pos.size()){
1664  if(dq_pos[dq_index+1]>(dq_pos[dq_index]+1)) {
1665  temp_name_list.push_back
1666  (subline.substr(dq_pos[dq_index]+1,dq_pos[dq_index+1]-dq_pos[dq_index]-1));
1667  }
1668  dq_index = dq_index + 2;
1669  }
1670  }
1671  else if((sq_pos.size()!=0) &&(sq_pos.size()%2==0)&& dq_pos.size()==0) {
1672  unsigned int sq_index= 0;
1673  while(sq_index < sq_pos.size()){
1674  if(sq_pos[sq_index+1]>(sq_pos[sq_index]+1)) {
1675  temp_name_list.push_back
1676  (subline.substr(sq_pos[sq_index]+1,sq_pos[sq_index+1]-sq_pos[sq_index]-1));
1677  }
1678  sq_index = sq_index+2;
1679  }
1680  }
1681 
1682  lrd_var_cache_file_list.insert(lrd_var_cache_file_list.end(),temp_name_list.begin(),temp_name_list.end());
1683  }
1684  }
1685  }
1686 
1687 
1688 #if 0
1689 
1690 for(int i =0; i<lrd_cache_dir_list.size();i++)
1691 cerr<<"lrd cache list is "<<lrd_cache_dir_list[i] <<endl;
1692 for(int i =0; i<lrd_non_cache_dir_list.size();i++)
1693 cerr<<"lrd non cache list is "<<lrd_non_cache_dir_list[i] <<endl;
1694 for(int i =0; i<lrd_var_cache_file_list.size();i++)
1695 cerr<<"lrd var cache file list is "<<lrd_var_cache_file_list[i] <<endl;
1696 #endif
1697 
1698 
1699  mcache_config_file.close();
1700  if(lrd_cache_dir_list.size()==0 && lrd_non_cache_dir_list.size()==0 && lrd_var_cache_file_list.size()==0)
1701  return false;
1702  else
1703  return true;
1704 }
1705 
1706 
1707 bool HDF5RequestHandler::read_das_from_disk_cache(const string & cache_filename,DAS *das_ptr) {
1708 
1709  BESDEBUG(HDF5_NAME, prolog << "Coming to read_das_from_disk_cache() " << cache_filename << endl);
1710  bool ret_value = true;
1711  FILE *md_file = NULL;
1712  md_file = fopen(cache_filename.c_str(),"rb");
1713 
1714  if(NULL == md_file) {
1715  string bes_error = "An error occurred trying to open a metadata cache file " + cache_filename;
1716  throw BESInternalError( bes_error, __FILE__, __LINE__);
1717  }
1718  else {
1719 
1720  int fd_md = fileno(md_file);
1721  struct flock *l_md;
1722  l_md = lock(F_RDLCK);
1723 
1724  // hold a read(shared) lock to read metadata from a file.
1725  if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
1726  fclose(md_file);
1727  ostringstream oss;
1728  oss << "cache process: " << l_md->l_pid << " triggered a locking error: " << get_errno();
1729  throw BESInternalError( oss.str(), __FILE__, __LINE__);
1730  }
1731 
1732  try {
1733 
1734  struct stat sb;
1735  if(stat(cache_filename.c_str(),&sb) != 0) {
1736  string bes_error = "An error occurred trying to stat a metadata cache file size " + cache_filename;
1737  throw BESInternalError( bes_error, __FILE__, __LINE__);
1738 
1739  }
1740 
1741 
1742  size_t bytes_expected_read=(size_t)sb.st_size;
1743  BESDEBUG(HDF5_NAME, prolog << "DAS Disk cache file size is " << bytes_expected_read << endl);
1744 
1745  vector<char> buf;
1746  buf.resize(bytes_expected_read);
1747  size_t bytes_to_read =fread((void*)&buf[0],1,bytes_expected_read,md_file);
1748  if(bytes_to_read != bytes_expected_read)
1749  throw InternalErr(__FILE__,__LINE__,"Fail to read the data from the das cache file.");
1750 
1751  char* temp_pointer =&buf[0];
1752 
1753  AttrTable*at = NULL;
1754  // recursively build DAS
1755 //#if 0
1756  temp_pointer = get_attr_info_from_dc(temp_pointer,das_ptr,at);
1757 //#endif
1758 
1759 
1760  }
1761  catch(...) {
1762  if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1763  fclose(md_file);
1764  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1765  }
1766 
1767  fclose(md_file);
1768  throw InternalErr(__FILE__,__LINE__,"Fail to parse a das cache file.");
1769  }
1770 
1771  // Unlock the cache file
1772  if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1773  fclose(md_file);
1774  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1775  }
1776  fclose(md_file);
1777  }
1778  return ret_value;
1779 
1780 }
1781 
1782 // This fucntion will NOT be used by default. Leave here for future improvement.
1783 bool HDF5RequestHandler::write_dds_to_disk_cache(const string& dds_cache_fname,DDS *dds_ptr) {
1784 
1785  BESDEBUG(HDF5_NAME, prolog << "Write DDS to disk cache " << dds_cache_fname << endl);
1786  FILE *dds_file = fopen(dds_cache_fname.c_str(),"w");
1787 
1788  if(NULL == dds_file) {
1789  string bes_error = "An error occurred trying to open a metadata cache file " + dds_cache_fname;
1790  throw BESInternalError( bes_error, __FILE__, __LINE__);
1791  }
1792  else {
1793 
1794  int fd_md = fileno(dds_file);
1795  struct flock *l_md;
1796  l_md = lock(F_WRLCK);
1797 
1798  // hold a read(shared) lock to read metadata from a file.
1799  if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
1800  fclose(dds_file);
1801  ostringstream oss;
1802  oss << "cache process: " << l_md->l_pid << " triggered a locking error: " << get_errno();
1803  throw BESInternalError( oss.str(), __FILE__, __LINE__);
1804  }
1805 
1806  try {
1807  dds_ptr->print(dds_file);
1808  }
1809  catch(...) {
1810  if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1811  fclose(dds_file);
1812  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1813  }
1814 
1815  fclose(dds_file);
1816  throw InternalErr(__FILE__,__LINE__,"Fail to parse a dds cache file.");
1817  }
1818 
1819  if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1820  fclose(dds_file);
1821  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1822  }
1823 
1824  fclose(dds_file);
1825  }
1826  return true;
1827 
1828 }
1829 
1830 // Write DAS to a binary cached file on disk.
1831 bool HDF5RequestHandler::write_das_to_disk_cache(const string & das_cache_fname, DAS *das_ptr) {
1832 
1833  BESDEBUG(HDF5_NAME, prolog << "Write DAS to disk cache " << das_cache_fname << endl);
1834  FILE *das_file = fopen(das_cache_fname.c_str(),"wb");
1835  if(NULL == das_file) {
1836  string bes_error = "An error occurred trying to open a metadata cache file " + das_cache_fname;
1837  throw BESInternalError( bes_error, __FILE__, __LINE__);
1838  }
1839  else {
1840  int fd_md = fileno(das_file);
1841  struct flock *l_md;
1842  l_md = lock(F_WRLCK);
1843 
1844  // hold a write(exclusive) lock to write metadata to a file.
1845  if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
1846  fclose(das_file);
1847  ostringstream oss;
1848  oss << "cache process: " << l_md->l_pid << " triggered a locking error: " << get_errno();
1849  throw BESInternalError( oss.str(), __FILE__, __LINE__);
1850  }
1851 
1852  try {
1853  write_das_to_file(das_ptr,das_file);
1854  }
1855  catch(...) {
1856  if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1857  fclose(das_file);
1858  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1859  }
1860 
1861  fclose(das_file);
1862  throw InternalErr(__FILE__,__LINE__,"Fail to parse a dds cache file.");
1863  }
1864 
1865  if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1866  fclose(das_file);
1867  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1868  }
1869 
1870  fclose(das_file);
1871 
1872  }
1873 
1874  return true;
1875 
1876 }
1877 
1878 // The wrapper function to call write_das_table_to_file to generate the cache.
1879 void write_das_to_file(DAS*das_ptr,FILE* das_file) {
1880 
1881  // When category_flag is 2, it marks the end of the file.
1882  uint8_t category_flag = 2;
1883  AttrTable* top_table = das_ptr->get_top_level_attributes();
1884  write_das_table_to_file(top_table,das_file);
1885 
1886  // Add the final ending flag for retrieving the info.
1887  fwrite((const void*)&category_flag,1,1,das_file);
1888  return;
1889 
1890 }
1891 
1892 // The main function to write DAS to a file
1893 void write_das_table_to_file(AttrTable*temp_table,FILE* das_file) {
1894 
1895  if(temp_table !=NULL) {
1896 
1897  // 2 is the end mark of an attribute table
1898  uint8_t category_flag = 2;
1899 
1900  // Loop through the whole DAS top table
1901  AttrTable::Attr_iter top_startit = temp_table->attr_begin();
1902  AttrTable::Attr_iter top_endit = temp_table->attr_end();
1903  AttrTable::Attr_iter top_it = top_startit;
1904  while(top_it !=top_endit) {
1905  AttrType atype = temp_table->get_attr_type(top_it);
1906  if(atype == Attr_unknown)
1907  throw InternalErr(__FILE__,__LINE__,"Unsupported DAS Attribute type");
1908  else if(atype!=Attr_container) {
1909  BESDEBUG(HDF5_NAME, prolog << "DAS to the disk cache, attr name is: "
1910  << temp_table->get_name(top_it) << endl);
1911  BESDEBUG(HDF5_NAME, prolog << "DAS to the disk cache, attr type is: "
1912  << temp_table->get_type(top_it) << endl);
1913  // For the debugging purpose
1914  //unsigned int num_attrs = temp_table->get_attr_num(temp_table->get_name(top_it));
1915  //cerr<<"Attribute values are "<<endl;
1916  //for (int i = 0; i <num_attrs;i++)
1917  // cerr<<(*(temp_table->get_attr_vector(temp_table->get_name(top_it))))[i]<<" ";
1918  //cerr<<endl;
1919  //write_das_attr_info(temp_table,top_it,das_file);
1920  // Write DAS attribute info to the file
1921  write_das_attr_info(temp_table,temp_table->get_name(top_it),temp_table->get_type(top_it),das_file);
1922  }
1923  else {
1924  BESDEBUG(HDF5_NAME, prolog << "DAS to the disk cache, attr container name is: "
1925  << (*top_it)->name << endl);
1926  // Write the container and then write the info. in this container
1927  AttrTable* sub_table = temp_table->get_attr_table(top_it);
1928  write_container_name_to_file(sub_table->get_name(),das_file);
1929  write_das_table_to_file(sub_table,das_file);
1930 
1931  // Write the end flag
1932  fwrite((const void*)&category_flag,1,1,das_file);
1933 
1934  }
1935  ++top_it;
1936  }
1937 
1938  }
1939 }
1940 
1941 // Write container name to the disk file
1942 void write_container_name_to_file(const string& cont_name,FILE *das_file) {
1943 
1944  // 1 marks the starting of a container
1945  uint8_t category_flag = 1;
1946  vector<char> buf;
1947  size_t bytes_to_write = cont_name.size()+sizeof(size_t)+1;
1948  buf.resize(bytes_to_write);
1949  char*temp_pointer =&buf[0];
1950  memcpy((void*)temp_pointer,(void*)&category_flag,1);
1951  temp_pointer++;
1952  temp_pointer=copy_str(temp_pointer,cont_name);
1953 
1954  size_t bytes_to_be_written = fwrite((const void*)&buf[0],1,bytes_to_write,das_file);
1955  if(bytes_to_be_written != bytes_to_write)
1956  throw InternalErr(__FILE__, __LINE__,"Failed to write a DAS container name to a cache");
1957  return;
1958 }
1959 
1960 
1961 // Write DAS attribute info. to the disk cache file
1962 void write_das_attr_info(AttrTable* dtp,const string& attr_name, const string & attr_type,FILE * das_file) {
1963 
1964  // 0 marks the starting of a DAS attribute
1965  uint8_t category_flag = 0;
1966 
1967  unsigned int num_attr_elems = dtp->get_attr_num(attr_name);
1968  vector<string> attr_values;
1969  size_t total_attr_values_size = 0;
1970  for (unsigned int i = 0; i <num_attr_elems;i++){
1971  attr_values.push_back((*(dtp->get_attr_vector(attr_name)))[i]);
1972  total_attr_values_size += attr_values[i].size();
1973  }
1974  // Need to add a flag, value as 0 to indicate the attribute.
1975  // DAS: category flag, sizeof attirubte name, attribute name, size of attribute type, attribute type
1976  size_t bytes_to_write_attr = 1 + attr_name.size() + attr_type.size() + 2* sizeof(size_t);
1977 
1978  // One unsigned int to store the number of element elements i
1979  // + sizeof(size_t) * number of elements to store the number of characters for each attribute
1980  // (in DAP, every attribute is in string format)
1981  // +total size of all attribute values
1982  bytes_to_write_attr += sizeof(unsigned int) + num_attr_elems*sizeof(size_t)+total_attr_values_size;
1983 
1984  vector<char>attr_buf;
1985  attr_buf.resize(bytes_to_write_attr);
1986  char* temp_attrp =&attr_buf[0];
1987 
1988  // The attribute flag
1989  memcpy((void*)temp_attrp,(void*)&category_flag,1);
1990  temp_attrp++;
1991 
1992  // The attribute name and type
1993  temp_attrp=copy_str(temp_attrp,attr_name);
1994  temp_attrp=copy_str(temp_attrp,attr_type);
1995 
1996  // Number of elements
1997  memcpy((void*)temp_attrp,(void*)&num_attr_elems,sizeof(unsigned int));
1998  temp_attrp+=sizeof(unsigned int);
1999 
2000  // All attributes
2001  for (unsigned int i = 0; i <num_attr_elems;i++)
2002  temp_attrp=copy_str(temp_attrp,(*(dtp->get_attr_vector(attr_name)))[i]);
2003 
2004  size_t bytes_to_be_written = fwrite((const void*)&attr_buf[0],1,bytes_to_write_attr,das_file);
2005  if(bytes_to_be_written != bytes_to_write_attr)
2006  throw InternalErr(__FILE__, __LINE__,"Failed to write a DAS attribute to a cache");
2007 
2008  return;
2009 
2010 }
2011 
2012 // Read DDS from a disk cache, this function is not used by default.
2013 void HDF5RequestHandler::read_dds_from_disk_cache(BESDDSResponse* bdds, BESDataDDSResponse* data_bdds,
2014  bool build_data,const string & container_name,const string & h5_fname,
2015  const string & dds_cache_fname,const string &das_cache_fname, hid_t h5_fd,
2016  bool das_from_dc) {
2017 
2018 
2019  BESDEBUG(HDF5_NAME, prolog << "BEGIN dds_cache_fname: " << dds_cache_fname << endl);
2020 
2021  DDS *dds;
2022  if(true == build_data)
2023  dds = data_bdds->get_dds();
2024  else
2025  dds = bdds->get_dds();
2026 
2027  // write a function to pass the following with the lock.
2028  BaseTypeFactory tf;
2029  DDS tdds(&tf,name_path(h5_fname),"3.2");
2030  tdds.filename(h5_fname);
2031 
2032  FILE *dds_file = fopen(dds_cache_fname.c_str(),"r");
2033  tdds.parse(dds_file);
2034  DDS* cache_dds = new DDS(tdds);
2035 #if 0
2036 cerr<<"before dds "<<endl;
2037 dds->dump(cerr);
2038 cerr<<"after dds "<<endl;
2039 cerr<<"before tdds "<<endl;
2040 cache_dds->dump(cerr);
2041 cerr<<"after tdds "<<endl;
2042 #endif
2043  if(dds != NULL)
2044  delete dds;
2045 
2046  Ancillary::read_ancillary_dds( *cache_dds, h5_fname ) ;
2047 
2048  add_das_to_dds(cache_dds,container_name,h5_fname,das_cache_fname,h5_fd,das_from_dc);
2049  if(true == build_data)
2050  data_bdds->set_dds(cache_dds);
2051  else
2052  bdds->set_dds(cache_dds);
2053  fclose(dds_file);
2054 
2055  if (dds_cache) {
2056  // add a copy
2057  BESDEBUG(HDF5_NAME, prolog << "For memory cache, DDS added to the cache for : " << h5_fname << endl);
2058  dds_cache->add(new DDS(*cache_dds), h5_fname);
2059  }
2060 
2061 }
2062 
2063 // Add DAS to DDS.
2064 void HDF5RequestHandler::add_das_to_dds(DDS *dds, const string &/*container_name*/, const string &filename,
2065  const string &das_cache_fname, hid_t h5_fd, bool das_from_dc) {
2066 
2067  BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
2068 
2069  // Check DAS memory cache
2070  DAS *das = 0 ;
2071  bool use_das_cache = false;
2072  if (das_cache)
2073  das = static_cast<DAS*>(das_cache->get(filename));
2074  if (das)
2075  use_das_cache = true;
2076 
2077  if (true == use_das_cache) {
2078  BESDEBUG(HDF5_NAME, prolog << "DAS Cached hit for : " << filename << endl);
2079  dds->transfer_attributes(das); // no need to copy the cached DAS
2080  }
2081 
2082  else {
2083 
2084  das = new DAS ;
2085 #if 0
2086  if (!container_name.empty())
2087  das->container_name(container_name);
2088 #endif
2089  if(das_from_dc == true)
2090  read_das_from_disk_cache(das_cache_fname,das);
2091  else {
2092  // This bool is for the case, when DDS is read from a cache then we need to open the HDF5 file.
2093  bool h5_file_open = true;
2094  if(h5_fd == -1)
2095  h5_file_open = false;
2096  if (true == _usecf) {
2097  // go to the CF option
2098  if(h5_file_open == false)
2099  h5_fd = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
2100 
2101  read_cfdas( *das,filename,h5_fd);
2102  if(h5_file_open == false)
2103  H5Fclose(h5_fd);
2104  }
2105  else {
2106  if(h5_file_open == false)
2107  h5_fd = get_fileid(filename.c_str());
2108  find_gloattr(h5_fd, *das);
2109  depth_first(h5_fd, "/", *das);
2110  if(h5_file_open == false)
2111  close_fileid(h5_fd);
2112  }
2113 
2114  Ancillary::read_ancillary_das( *das, filename ) ;
2115 
2116  if(das_cache_fname!="" && das_from_dc == false)
2117  write_das_to_disk_cache(das_cache_fname,das);
2118  }
2119 
2120  dds->transfer_attributes(das);
2121 
2122  if (das_cache) {
2123  // add a copy
2124  BESDEBUG(HDF5_NAME, prolog << "For memory cache, DAS added to the cache for : " << filename << endl);
2125  das_cache->add(new DAS(*das), filename);
2126  }
2127  delete das;
2128 
2129  }
2130 
2131 }
2132 
2133 bool check_beskeys(const string key) {
2134 
2135  bool found = false;
2136  string doset ="";
2137  const string dosettrue ="true";
2138  const string dosetyes = "yes";
2139 
2140  TheBESKeys::TheKeys()->get_value( key, doset, found ) ;
2141  if( true == found ) {
2142  doset = BESUtil::lowercase( doset ) ;
2143  }
2144  BESDEBUG(HDF5_NAME, prolog << "Key: " << key << (found?(" was found. value: "+doset):" was not found.") << endl);
2145  return found && (dosettrue == doset || dosetyes == doset);
2146 
2147 }
2148 
2149 // get_uint_key and get_float_key are copied from the netCDF handler.
2150 
2151 static unsigned int get_uint_key(const string &key, unsigned int def_val)
2152 {
2153  bool found = false;
2154  string doset = "";
2155 
2156  TheBESKeys::TheKeys()->get_value(key, doset, found);
2157  if (true == found) {
2158  // In C++11, stoi is better.
2159  return atoi(doset.c_str()); // use better code TODO
2160  }
2161  else {
2162  return def_val;
2163  }
2164 }
2165 
2166 static unsigned long get_ulong_key(const string &key, unsigned long def_val)
2167 {
2168  bool found = false;
2169  string doset = "";
2170 
2171  TheBESKeys::TheKeys()->get_value(key, doset, found);
2172  if (true == found) {
2173  // In C++11, stoull is better.
2174  return atol(doset.c_str()); // use better code TODO
2175  }
2176  else {
2177  return def_val;
2178  }
2179 }
2180 static float get_float_key(const string &key, float def_val)
2181 {
2182  bool found = false;
2183  string doset = "";
2184 
2185  TheBESKeys::TheKeys()->get_value(key, doset, found);
2186  if (true == found) {
2187  return atof(doset.c_str()); // use better code TODO
2188  }
2189  else {
2190  return def_val;
2191  }
2192 }
2193 
2194 static string get_beskeys(const string &key) {
2195 
2196  bool found = false;
2197  string ret_value ="";
2198 
2199  TheBESKeys::TheKeys()->get_value( key, ret_value, found ) ;
2200  return ret_value;
2201 
2202 }
2203 
2204 // The function to copy a string to a memory buffer.
2205 char* copy_str(char*temp_ptr,const string & str) {
2206 
2207  size_t str_size=str.size();
2208  memcpy((void*)temp_ptr,(void*)&str_size,sizeof(size_t));
2209  temp_ptr+=sizeof(size_t);
2210  vector<char>temp_vc2(str.begin(),str.end());
2211  memcpy((void*)temp_ptr,(void*)&temp_vc2[0],str.size());
2212  temp_ptr+=str.size();
2213  return temp_ptr;
2214 
2215 }
2216 
2217 
2218 // Obtain the string from a memory buffer.
2219 // Note: both char* and string(as a reference) will be returned
2220 // The attribute binary first stores the size of the string, then the string itself
2221 char* obtain_str(char*temp_ptr,string & str) {
2222 
2223  size_t oname_size = *((size_t *)temp_ptr);
2224  temp_ptr = temp_ptr + sizeof(size_t);
2225  string oname;
2226  for(unsigned int i =0; i<oname_size; i++){
2227  oname.push_back(*temp_ptr);
2228  ++temp_ptr;
2229  }
2230  str = oname;
2231  return temp_ptr;
2232 
2233 }
2234 
2235 // For our case, there are no global attributes for DAS.
2236 // The global attribures are always under HDF_GLOBAL.
2237 // The main function to obtain the DAS info. from the cache.
2238 char* get_attr_info_from_dc(char*temp_pointer,DAS *das,AttrTable *at_par) {
2239 
2240  // 3 is only for the code to come into the loop.
2241  uint8_t flag =3;
2242  while(flag !=2) {
2243  flag = *((uint8_t*)(temp_pointer));
2244  BESDEBUG(HDF5_NAME, prolog << "Build DAS from the disk cache file flag: "
2245  <<" flag = 0, attribute; flag = 1, container; flag =2; end of container;"
2246  <<" flag = 3; the initial value to get the attribute retrieval process started."
2247  <<" The flag value is "
2248  << (int)flag <<endl);
2249  temp_pointer++;
2250 
2251  if(flag ==1) {
2252  string container_name;
2253  temp_pointer = obtain_str(temp_pointer,container_name);
2254  BESDEBUG(HDF5_NAME, prolog << "DAS from the disk cache, container name is " << container_name << endl);
2255 
2256  // Remember the current Attribute table state
2257  AttrTable*temp_at_par = at_par;
2258  if(at_par == NULL)
2259  at_par = das->add_table(container_name, new AttrTable);
2260  else
2261  at_par = at_par->append_container(container_name);
2262 
2263  temp_pointer = get_attr_info_from_dc(temp_pointer,das,at_par);
2264  // MUST resume the original state
2265  at_par = temp_at_par;
2266 
2267  }
2268  else if(flag == 0) {
2269  // The attribute must have a table.
2270  if(at_par ==NULL)
2271  throw BESInternalError( "The AttrTable must exist for DAS attributes", __FILE__, __LINE__ ) ;
2272 
2273  // Attribute name
2274  string attr_name;
2275  temp_pointer = obtain_str(temp_pointer,attr_name);
2276  BESDEBUG(HDF5_NAME, prolog << "DAS from the disk cache, attr name is: " << attr_name << endl);
2277 
2278  // Attribute type
2279  string attr_type;
2280  temp_pointer = obtain_str(temp_pointer,attr_type);
2281  BESDEBUG(HDF5_NAME, prolog << "DAS from the disk cache, attr type is: " << attr_type << endl);
2282 
2283  // Attribute values
2284  unsigned int num_values = *((unsigned int*)(temp_pointer));
2285  BESDEBUG(HDF5_NAME, prolog << "DAS from the disk cache, number of attribute values is: " << num_values << endl);
2286  temp_pointer+=sizeof(unsigned int);
2287 
2288  vector <string> attr_values;
2289 
2290  for(unsigned int i = 0; i<num_values; i++) {
2291  string attr_value;
2292  temp_pointer = obtain_str(temp_pointer,attr_value);
2293  attr_values.push_back(attr_value);
2294  BESDEBUG(HDF5_NAME, prolog << "DAS from the disk cache, attribute value is: " << attr_value << endl);
2295  }
2296 
2297  at_par->append_attr(attr_name,attr_type,&attr_values);
2298  }
2299 
2300  }
2301  return temp_pointer;
2302 
2303 }
2304 
2305 // The debugging function to get attribute info.
2306 void get_attr_contents(AttrTable*temp_table) {
2307  if(temp_table !=NULL) {
2308  AttrTable::Attr_iter top_startit = temp_table->attr_begin();
2309  AttrTable::Attr_iter top_endit = temp_table->attr_end();
2310  AttrTable::Attr_iter top_it = top_startit;
2311  while(top_it !=top_endit) {
2312  AttrType atype = temp_table->get_attr_type(top_it);
2313  if(atype == Attr_unknown)
2314  cerr<<"unsupported DAS attributes" <<endl;
2315  else if(atype!=Attr_container) {
2316 
2317  cerr<<"Attribute name is "<<temp_table->get_name(top_it)<<endl;
2318  cerr<<"Attribute type is "<<temp_table->get_type(top_it)<<endl;
2319  unsigned int num_attrs = temp_table->get_attr_num(temp_table->get_name(top_it));
2320  cerr<<"Attribute values are "<<endl;
2321  for (unsigned int i = 0; i <num_attrs;i++)
2322  cerr<<(*(temp_table->get_attr_vector(temp_table->get_name(top_it))))[i]<<" ";
2323  cerr<<endl;
2324  }
2325  else {
2326  cerr<<"Coming to the attribute container. "<<endl;
2327  cerr<<"container name is "<<(*top_it)->name <<endl;
2328  AttrTable* sub_table = temp_table->get_attr_table(top_it);
2329  cerr<<"container table name is "<<sub_table->get_name() <<endl;
2330  get_attr_contents(sub_table);
2331  }
2332  ++top_it;
2333  }
2334 
2335  }
2336 }
2337 
2338 
2339 void HDF5RequestHandler::add_attributes(BESDataHandlerInterface &dhi) {
2340 
2341 
2342  BESResponseObject *response = dhi.response_handler->get_response_object();
2343  BESDataDDSResponse *bdds = dynamic_cast<BESDataDDSResponse *>(response);
2344  if (!bdds)
2345  throw BESInternalError("cast error", __FILE__, __LINE__);
2346  DDS *dds = bdds->get_dds();
2347  string container_name = bdds->get_explicit_containers() ? dhi.container->get_symbolic_name(): "";
2348  string filename = dhi.container->access();
2349  DAS* das = 0;
2350  bool das_from_mcache = false;
2351  if(das_cache) {
2352  das = static_cast<DAS*>(das_cache->get(filename));
2353  if(das) {
2354  BESDEBUG(HDF5_NAME, prolog << "DAS Cached hit for : " << filename << endl);
2355  dds->transfer_attributes(das); // no need to copy the cached DAS
2356  das_from_mcache = true;
2357  }
2358  }
2359 
2360  if(false == das_from_mcache) {
2361  das = new DAS;
2362  // This looks at the 'use explicit containers' prop, and if true
2363  // sets the current container for the DAS.
2364  if (!container_name.empty()) das->container_name(container_name);
2365 
2366  hid_t h5_fd =-1;
2367  if (true == _usecf) {
2368  // go to the CF option
2369  h5_fd = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
2370 
2371  read_cfdas( *das,filename,h5_fd);
2372 
2373  H5Fclose(h5_fd);
2374  }
2375  else {
2376  h5_fd = get_fileid(filename.c_str());
2377  find_gloattr(h5_fd, *das);
2378  depth_first(h5_fd, "/", *das);
2379  close_fileid(h5_fd);
2380  }
2381 
2382 
2383  Ancillary::read_ancillary_das(*das, filename);
2384 
2385  dds->transfer_attributes(das);
2386 
2387  // Only free the DAS if it's not added to the cache
2388  if (das_cache) {
2389  // add a copy
2390  BESDEBUG(HDF5_NAME, prolog << "DAS added to the cache for : " << filename << endl);
2391  das_cache->add(das, filename);
2392  }
2393  else {
2394  delete das;
2395  }
2396  }
2397  BESDEBUG(HDF5_NAME, prolog << "Data ACCESS in add_attributes(): set the including attribute flag to true: "<<filename << endl);
2398  bdds->set_ia_flag(true);
2399  return;
2400 
2401 
2402 }
include the entry functions to execute the handlers
std::string get_symbolic_name() const
retrieve the symbolic name for this container
Definition: BESContainer.h:221
virtual std::string access()=0
returns the true name of this container
Represents an OPeNDAP DAS DAP2 data object within the BES.
virtual void clear_container()
clear the container in the DAP response object
virtual void set_container(const std::string &cn)
set the container in the DAP response object
Holds a DDS object within the BES.
virtual void set_container(const std::string &cn)
set the container in the DAP response object
void set_dds(libdap::DDS *ddsIn)
virtual void clear_container()
clear the container in the DAP response object
libdap::DDS * get_dds()
Represents an OPeNDAP DMR DAP4 data object within the BES.
error object created from libdap error objects and can handle those errors
Definition: BESDapError.h:59
virtual void set_dap4_function(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_dap4_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
bool get_explicit_containers() const
Should containers be explicitly represented in the DD* responses?
std::string get_request_xml_base() const
Return the xml:base URL for this request.
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
void set_dds(libdap::DDS *ddsIn)
virtual void set_container(const std::string &cn)
set the container in the DAP response object
virtual void clear_container()
clear the container in the DAP response object
Structure storing information used by the BES to handle the request.
BESContainer * container
pointer to current container in this interface
Abstract exception class for the BES with basic string message.
Definition: BESError.h:58
virtual std::string get_message()
get the error message for this exception
Definition: BESError.h:99
informational response object
Definition: BESInfo.h:63
virtual void add_data(const std::string &s)
add data to this informational object. If buffering is not set then the information is output directl...
Definition: BESInfo.cc:160
exception thrown if internal error encountered
exception thrown if an internal error is found and is fatal to the BES
Represents a specific data type request handler.
virtual BESResponseObject * get_response_object()
return the current response object
Abstract base class representing a specific set of information in response to a request to the BES.
virtual void services_handled(const std::string &handler, std::list< std::string > &services)
returns the list of servies provided by the handler in question
virtual bool start(std::string name)
Definition: BESStopWatch.cc:67
static std::string lowercase(const std::string &s)
Definition: BESUtil.cc:200
static std::string implode(const std::list< std::string > &values, char delim)
Definition: BESUtil.cc:638
An in-memory cache for DapObj (DAS, DDS, ...) objects.
Definition: ObjMemCache.h:84
virtual void add(libdap::DapObj *obj, const std::string &key)
Add an object to the cache and associate it with a key.
Definition: ObjMemCache.cc:63
virtual libdap::DapObj * get(const std::string &key)
Get the cached pointer.
Definition: ObjMemCache.cc:105
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:339
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:71
Helper functions for generating DAS attributes and a function to check BES Key.
void depth_first(hid_t pid, const char *gname, DAS &das)
Definition: h5das.cc:62
void find_gloattr(hid_t file, DAS &das)
Definition: h5das.cc:487
bool breadth_first(hid_t pid, char *gname, D4Group *par_grp, const char *fname, bool use_dimscale)
Definition: h5dmr.cc:319
hid_t get_fileid(const char *filename)
Definition: h5get.cc:405
void close_fileid(hid_t fid)
Definition: h5get.cc:427
The main header of the HDF5 OPeNDAP handler.