bes Updated for version 3.20.10
HDF5RequestHandler.cc
Go to the documentation of this file.
1
2// -*- mode: c++; c-basic-offset:4 -*-
3
4// This file is part of hdf5_handler, a data handler for the OPeNDAP data
5// server.
6
7// Copyright (c) 2002,2003 OPeNDAP, Inc.
8// Copyright (c) 2007-2016 The HDF Group, Inc. and OPeNDAP, Inc.
9// Author: James Gallagher <jgallagher@opendap.org>
10//
11// This is free software; you can redistribute it and/or modify it under the
12// terms of the GNU Lesser General Public License as published by the Free
13// Software Foundation; either version 2.1 of the License, or (at your
14// option) any later version.
15//
16// This software is distributed in the hope that it will be useful, but
17// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
18// or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
19// License for more details.
20//
21// You should have received a copy of the GNU Lesser General Public
22// License along with this library; if not, write to the Free Software
23// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24//
25// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
26
27// HDF5RequestHandler.cc
33
34
35#include <iostream>
36#include <fstream>
37#include <sstream>
38#include <string>
39#include <sys/types.h>
40#include <sys/stat.h>
41#include <fcntl.h>
42#include <unistd.h>
43
44#include <libdap/DMR.h>
45#include <libdap/D4BaseTypeFactory.h>
46#include <BESDMRResponse.h>
47#include <ObjMemCache.h>
48#include "HDF5_DMR.h"
49
50#include <libdap/mime_util.h>
51#include "hdf5_handler.h"
52#include "HDF5RequestHandler.h"
53#include "HDF5_DDS.h"
54
55#include <BESDASResponse.h>
56#include <BESDDSResponse.h>
57#include <BESDataDDSResponse.h>
58#include <libdap/Ancillary.h>
59#include <BESInfo.h>
60#include <BESDapNames.h>
61#include <BESResponseNames.h>
62#include <BESContainer.h>
63#include <BESResponseHandler.h>
64#include <BESVersionInfo.h>
65#include <BESServiceRegistry.h>
66#include <BESUtil.h>
67#include <BESDapError.h>
68#include <BESInternalFatalError.h>
69#include <TheBESKeys.h>
70#include <BESDebug.h>
71#include <BESStopWatch.h>
72#include "h5get.h"
73#include "config_hdf5.h"
74
75#define HDF5_NAME "h5"
76#include "h5cfdaputil.h"
77
78using namespace std;
79using namespace libdap;
80
81#define prolog std::string("HDF5RequestHandler::").append(__func__).append("() - ")
82
83// The debug function to dump all the contents of a DAS table.
84void get_attr_contents(AttrTable* temp_table);
85
86// Five functions to generate a DAS cache file.
87// 1. The wrapper function to call write_das_table_to_file to generate the cache.
88void write_das_to_file(DAS*das_ptr,FILE* das_file);
89
90// 2. The main function to generate the DAS cache.
91void write_das_table_to_file(AttrTable*temp_table,FILE* das_file);
92
93// 3. The function to write the DAS container/table name to the cache
94void write_container_name_to_file(const string&,FILE* das_file);
95
96// 4. The function to write the DAS attribute name,type and values to the cache
97void write_das_attr_info(AttrTable* dtp,const string&,const string&,FILE * das_file);
98
99// 5. The function to copy a string to a memory buffer.
100char* copy_str(char *temp_ptr,const string & str);
101
102// These two functions are for reading the DAS from the DAS cache file
103// 1. Obtain the string from a memory buffer.
104char* obtain_str(char*temp_ptr,string & str);
105
106// 2. The main function to obtain DAS info from the cache.
107char* get_attr_info_from_dc(char*temp_pointer,DAS *das,AttrTable *at);
108
109// Check if the BES key is set.
110//bool check_and_set_beskeys(const string);
111
112// Obtain the BES key as an integer
113static unsigned int get_uint_key(const string &key,unsigned int def_val);
114static unsigned long get_ulong_key(const string &key,unsigned long def_val);
115
116// Obtain the BES key as a floating-pointer number.
117static float get_float_key(const string &key, float def_val);
118
119// Obtain the BES key as a string.
120static string get_beskeys(const string&);
121
122//static bool is_beskeys_set_true(const string &);
123bool obtain_beskeys_info(const string &, bool &);
124
125// For the CF option
126extern void read_cfdas(DAS &das, const string & filename,hid_t fileid);
127extern void read_cfdds(DDS &dds, const string & filename,hid_t fileid);
128extern void read_cfdmr(DMR *dmr, const string & filename,hid_t fileid);
129
130
131
132// Check the description of cache_entries and cache_purge_level at h5.conf.in.
133unsigned int HDF5RequestHandler::_mdcache_entries = 500;
134unsigned int HDF5RequestHandler::_lrdcache_entries = 0;
135unsigned int HDF5RequestHandler::_srdcache_entries = 0;
136float HDF5RequestHandler::_cache_purge_level = 0.2;
137
138// Metadata object cache at DAS,DDS and DMR.
139ObjMemCache *HDF5RequestHandler::das_cache = 0;
140ObjMemCache *HDF5RequestHandler::dds_cache = 0;
141ObjMemCache *HDF5RequestHandler::datadds_cache = 0;
142ObjMemCache *HDF5RequestHandler::dmr_cache = 0;
143
144ObjMemCache *HDF5RequestHandler::lrdata_mem_cache = 0;
145ObjMemCache *HDF5RequestHandler::srdata_mem_cache = 0;
146
147// Set default values of all BES keys according to h5.conf.in.
148// This will help the generation of DMRPP. No need to
149// set multiple keys if the user's setting is the same as
150// the h5.conf.in. KY 2021-08-23
151bool HDF5RequestHandler::_usecf = true;
152bool HDF5RequestHandler::_pass_fileid = false;
153bool HDF5RequestHandler::_disable_structmeta = true;
154bool HDF5RequestHandler::_disable_ecsmeta = false;
155bool HDF5RequestHandler::_keep_var_leading_underscore = false;
156bool HDF5RequestHandler::_check_name_clashing = false;
157bool HDF5RequestHandler::_add_path_attrs = true;
158bool HDF5RequestHandler::_drop_long_string = true;
159bool HDF5RequestHandler::_fillvalue_check = true;
160bool HDF5RequestHandler::_check_ignore_obj = false;
161bool HDF5RequestHandler::_flatten_coor_attr = true;
162bool HDF5RequestHandler::_default_handle_dimension = true; //Ignored when _usecf=true.
163bool HDF5RequestHandler::_eos5_rm_convention_attr_path = true;
164bool HDF5RequestHandler::_dmr_long_int = true;
165bool HDF5RequestHandler::_no_zero_size_fullnameattr = false;
166bool HDF5RequestHandler::_enable_coord_attr_add_path = true;
167
168bool HDF5RequestHandler::_usecfdmr = true;
169
170bool HDF5RequestHandler::_common_cache_dirs = false;
171
172bool HDF5RequestHandler::_use_disk_cache =false;
173bool HDF5RequestHandler::_use_disk_dds_cache =false;
174string HDF5RequestHandler::_disk_cache_dir ="";
175string HDF5RequestHandler::_disk_cachefile_prefix ="";
176unsigned long long HDF5RequestHandler::_disk_cache_size =0;
177
178
179bool HDF5RequestHandler::_disk_cache_comp_data =false;
180bool HDF5RequestHandler::_disk_cache_float_only_comp_data =false;
181float HDF5RequestHandler::_disk_cache_comp_threshold =1.0;
182unsigned long HDF5RequestHandler::_disk_cache_var_size =0;
183
184bool HDF5RequestHandler::_use_disk_meta_cache = false;
185string HDF5RequestHandler::_disk_meta_cache_path ="";
186
187bool HDF5RequestHandler::_use_latlon_disk_cache = false;
188long HDF5RequestHandler::_latlon_disk_cache_size =0;
189string HDF5RequestHandler::_latlon_disk_cache_dir ="";
190string HDF5RequestHandler::_latlon_disk_cachefile_prefix="";
191
192DMR* HDF5RequestHandler::dmr_int64 = 0;
193
194
195#if 0
196//BaseTypeFactory factory;
197//libdap::DDS HDF5RequestHandler::hd_dds(&factory,"");
198#endif
199string HDF5RequestHandler::_stp_east_filename;
200string HDF5RequestHandler::_stp_north_filename;
201vector<string> HDF5RequestHandler::lrd_cache_dir_list;
202vector<string> HDF5RequestHandler::lrd_non_cache_dir_list;
203vector<string> HDF5RequestHandler::lrd_var_cache_file_list;
204
205#if 0
206//libdap::DDS*cache_dds;
207#endif
208
209HDF5RequestHandler::HDF5RequestHandler(const string & name)
210 :BESRequestHandler(name)
211{
212
213 BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
214
215 add_handler(DAS_RESPONSE, HDF5RequestHandler::hdf5_build_das);
216 add_handler(DDS_RESPONSE, HDF5RequestHandler::hdf5_build_dds);
217 add_handler(DATA_RESPONSE, HDF5RequestHandler::hdf5_build_data);
218 add_handler(DMR_RESPONSE, HDF5RequestHandler::hdf5_build_dmr);
219 add_handler(DAP4DATA_RESPONSE, HDF5RequestHandler::hdf5_build_dmr);
220
221 add_handler(HELP_RESPONSE, HDF5RequestHandler::hdf5_build_help);
222 add_handler(VERS_RESPONSE, HDF5RequestHandler::hdf5_build_version);
223
224#if !(DYNAMIC_CONFIG_ENABLED)
225 load_config();
226#endif
227
228 BESDEBUG(HDF5_NAME, prolog << "END" << endl);
229}
230
231HDF5RequestHandler::~HDF5RequestHandler()
232{
233 // delete the cache.
234 delete das_cache;
235 delete dds_cache;
236 delete datadds_cache;
237 delete dmr_cache;
238 delete lrdata_mem_cache;
239 delete srdata_mem_cache;
240
241}
242
246void HDF5RequestHandler::load_config()
247{
248 BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
249 BESStopWatch sw;
250 if (BESISDEBUG(HDF5_NAME)){
251 sw.start(prolog,"ClockTheBESKeys");
252 }
253
254 // Obtain the metadata cache entries and purge level.
255 HDF5RequestHandler::_mdcache_entries = get_uint_key("H5.MetaDataMemCacheEntries", 0);
256 HDF5RequestHandler::_lrdcache_entries = get_uint_key("H5.LargeDataMemCacheEntries", 0);
257 HDF5RequestHandler::_srdcache_entries = get_uint_key("H5.SmallDataMemCacheEntries", 0);
258 HDF5RequestHandler::_cache_purge_level = get_float_key("H5.CachePurgeLevel", 0.2);
259
260 if (get_mdcache_entries()) { // else it stays at its default of null
261 das_cache = new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
262 dds_cache = new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
263 datadds_cache = new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
264 dmr_cache = new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
265 }
266
267 // Starting from hyrax 1.16.5, users don't need to explicitly set the BES keys if
268 // they are happy with the default settings in the h5.conf.in.
269 // In the previous releases, we required users to set BES key values. Otherwise,
270 // the BES key values of true/false will always be false if the key is not found.
271 // Hopefully this change will make it convenient for the general users.
272 // In the mean time, the explicit BES key settings will not be affected.
273 // KY 2021-10-13
274 //
275 // Check if the EnableCF key is set.
276 bool has_key = false;
277 bool key_value = obtain_beskeys_info("H5.EnableCF",has_key);
278 if(has_key)
279 _usecf = key_value;
280 BESDEBUG(HDF5_NAME, prolog << "H5.EnableCF: " << (_usecf?"true":"false") << endl);
281
282 // The DefaultHandleDimension is effective only when EnableCF=false
283 //_default_handle_dimension = check_and_set_beskeys("H5.DefaultHandleDimension");
284 key_value = obtain_beskeys_info("H5.DefaultHandleDimension",has_key);
285 if(has_key)
286 _default_handle_dimension = key_value;
287 BESDEBUG(HDF5_NAME, prolog << "H5.DefaultHandleDimension: " << (_default_handle_dimension?"true":"false") << endl);
288
289 // The following keys are only effective when EnableCF is true or unset(EnableCF is true if users don't set the key).
290 //_pass_fileid = obtain_beskeys_info("H5.EnablePassFileID",has_key);
291 key_value = obtain_beskeys_info("H5.EnablePassFileID",has_key);
292 if(has_key)
293 _pass_fileid = key_value;
294 BESDEBUG(HDF5_NAME, prolog << "H5.EnablePassFileID: " << (_pass_fileid?"true":"false") << endl);
295
296 //_disable_structmeta = check_and_set_beskeys("H5.DisableStructMetaAttr");
297 key_value = obtain_beskeys_info("H5.DisableStructMetaAttr",has_key);
298 if(has_key)
299 _disable_structmeta = key_value;
300 BESDEBUG(HDF5_NAME, prolog << "H5.DisableStructMetaAttr: " << (_disable_structmeta?"true":"false") << endl);
301
302 //_disable_ecsmeta = check_and_set_beskeys("H5.DisableECSMetaAttr");
303 key_value = obtain_beskeys_info("H5.DisableECSMetaAttr",has_key);
304 if(has_key)
305 _disable_ecsmeta = key_value;
306 BESDEBUG(HDF5_NAME, prolog << "H5.DisableECSMetaAttr: " << (_disable_ecsmeta?"true":"false") << endl);
307
308 //_keep_var_leading_underscore = check_and_set_beskeys("H5.KeepVarLeadingUnderscore");
309 key_value = obtain_beskeys_info("H5.KeepVarLeadingUnderscore",has_key);
310 if(has_key)
311 _keep_var_leading_underscore = key_value;
312 BESDEBUG(HDF5_NAME, prolog << "H5.KeepVarLeadingUnderscore: " << (_keep_var_leading_underscore?"true":"false") << endl);
313
314 //_check_name_clashing = check_and_set_beskeys("H5.EnableCheckNameClashing");
315 key_value = obtain_beskeys_info("H5.EnableCheckNameClashing",has_key);
316 if(has_key)
317 _check_name_clashing = key_value;
318 BESDEBUG(HDF5_NAME, prolog << "H5.EnableCheckNameClashing: " << (_check_name_clashing?"true":"false") << endl);
319
320 //_add_path_attrs = check_and_set_beskeys("H5.EnableAddPathAttrs");
321 key_value = obtain_beskeys_info("H5.EnableAddPathAttrs",has_key);
322 if(has_key)
323 _add_path_attrs = key_value;
324 BESDEBUG(HDF5_NAME, prolog << "H5.EnableAddPathAttrs: " << (_add_path_attrs?"true":"false") << endl);
325
326 //_drop_long_string = check_and_set_beskeys("H5.EnableDropLongString");
327 key_value = obtain_beskeys_info("H5.EnableDropLongString",has_key);
328 if(has_key)
329 _drop_long_string = key_value;
330 BESDEBUG(HDF5_NAME, prolog << "H5.EnableDropLongString: " << (_drop_long_string?"true":"false") << endl);
331
332 //_fillvalue_check = check_and_set_beskeys("H5.EnableFillValueCheck");
333 key_value = obtain_beskeys_info("H5.EnableFillValueCheck",has_key);
334 if(has_key)
335 _fillvalue_check = key_value;
336 BESDEBUG(HDF5_NAME, prolog << "H5.EnableFillValueCheck: " << (_fillvalue_check?"true":"false") << endl);
337
338
339 //_check_ignore_obj = check_and_set_beskeys("H5.CheckIgnoreObj");
340 key_value = obtain_beskeys_info("H5.CheckIgnoreObj",has_key);
341 if(has_key)
342 _check_ignore_obj = key_value;
343 BESDEBUG(HDF5_NAME, prolog << "H5.CheckIgnoreObj: " << (_check_ignore_obj?"true":"false") << endl);
344
345 //_flatten_coor_attr = check_and_set_beskeys("H5.ForceFlattenNDCoorAttr");
346 key_value = obtain_beskeys_info("H5.ForceFlattenNDCoorAttr",has_key);
347 if(has_key)
348 _flatten_coor_attr = key_value;
349 BESDEBUG(HDF5_NAME, prolog << "H5.ForceFlattenNDCoorAttr: " << (_flatten_coor_attr?"true":"false") << endl);
350
351
352 //_eos5_rm_convention_attr_path= check_and_set_beskeys("H5.RmConventionAttrPath");
353 key_value = obtain_beskeys_info("H5.RmConventionAttrPath",has_key);
354 if(has_key)
355 _eos5_rm_convention_attr_path = key_value;
356 BESDEBUG(HDF5_NAME, prolog << "H5.RmConventionAttrPath: " << (_eos5_rm_convention_attr_path?"true":"false") << endl);
357
358 //_dmr_long_int = check_and_set_beskeys("H5.EnableDMR64bitInt");
359 key_value = obtain_beskeys_info("H5.EnableDMR64bitInt",has_key);
360 if(has_key)
361 _dmr_long_int = key_value;
362 BESDEBUG(HDF5_NAME, prolog << "H5.EnableDMR64bitInt: " << (_dmr_long_int?"true":"false") << endl);
363
364 //_no_zero_size_fullnameattr = check_and_set_beskeys("H5.NoZeroSizeFullnameAttr");
365 key_value = obtain_beskeys_info("H5.NoZeroSizeFullnameAttr",has_key);
366 if(has_key)
367 _no_zero_size_fullnameattr = key_value;
368 BESDEBUG(HDF5_NAME, prolog << "H5.NoZeroSizeFullnameAttr: " << (_no_zero_size_fullnameattr?"true":"false") << endl);
369
370 //_enable_coord_attr_add_path = check_and_set_beskeys("H5.EnableCoorattrAddPath");
371 key_value = obtain_beskeys_info("H5.EnableCoorattrAddPath",has_key);
372 if(has_key)
373 _enable_coord_attr_add_path = key_value;
374 BESDEBUG(HDF5_NAME, prolog << "H5.EnableCoorattrAddPath: " << (_enable_coord_attr_add_path?"true":"false") << endl);
375
376 //_usecfdmr = check_and_set_beskeys("H5.EnableCFDMR");
377 key_value = obtain_beskeys_info("H5.EnableCFDMR",has_key);
378 if(has_key)
379 _usecfdmr = key_value;
380 BESDEBUG(HDF5_NAME, prolog << "H5.EnableCFDMR: " << (_usecfdmr?"true":"false") << endl);
381
382 //_use_disk_cache = check_and_set_beskeys("H5.EnableDiskDataCache");
383 key_value = obtain_beskeys_info("H5.EnableDiskDataCache",has_key);
384 if(has_key)
385 _use_disk_cache = key_value;
386 BESDEBUG(HDF5_NAME, prolog << "H5.EnableDiskDataCache: " << (_use_disk_cache?"true":"false") << endl);
387 _disk_cache_dir = get_beskeys("H5.DiskCacheDataPath");
388 _disk_cachefile_prefix = get_beskeys("H5.DiskCacheFilePrefix");
389 _disk_cache_size = get_ulong_key("H5.DiskCacheSize",0);
390
391 //_disk_cache_comp_data = check_and_set_beskeys("H5.DiskCacheComp");
392 key_value = obtain_beskeys_info("H5.DiskCacheComp",has_key);
393 if(has_key)
394 _disk_cache_comp_data = key_value;
395 BESDEBUG(HDF5_NAME, prolog << "H5.DiskCacheComp: " << (_disk_cache_comp_data?"true":"false") << endl);
396
397 //_disk_cache_float_only_comp_data = check_and_set_beskeys("H5.DiskCacheFloatOnlyComp");
398 key_value = obtain_beskeys_info("H5.DiskCacheFloatOnlyComp",has_key);
399 if(has_key)
400 _disk_cache_float_only_comp_data = key_value;
401 BESDEBUG(HDF5_NAME, prolog << "H5.DiskCacheFloatOnlyComp: " << (_disk_cache_float_only_comp_data?"true":"false") << endl);
402 _disk_cache_comp_threshold = get_float_key("H5.DiskCacheCompThreshold",1.0);
403 _disk_cache_var_size = 1024*get_uint_key("H5.DiskCacheCompVarSize",0);
404
405 //_use_disk_meta_cache = check_and_set_beskeys("H5.EnableDiskMetaDataCache");
406 key_value = obtain_beskeys_info("H5.EnableDiskMetaDataCache",has_key);
407 if(has_key)
408 _use_disk_meta_cache = key_value;
409 BESDEBUG(HDF5_NAME, prolog << "H5.EnableDiskMetaDataCache: " << (_use_disk_meta_cache?"true":"false") << endl);
410
411 //_use_disk_dds_cache = check_and_set_beskeys("H5.EnableDiskDDSCache");
412 key_value = obtain_beskeys_info("H5.EnableDiskDDSCache",has_key);
413 if(has_key)
414 _use_disk_dds_cache = key_value;
415 BESDEBUG(HDF5_NAME, prolog << "H5.EnableDiskDDSCache: " << (_use_disk_dds_cache?"true":"false") << endl);
416 _disk_meta_cache_path = get_beskeys("H5.DiskMetaDataCachePath");
417
418 //_use_latlon_disk_cache = check_and_set_beskeys("H5.EnableEOSGeoCacheFile");
419 key_value = obtain_beskeys_info("H5.EnableEOSGeoCacheFile",has_key);
420 if(has_key)
421 _use_latlon_disk_cache = key_value;
422 BESDEBUG(HDF5_NAME, prolog << "H5.EnableEOSGeoCacheFile: " << (_use_latlon_disk_cache?"true":"false") << endl);
423 _latlon_disk_cache_size = get_uint_key("H5.Cache.latlon.size",0);
424 _latlon_disk_cache_dir = get_beskeys("H5.Cache.latlon.path");
425 _latlon_disk_cachefile_prefix= get_beskeys("H5.Cache.latlon.prefix");
426
427#if 0
428 //_default_handle_dimension = check_and_set_beskeys("H5.DefaultHandleDimension");
429 key_value = obtain_beskeys_info("H5.DefaultHandleDimension",has_key);
430 if(has_key)
431 _default_handle_dimension = key_value;
432 BESDEBUG(HDF5_NAME, prolog << "H5.DefaultHandleDimension: " << (_default_handle_dimension?"true":"false") << endl);
433#endif
434
435 if(get_usecf()) {
436 if(get_lrdcache_entries()) {
437 lrdata_mem_cache = new ObjMemCache(get_lrdcache_entries(), get_cache_purge_level());
438 bool has_LFMC_config = false;
439 key_value = obtain_beskeys_info("H5.LargeDataMemCacheConfig",has_key);
440 if(has_key)
441 has_LFMC_config = key_value;
442 BESDEBUG(HDF5_NAME, prolog << "H5.LargeDataMemCacheConfig: " << (has_LFMC_config?"true":"false") << endl);
443 if(true == has_LFMC_config) {
444 _common_cache_dirs =obtain_lrd_common_cache_dirs();
445#if 0
446 if(false == _common_cache_dirs)
447cerr<<"No specific cache info"<<endl;
448#endif
449 }
450 }
451 if(get_srdcache_entries()) {
452
453 BESDEBUG(HDF5_NAME, prolog << "Generate memory cache for smaller coordinate variables" << endl);
454 srdata_mem_cache = new ObjMemCache(get_srdcache_entries(),get_cache_purge_level());
455
456 }
457
458 if(_disk_cache_comp_data == true && _use_disk_cache == true) {
459 if(_disk_cache_comp_threshold < 1.0) {
460 ostringstream ss;
461 ss<< _disk_cache_comp_threshold;
462 string _comp_threshold_str(ss.str());
463 string invalid_comp_threshold ="The Compression Threshold is the total size of the variable array";
464 invalid_comp_threshold+=" divided by the storage size of compressed array. It should always be >1";
465 invalid_comp_threshold+=" The current threhold set at h5.conf is ";
466 invalid_comp_threshold+=_comp_threshold_str;
467 invalid_comp_threshold+=" . Go back to h5.conf and change the H5.DiskCacheCompThreshold to a >1.0 number.";
468 throw BESInternalError(invalid_comp_threshold,__FILE__,__LINE__);
469 }
470 }
471 _stp_east_filename = get_beskeys("H5.STPEastFileName");
472 _stp_north_filename = get_beskeys("H5.STPNorthFileName");
473 }
474 BESDEBUG(HDF5_NAME, prolog << "END" << endl);
475}
476
477
478// Build DAS
479bool HDF5RequestHandler::hdf5_build_das(BESDataHandlerInterface & dhi)
480{
481 BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
482#if DYNAMIC_CONFIG_ENABLED
483 load_config();
484#endif
485
486 // For the time being, separate CF file ID from the default file ID(mainly for debugging)
487 hid_t cf_fileid = -1;
488
489 // Obtain the HDF5 file name.
490 string filename = dhi.container->access();
491
492 // Obtain the BES object from the client
493 BESResponseObject *response = dhi.response_handler->get_response_object() ;
494
495 // Convert to the BES DAS response
496 BESDASResponse *bdas = dynamic_cast < BESDASResponse * >(response) ;
497 if( !bdas )
498 throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
499
500 try {
502 DAS *das = bdas->get_das();
503
504 // Look inside the memory cache to see if it's initialized
505 DAS *cached_das_ptr = 0;
506 bool use_das_cache = false;
507 if (das_cache)
508 cached_das_ptr = static_cast<DAS*>(das_cache->get(filename));
509 if (cached_das_ptr)
510 use_das_cache = true;
511
512 if (true == use_das_cache) {
513
514 // copy the cached DAS into the BES response object
515 BESDEBUG(HDF5_NAME, prolog << "DAS Cached hit for : " << filename << endl);
516 *das = *cached_das_ptr;
517 }
518 else {
519
520 bool das_from_dc = false;
521 string das_cache_fname;
522
523 // If the use_disk_meta_cache is set, check if the cache file exists and sets the flag.
524 if(_use_disk_meta_cache == true) {
525
526 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
527 das_cache_fname = _disk_meta_cache_path+"/" +base_filename+"_das";
528
529 if(access(das_cache_fname.c_str(),F_OK) !=-1)
530 das_from_dc = true;
531
532 }
533
534 // If reading DAS from the disk cache, call the corresponding function
535 if(true == das_from_dc) {
536 read_das_from_disk_cache(das_cache_fname,das);
537
538 // If the memory cache is set, adding the DAS copy to the memory cache
539 if (das_cache) {
540 // add a copy
541 BESDEBUG(HDF5_NAME, prolog << "HDF5 DAS reading DAS from the disk cache. For memory cache, DAS added to the cache for : " << filename << endl);
542 das_cache->add(new DAS(*das), filename);
543 }
544 }
545
546 else {// Need to build from the HDF5 file
547 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
548 if (true == _usecf) {//CF option
549
550 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
551 if (cf_fileid < 0){
552 string invalid_file_msg="Could not open this HDF5 file ";
553 invalid_file_msg +=filename;
554 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
555 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
556 invalid_file_msg +=" distributor.";
557 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
558 }
559 // Need to check if DAP4 DMR CF 64-bit integer mapping is on.
560 if(HDF5RequestHandler::get_dmr_64bit_int()!=NULL)
561 HDF5RequestHandler::set_dmr_64bit_int(NULL);
562 read_cfdas( *das,filename,cf_fileid);
563 H5Fclose(cf_fileid);
564 }
565 else {// Default option
566 hid_t fileid = get_fileid(filename.c_str());
567 if (fileid < 0) {
568 string invalid_file_msg="Could not open this HDF5 file ";
569 invalid_file_msg +=filename;
570 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
571 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
572 invalid_file_msg +=" distributor.";
573 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
574
575 }
576
577 find_gloattr(fileid, *das);
578 depth_first(fileid, "/", *das);
579 close_fileid(fileid);
580 }
581
582 Ancillary::read_ancillary_das( *das, filename ) ;
583
584#if 0
585// Dump all attribute contents
586AttrTable* top_table = das->get_top_level_attributes();
587get_attr_contents(top_table);
588
589// Dump all variable contents
590AttrTable::Attr_iter start_aiter=das->var_begin();
591AttrTable::Attr_iter it = start_aiter;
592AttrTable::Attr_iter end_aiter = das->var_end();
593while(it != end_aiter) {
594AttrTable* temp_table = das->get_table(it);
595if(temp_table!=0){
596cerr<<"var_begin"<<endl;
597temp_table->print(cerr);
598}
599++it;
600}
601#endif
602 // If the memory cache is turned on
603 if(das_cache) {
604 // add a copy
605 BESDEBUG(HDF5_NAME, prolog << "DAS added to the cache for : " << filename << endl);
606 das_cache->add(new DAS(*das), filename);
607 }
608
609 // DAS disk cache fname will be set only when the metadata disk cache is turned on
610 // So if it comes here, the das cache should be generated.
611 if(das_cache_fname!="") {
612 BESDEBUG(HDF5_NAME, prolog << "HDF5 Build DAS: Write DAS to disk cache " << das_cache_fname << endl);
613 write_das_to_disk_cache(das_cache_fname,das);
614 }
615 }
616 }
617
618 bdas->clear_container() ;
619 }
620 catch(BESError & e) {
621 if(cf_fileid !=-1)
622 H5Fclose(cf_fileid);
623 BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
624 throw;
625 }
626 catch(InternalErr & e) {
627
628 if(cf_fileid !=-1)
629 H5Fclose(cf_fileid);
630 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
631 __FILE__, __LINE__);
632 }
633 catch(Error & e) {
634
635 if(cf_fileid !=-1)
636 H5Fclose(cf_fileid);
637 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
638 __FILE__, __LINE__);
639 }
640 catch(...) {
641
642 if(cf_fileid !=-1)
643 H5Fclose(cf_fileid);
644 string s = "unknown exception caught building HDF5 DAS";
645 throw BESInternalFatalError(s, __FILE__, __LINE__);
646 }
647
648 BESDEBUG(HDF5_NAME, prolog << "END" << endl);
649 return true;
650}
651
652// Convenient function that helps build DDS and Data
653// Since this function will be used by both the DDS and Data services, we need to pass both BESDDSResponse and BESDataDDSResponse.
654// This two parameters are necessary for the future DDS disk cache feature.
655void HDF5RequestHandler::get_dds_with_attributes( BESDDSResponse*bdds,BESDataDDSResponse*data_bdds,const string &container_name, const string& filename,const string &dds_cache_fname, const string &das_cache_fname,bool dds_from_dc,bool das_from_dc, bool build_data)
656{
657 DDS *dds;
658 if(true == build_data)
659 dds = data_bdds->get_dds();
660 else dds = bdds->get_dds();
661
662 // For the time being, separate CF file ID from the default file ID(mainly for debugging)
663 hid_t fileid = -1;
664 hid_t cf_fileid = -1;
665
666 try {
667
668 // Look in memory cache to see if it's initialized
669 DDS* cached_dds_ptr = 0;
670 bool use_dds_cache = false;
671 if (dds_cache)
672 cached_dds_ptr = static_cast<DDS*>(dds_cache->get(filename));
673 if (cached_dds_ptr)
674 use_dds_cache = true;
675 if (true == use_dds_cache) {
676 // copy the cached DDS into the BES response object. Assume that any cached DDS
677 // includes the DAS information.
678 BESDEBUG(HDF5_NAME, prolog << "DDS Metadata Cached hit for : " << filename << endl);
679 *dds = *cached_dds_ptr; // Copy the referenced object
680 }
681 else if (true ==dds_from_dc) {//Currently the dds_from_ds is always false by default.
682 read_dds_from_disk_cache(bdds,data_bdds,build_data,container_name,filename,dds_cache_fname,das_cache_fname,-1,das_from_dc);
683 }
684 else {
685 BESDEBUG(HDF5_NAME, prolog << "Build DDS from the HDF5 file. " << filename << endl);
686 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
687 dds->filename(filename);
688
689 // For the time being, not mess up CF's fileID with Default's fileID
690 if(true == _usecf) {
691
692 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
693 if (cf_fileid < 0){
694 string invalid_file_msg="Could not open this HDF5 file ";
695 invalid_file_msg +=filename;
696 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
697 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
698 invalid_file_msg +=" distributor.";
699 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
700 }
701 // The following is for DAP4 CF(DMR) 64-bit mapping, we need to set the flag
702 // to let the handler map the 64-bit integer.
703 if(HDF5RequestHandler::get_dmr_64bit_int() != NULL)
704 HDF5RequestHandler::set_dmr_64bit_int(NULL);
705 read_cfdds(*dds,filename,cf_fileid);
706 }
707 else {
708
709 fileid = get_fileid(filename.c_str());
710 if (fileid < 0) {
711 string invalid_file_msg="Could not open this HDF5 file ";
712 invalid_file_msg +=filename;
713 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
714 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
715 invalid_file_msg +=" distributor.";
716 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
717 }
718
719 depth_first(fileid, (char*)"/", *dds, filename.c_str());
720
721 }
722 // Check semantics
723 if (!dds->check_semantics()) { // DDS didn't comply with the semantics
724 dds->print(cerr);
725 throw InternalErr(__FILE__, __LINE__,
726 "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
727 }
728
729 Ancillary::read_ancillary_dds( *dds, filename ) ;
730
731 // Generate the DDS cached file if needed,currently this is always false by default
732 if(dds_cache_fname!="" && dds_from_dc == false)
733 write_dds_to_disk_cache(dds_cache_fname,dds);
734
735 // Add attributes
736 {
737 hid_t h5_fd = -1;
738 if(_usecf == true)
739 h5_fd = cf_fileid;
740 else
741 h5_fd = fileid;
742 add_das_to_dds(dds,container_name,filename,das_cache_fname,h5_fd,das_from_dc);
743 }
744
745 // Add memory cache if possible
746 if (dds_cache) {
747 // add a copy
748 BESDEBUG(HDF5_NAME, prolog << "DDS added to the cache for : " << filename << endl);
749 dds_cache->add(new DDS(*dds), filename);
750 }
751
752 if(cf_fileid != -1)
753 H5Fclose(cf_fileid);
754 if(fileid != -1)
755 H5Fclose(fileid);
756
757 }
758
759 }
760 catch(InternalErr & e) {
761
762 if(cf_fileid !=-1)
763 H5Fclose(cf_fileid);
764
765 if(fileid != -1)
766 H5Fclose(fileid);
767
768 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
769 __FILE__, __LINE__);
770 }
771 catch(Error & e) {
772
773 if(cf_fileid !=-1)
774 H5Fclose(cf_fileid);
775 if(fileid !=-1)
776 H5Fclose(fileid);
777
778 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
779 __FILE__, __LINE__);
780 }
781 catch(...) {
782
783 if(cf_fileid !=-1)
784 H5Fclose(cf_fileid);
785 if(fileid !=-1)
786 H5Fclose(fileid);
787
788 string s = "unknown exception caught building HDF5 DDS";
789 throw BESInternalFatalError(s, __FILE__, __LINE__);
790 }
791
792}
793
794void HDF5RequestHandler::get_dds_without_attributes_datadds(BESDataDDSResponse*data_bdds,const string &container_name, const string& filename)
795{
796 DDS *dds = data_bdds->get_dds();
797
798 // For the time being, separate CF file ID from the default file ID(mainly for debugging)
799 hid_t fileid = -1;
800 hid_t cf_fileid = -1;
801
802 try {
803
804 // Look in memory cache to see if it's initialized
805 DDS* cached_dds_ptr = 0;
806 bool use_datadds_cache = false;
807 if (datadds_cache)
808 cached_dds_ptr = static_cast<DDS*>(datadds_cache->get(filename));
809 if (cached_dds_ptr)
810 use_datadds_cache = true;
811 if (true == use_datadds_cache) {
812 // copy the cached DDS into the BES response object.
813 // The DAS information is not included.
814 BESDEBUG(HDF5_NAME, prolog << "DataDDS Metadata Cached hit for : " << filename << endl);
815 *dds = *cached_dds_ptr; // Copy the referenced object
816 }
817 else {
818 BESDEBUG(HDF5_NAME, prolog << "Build DDS from the HDF5 file. " << filename << endl);
819 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
820 dds->filename(filename);
821
822 // For the time being, not mess up CF's fileID with Default's fileID
823 if(true == _usecf) {
824
825 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
826 if (cf_fileid < 0){
827 string invalid_file_msg="Could not open this HDF5 file ";
828 invalid_file_msg +=filename;
829 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
830 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
831 invalid_file_msg +=" distributor.";
832 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
833 }
834 // The following is for DAP4 CF(DMR) 64-bit mapping, we need to set the flag
835 // to let the handler map the 64-bit integer.
836 if(HDF5RequestHandler::get_dmr_64bit_int() != NULL)
837 HDF5RequestHandler::set_dmr_64bit_int(NULL);
838 read_cfdds(*dds,filename,cf_fileid);
839 }
840 else {
841
842 fileid = get_fileid(filename.c_str());
843 if (fileid < 0) {
844 string invalid_file_msg="Could not open this HDF5 file ";
845 invalid_file_msg +=filename;
846 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
847 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
848 invalid_file_msg +=" distributor.";
849 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
850 }
851
852 depth_first(fileid, (char*)"/", *dds, filename.c_str());
853
854 }
855 // Check semantics
856 if (!dds->check_semantics()) { // DDS didn't comply with the semantics
857 dds->print(cerr);
858 throw InternalErr(__FILE__, __LINE__,
859 "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
860 }
861
862 Ancillary::read_ancillary_dds( *dds, filename ) ;
863
864#if 0
865 // Add attributes
866 {
867 hid_t h5_fd = -1;
868 if(_usecf == true)
869 h5_fd = cf_fileid;
870 else
871 h5_fd = fileid;
872 add_das_to_dds(dds,container_name,filename,das_cache_fname,h5_fd,das_from_dc);
873 }
874#endif
875
876 // Add memory cache if possible
877 if (datadds_cache) {
878 // add a copy
879 BESDEBUG(HDF5_NAME, prolog << "DataDDS added to the cache for : " << filename << endl);
880 datadds_cache->add(new DDS(*dds), filename);
881 }
882
883 if(cf_fileid != -1)
884 H5Fclose(cf_fileid);
885 if(fileid != -1)
886 H5Fclose(fileid);
887
888 }
889 BESDEBUG(HDF5_NAME, prolog << "Data ACCESS build_data(): set the including attribute flag to false: "<<filename << endl);
890 data_bdds->set_ia_flag(false);
891
892 }
893 catch(InternalErr & e) {
894
895 if(cf_fileid !=-1)
896 H5Fclose(cf_fileid);
897
898 if(fileid != -1)
899 H5Fclose(fileid);
900
901 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
902 __FILE__, __LINE__);
903 }
904 catch(Error & e) {
905
906 if(cf_fileid !=-1)
907 H5Fclose(cf_fileid);
908 if(fileid !=-1)
909 H5Fclose(fileid);
910
911 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
912 __FILE__, __LINE__);
913 }
914 catch(...) {
915
916 if(cf_fileid !=-1)
917 H5Fclose(cf_fileid);
918 if(fileid !=-1)
919 H5Fclose(fileid);
920
921 string s = "unknown exception caught building HDF5 DDS";
922 throw BESInternalFatalError(s, __FILE__, __LINE__);
923 }
924
925}
926
927
928#if 0
929// OLD function: Keep it for a while
930// Convenient function that helps build DDS and Data
931void HDF5RequestHandler::get_dds_with_attributes(const string &filename, const string &container_name, DDS*dds) {
932
933 // For the time being, separate CF file ID from the default file ID(mainly for debugging)
934 hid_t fileid = -1;
935 hid_t cf_fileid = -1;
936
937 try {
938
939 // Look in memory cache to see if it's initialized
940 DDS* cached_dds_ptr = 0;
941 if (dds_cache && (cached_dds_ptr = static_cast<DDS*>(dds_cache->get(filename)))) {
942 // copy the cached DDS into the BES response object. Assume that any cached DDS
943 // includes the DAS information.
944 BESDEBUG(HDF5_NAME, prolog << "DDS Cached hit for : " << filename << endl);
945 *dds = *cached_dds_ptr; // Copy the referenced object
946 }
947
948 else {
949
950 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
951 if (!container_name.empty())
952 dds->container_name(container_name);
953 dds->filename(filename);
954
955 // For the time being, not mess up CF's fileID with Default's fileID
956 if(true == _usecf) {
957// This block cannot be used to cache the data
958#if 0
959 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
960 string dds_filename = "/tmp/"+base_filename+"_dds";
961 FILE *dds_file = fopen(dds_filename.c_str(),"r");
962cerr<<"before parsing "<<endl;
963BaseTypeFactory tf;
964DDS tdds(&tf,name_path(filename),"3.2");
965tdds.filename(filename);
966 //dds->parse(dds_file);
967 tdds.parse(dds_file);
968 //DDS *cache_dds = new DDS(tdds);
969 cache_dds = new DDS(tdds);
970if(dds!=NULL)
971 delete dds;
972dds = cache_dds;
973tdds.print(cout);
974dds->print(cout);
975cerr<<"after parsing "<<endl;
976//dds->print(cout);
977 // fclose(dds_file);
978//#endif
979
980#endif
981// end of this block
982
983
984 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
985 if (cf_fileid < 0){
986 string invalid_file_msg="Could not open this HDF5 file ";
987 invalid_file_msg +=filename;
988 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
989 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
990 invalid_file_msg +=" distributor.";
991 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
992 }
993//#if 0
994 read_cfdds(*dds,filename,cf_fileid);
995//#endif
996 // Generate the DDS cached file
997 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
998 string dds_filename = "/tmp/"+base_filename+"_dds";
999 FILE *dds_file = fopen(dds_filename.c_str(),"w");
1000 dds->print(dds_file);
1001 fclose(dds_file);
1002
1003 }
1004 else {
1005
1006 fileid = get_fileid(filename.c_str());
1007 if (fileid < 0) {
1008 string invalid_file_msg="Could not open this HDF5 file ";
1009 invalid_file_msg +=filename;
1010 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1011 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1012 invalid_file_msg +=" distributor.";
1013 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1014 }
1015
1016 depth_first(fileid, (char*)"/", *dds, filename.c_str());
1017
1018 }
1019
1020
1021 // Check semantics
1022 if (!dds->check_semantics()) { // DDS didn't comply with the semantics
1023 dds->print(cerr);
1024 throw InternalErr(__FILE__, __LINE__,
1025 "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
1026 }
1027
1028 Ancillary::read_ancillary_dds( *dds, filename ) ;
1029
1030
1031 // Check DAS cache
1032 DAS *das = 0 ;
1033
1034 if (das_cache && (das = static_cast<DAS*>(das_cache->get(filename)))) {
1035 BESDEBUG(HDF5_NAME, prolog << "DAS Cached hit for : " << filename << endl);
1036 dds->transfer_attributes(das); // no need to copy the cached DAS
1037 }
1038
1039 else {
1040
1041 das = new DAS ;
1042
1043 if (!container_name.empty())
1044 das->container_name(container_name);
1045
1046 if (true == _usecf) {
1047
1048 // go to the CF option
1049 read_cfdas( *das,filename,cf_fileid);
1050
1051 }
1052 else {
1053
1054 find_gloattr(fileid, *das);
1055 depth_first(fileid, "/", *das);
1056 close_fileid(fileid);
1057 }
1058
1059 if(cf_fileid != -1)
1060 H5Fclose(cf_fileid);
1061
1062 Ancillary::read_ancillary_das( *das, filename ) ;
1063
1064 dds->transfer_attributes(das);
1065
1066
1067 // Only free the DAS if it's not added to the cache
1068 if (das_cache) {
1069 // add a copy
1070 BESDEBUG(HDF5_NAME, prolog << "DAS added to the cache for : " << filename << endl);
1071 //das_cache->add(new DAS(*das), filename);
1072 das_cache->add(das, filename);
1073 }
1074 else {
1075 delete das;
1076 }
1077 }
1078
1079 if (dds_cache) {
1080 // add a copy
1081 BESDEBUG(HDF5_NAME, prolog << "DDS added to the cache for : " << filename << endl);
1082 dds_cache->add(new DDS(*dds), filename);
1083 }
1084
1085 }
1086
1087//dds->print(cout);
1088
1089 }
1090 catch(InternalErr & e) {
1091
1092 if(cf_fileid !=-1)
1093 H5Fclose(cf_fileid);
1094
1095 if(fileid != -1)
1096 H5Fclose(fileid);
1097
1098 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1099 __FILE__, __LINE__);
1100 }
1101 catch(Error & e) {
1102
1103 if(cf_fileid !=-1)
1104 H5Fclose(cf_fileid);
1105 if(fileid !=-1)
1106 H5Fclose(fileid);
1107
1108 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1109 __FILE__, __LINE__);
1110 }
1111 catch(...) {
1112
1113 if(cf_fileid !=-1)
1114 H5Fclose(cf_fileid);
1115 if(fileid !=-1)
1116 H5Fclose(fileid);
1117
1118 string s = "unknown exception caught building HDF5 DDS";
1119 throw BESInternalFatalError(s, __FILE__, __LINE__);
1120 }
1121
1122}
1123#endif
1124
1125// Build DDS
1126bool HDF5RequestHandler::hdf5_build_dds(BESDataHandlerInterface & dhi)
1127{
1128 BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
1129#if DYNAMIC_CONFIG_ENABLED
1130 load_config();
1131#endif
1132
1133 // Obtain the HDF5 file name.
1134 string filename = dhi.container->access();
1135
1136 string container_name = dhi.container->get_symbolic_name();
1137 BESResponseObject *response = dhi.response_handler->get_response_object();
1138 BESDDSResponse *bdds = dynamic_cast < BESDDSResponse * >(response);
1139 if( !bdds )
1140 throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
1141 bdds->set_container(container_name);
1142
1143 try {
1144
1145 bool dds_from_dc = false;
1146 bool das_from_dc = false;
1147 bool build_data = false;
1148 string dds_cache_fname;
1149 string das_cache_fname;
1150
1151 if(_use_disk_meta_cache) {
1152
1153 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
1154
1155 // The _use_disk_dds_cache is always set to false by default
1156 if(_use_disk_dds_cache) {
1157 dds_cache_fname = _disk_meta_cache_path+"/" +base_filename+"_dds";
1158 if(access(dds_cache_fname.c_str(),F_OK) !=-1)
1159 dds_from_dc = true;
1160 }
1161
1162 das_cache_fname = _disk_meta_cache_path+"/" +base_filename+"_das";
1163 // Check if das files exist
1164 if(access(das_cache_fname.c_str(),F_OK) !=-1)
1165 das_from_dc = true;
1166
1167 }
1168
1169 get_dds_with_attributes(bdds, NULL,container_name,filename, dds_cache_fname,das_cache_fname,dds_from_dc,das_from_dc,build_data);
1170
1171 // The following block reads dds from a dds cache file.
1172#if 0
1173 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
1174 string dds_filename = "/tmp/"+base_filename+"_dds";
1175
1176 BaseTypeFactory tf;
1177 DDS tdds(&tf,name_path(filename),"3.2");
1178 tdds.filename(filename);
1179
1180
1181 FILE *dds_file = fopen(dds_filename.c_str(),"r");
1182 tdds.parse(dds_file);
1183//cerr<<"before parsing "<<endl;
1184 DDS* cache_dds = new DDS(tdds);
1185 if(dds != NULL)
1186 delete dds;
1187 bdds->set_dds(cache_dds);
1188 fclose(dds_file);
1189#endif
1190
1191 bdds->set_constraint( dhi ) ;
1192 bdds->clear_container() ;
1193
1194 }
1195 catch(BESError & e) {
1196 BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
1197 throw;
1198 }
1199 catch(InternalErr & e) {
1200
1201 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1202 __FILE__, __LINE__);
1203 }
1204 catch(Error & e) {
1205
1206 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1207 __FILE__, __LINE__);
1208 }
1209 catch(...) {
1210
1211 string s = "unknown exception caught building HDF5 DDS";
1212 throw BESInternalFatalError(s, __FILE__, __LINE__);
1213 }
1214
1215 BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1216 return true;
1217}
1218
1219bool HDF5RequestHandler::hdf5_build_data(BESDataHandlerInterface & dhi)
1220{
1221 BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
1222#if DYNAMIC_CONFIG_ENABLED
1223 load_config();
1224#endif
1225
1226 if(true ==_usecf) {
1227
1228 if(true == _pass_fileid)
1229 return hdf5_build_data_with_IDs(dhi);
1230 }
1231
1232 string filename = dhi.container->access();
1233
1234 string container_name = dhi.container->get_symbolic_name();
1235 BESResponseObject *response = dhi.response_handler->get_response_object();
1236 BESDataDDSResponse *bdds = dynamic_cast < BESDataDDSResponse * >(response);
1237 if( !bdds )
1238 throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
1239 bdds->set_container(container_name);
1240
1241 try {
1242
1243 bool dds_from_dc = false;
1244 bool das_from_dc = false;
1245 bool build_data = true;
1246 string dds_cache_fname;
1247 string das_cache_fname;
1248
1249
1250 // Only DAS is read from the cache. dds_from_dc is always false.
1251 if(_use_disk_meta_cache == true) {
1252
1253 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
1254 das_cache_fname = _disk_meta_cache_path+"/" +base_filename+"_das";
1255
1256 if(access(das_cache_fname.c_str(),F_OK) !=-1)
1257 das_from_dc = true;
1258
1259 }
1260
1261 //get_dds_with_attributes(NULL,bdds, container_name,filename, dds_cache_fname,das_cache_fname,dds_from_dc,das_from_dc,build_data);
1262 get_dds_without_attributes_datadds(bdds,container_name,filename);
1263
1264 bdds->set_constraint( dhi ) ;
1265 bdds->clear_container() ;
1266
1267 }
1268 catch(BESError & e) {
1269 BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
1270 throw;
1271 }
1272 catch(InternalErr & e) {
1273
1274 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1275 __FILE__, __LINE__);
1276 }
1277 catch(Error & e) {
1278
1279 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1280 __FILE__, __LINE__);
1281 }
1282 catch(...) {
1283
1284 string s = "unknown exception caught building HDF5 DDS";
1285 throw BESInternalFatalError(s, __FILE__, __LINE__);
1286 }
1287
1288 BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1289 return true;
1290}
1291
1292// Obtain data when turning on the pass fileID key.The memory cache is not used.
1293bool HDF5RequestHandler::hdf5_build_data_with_IDs(BESDataHandlerInterface & dhi)
1294{
1295 BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
1296#if DYNAMIC_CONFIG_ENABLED
1297 load_config();
1298#endif
1299
1300 BESDEBUG(HDF5_NAME,prolog << "Building DataDDS by passing file IDs. "<<endl);
1301 hid_t cf_fileid = -1;
1302
1303 string filename = dhi.container->access();
1304
1305 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
1306 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1307 if (cf_fileid < 0){
1308 string invalid_file_msg="Could not open this HDF5 file ";
1309 invalid_file_msg +=filename;
1310 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1311 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1312 invalid_file_msg +=" distributor.";
1313 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1314 }
1315
1316 BESResponseObject *response = dhi.response_handler->get_response_object();
1317 BESDataDDSResponse *bdds = dynamic_cast < BESDataDDSResponse * >(response);
1318 if( !bdds )
1319 throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
1320
1321 try {
1322
1323 bdds->set_container( dhi.container->get_symbolic_name() ) ;
1324
1325 HDF5DDS *hdds = new HDF5DDS(bdds->get_dds());
1326 delete bdds->get_dds();
1327
1328 bdds->set_dds(hdds);
1329
1330 hdds->setHDF5Dataset(cf_fileid);
1331
1332 read_cfdds( *hdds,filename,cf_fileid);
1333
1334 if (!hdds->check_semantics()) { // DDS didn't comply with the DAP semantics
1335 hdds->print(cerr);
1336 throw InternalErr(__FILE__, __LINE__,
1337 "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1338 }
1339
1340 Ancillary::read_ancillary_dds( *hdds, filename ) ;
1341
1342 DAS *das = new DAS ;
1343 BESDASResponse bdas( das ) ;
1345 read_cfdas( *das,filename,cf_fileid);
1346 Ancillary::read_ancillary_das( *das, filename ) ;
1347
1348 hdds->transfer_attributes(das);
1349 bdds->set_constraint( dhi ) ;
1350 bdds->clear_container() ;
1351
1352 }
1353
1354 catch(BESError & e) {
1355 if(cf_fileid !=-1)
1356 H5Fclose(cf_fileid);
1357 BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
1358 throw;
1359 }
1360 catch(InternalErr & e) {
1361 if(cf_fileid !=-1)
1362 H5Fclose(cf_fileid);
1363 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1364 __FILE__, __LINE__);
1365 }
1366 catch(Error & e) {
1367 if(cf_fileid !=-1)
1368 H5Fclose(cf_fileid);
1369 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1370 __FILE__, __LINE__);
1371 }
1372 catch(...) {
1373 if(cf_fileid !=-1)
1374 H5Fclose(cf_fileid);
1375 string s = "unknown exception caught building HDF5 DataDDS";
1376 throw BESInternalFatalError(s, __FILE__, __LINE__);
1377 }
1378
1379 BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1380 return true;
1381}
1382
1383bool HDF5RequestHandler::hdf5_build_dmr(BESDataHandlerInterface & dhi)
1384{
1385 BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
1386#if DYNAMIC_CONFIG_ENABLED
1387 load_config();
1388#endif
1389
1390 // Extract the DMR Response object - this holds the DMR used by the
1391 // other parts of the framework.
1392 BESResponseObject *response = dhi.response_handler->get_response_object();
1393 BESDMRResponse &bes_dmr_response = dynamic_cast<BESDMRResponse &>(*response);
1394
1395 string filename = dhi.container->access();
1396
1397 DMR *dmr = bes_dmr_response.get_dmr();
1398
1399 // For the time being, separate CF file ID from the default file ID(mainly for debugging)
1400 hid_t fileid = -1;
1401 hid_t cf_fileid = -1;
1402
1403 try {
1404
1405 DMR* cached_dmr_ptr = 0;
1406 if (dmr_cache){
1407 BESDEBUG(HDF5_NAME, prolog << "Checking DMR cache for : " << filename << endl);
1408 cached_dmr_ptr = static_cast<DMR*>(dmr_cache->get(filename));
1409 }
1410
1411 if (cached_dmr_ptr) {
1412 // copy the cached DMR into the BES response object
1413 BESDEBUG(HDF5_NAME, prolog << "DMR cache hit for : " << filename << endl);
1414 *dmr = *cached_dmr_ptr; // Copy the referenced object
1415 dmr->set_request_xml_base(bes_dmr_response.get_request_xml_base());
1416 }
1417 else {// No cache
1418
1419 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
1420 D4BaseTypeFactory MyD4TypeFactory;
1421 dmr->set_factory(&MyD4TypeFactory);
1422
1423 if(true ==_usecf) {// CF option
1424
1425 if(true == _usecfdmr) {
1426 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1427 if (cf_fileid < 0){
1428 string invalid_file_msg="Could not open this HDF5 file ";
1429 invalid_file_msg +=filename;
1430 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1431 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1432 invalid_file_msg +=" distributor.";
1433 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1434 }
1435 read_cfdmr(dmr,filename,cf_fileid);
1436 H5Fclose(cf_fileid);
1437 bes_dmr_response.set_dap4_constraint(dhi);
1438 bes_dmr_response.set_dap4_function(dhi);
1439 dmr->set_factory(0);
1440
1441 BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1442
1443 return true;
1444 }
1445
1446 if(true == _pass_fileid)
1447 return hdf5_build_dmr_with_IDs(dhi);
1448
1449 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1450 if (cf_fileid < 0){
1451 string invalid_file_msg="Could not open this HDF5 file ";
1452 invalid_file_msg +=filename;
1453 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1454 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1455 invalid_file_msg +=" distributor.";
1456 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1457 }
1458
1459
1460 BaseTypeFactory factory;
1461 DDS dds(&factory, name_path(filename), "3.2");
1462 dds.filename(filename);
1463
1464 DAS das;
1465
1466 // For the CF option dmr response, we need to map 64-bit integer separately
1467 // So set the flag to map 64-bit integer.
1468 HDF5RequestHandler::set_dmr_64bit_int(dmr);
1469 read_cfdds( dds,filename,cf_fileid);
1470 if (!dds.check_semantics()) { // DDS didn't comply with the DAP semantics
1471 dds.print(cerr);
1472 throw InternalErr(__FILE__, __LINE__,
1473 "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1474 }
1475
1476 read_cfdas(das,filename,cf_fileid);
1477 Ancillary::read_ancillary_das( das, filename ) ;
1478
1479 dds.transfer_attributes(&das);
1480
1482 if(cf_fileid !=-1)
1483 H5Fclose(cf_fileid);
1484
1485 dmr->build_using_dds(dds);
1486
1487 }// "if(true == _usecf)"
1488 else {// default option
1489
1490 // Obtain the HDF5 file ID.
1491 fileid = get_fileid(filename.c_str());
1492 if (fileid < 0) {
1493 string invalid_file_msg="Could not open this HDF5 file ";
1494 invalid_file_msg +=filename;
1495 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1496 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1497 invalid_file_msg +=" distributor.";
1498 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1499 }
1500
1501 bool use_dimscale = false;
1502 if(true == _default_handle_dimension)
1503 use_dimscale = check_dimscale(fileid);
1504 dmr->set_name(name_path(filename));
1505 dmr->set_filename(name_path(filename));
1506
1507 // The breadth_first() function builds the variables and attributes and
1508 // loads them into the root group (building child groups as needed).
1509 // jhrg 4/30/20
1510 D4Group* root_grp = dmr->root();
1511 BESDEBUG("h5", "use_dimscale is "<< use_dimscale <<endl);
1512
1513 // It is possible that a dimension variable has hardlinks. To make it
1514 // right for the netCDF-4 data model and the current DAP4 implementation,
1515 // we need to choose the shortest path of all hardlinks as the dimension path.
1516 // So to avoid iterate all HDF5 objects multiple times, save the found
1517 // hardlinks and search them when necessary. Note we have to search hardlinks from the root.
1518 // KY 2021-11-15
1519 vector<link_info_t> hdf5_hls;
1520 breadth_first(fileid, fileid,(char*)"/",root_grp,filename.c_str(),use_dimscale,hdf5_hls);
1521#if 0
1522 BESDEBUG("h5", "build_dmr - before obtain dimensions"<< endl);
1523 D4Dimensions *root_dims = root_grp->dims();
1524 for(D4Dimensions::D4DimensionsIter di = root_dims->dim_begin(), de = root_dims->dim_end(); di != de; ++di) {
1525 BESDEBUG("fonc", "transform_dap4() - check dimensions"<< endl);
1526 BESDEBUG("fonc", "transform_dap4() - dim name is: "<<(*di)->name()<<endl);
1527 BESDEBUG("fonc", "transform_dap4() - dim size is: "<<(*di)->size()<<endl);
1528 BESDEBUG("fonc", "transform_dap4() - fully_qualfied_dim name is: "<<(*di)->fully_qualified_name()<<endl);
1529 //cout <<"dim size is: "<<(*di)->size()<<endl;
1530 //cout <<"dim fully_qualified_name is: "<<(*di)->fully_qualified_name()<<endl;
1531 }
1532 BESDEBUG("h5", "build_dmr - after obtain dimensions"<< endl);
1533#endif
1534
1535#if 0
1536 if(true == use_dimscale)
1537 //breadth_first(fileid,(char*)"/",*dmr,root_grp,filename.c_str(),true);
1538 breadth_first(fileid,(char*)"/",root_grp,filename.c_str(),true);
1539 else
1540 depth_first(fileid,(char*)"/",root_grp,filename.c_str());
1541 //depth_first(fileid,(char*)"/",*dmr,root_grp,filename.c_str());
1542#endif
1543
1544 close_fileid(fileid);
1545
1546 }// else (default option)
1547
1548 // If the cache is turned on, add the memory cache.
1549 if (dmr_cache) {
1550 // add a copy
1551 BESDEBUG(HDF5_NAME, prolog << "DMR added to the cache for : " << filename << endl);
1552 dmr_cache->add(new DMR(*dmr), filename);
1553 }
1554 }// else no cache
1555 }// try
1556 catch(BESError & e) {
1557 if(cf_fileid !=-1)
1558 H5Fclose(cf_fileid);
1559 if(fileid !=-1)
1560 H5Fclose(fileid);
1561 BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
1562 throw;
1563 }
1564 catch(InternalErr & e) {
1565
1566 if(cf_fileid !=-1)
1567 H5Fclose(cf_fileid);
1568 if(fileid !=-1)
1569 H5Fclose(fileid);
1570
1571 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1572 __FILE__, __LINE__);
1573 }
1574 catch(Error & e) {
1575
1576 if(cf_fileid !=-1)
1577 H5Fclose(cf_fileid);
1578 if(fileid !=-1)
1579 H5Fclose(fileid);
1580 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1581 __FILE__, __LINE__);
1582 }
1583 catch(...) {
1584
1585 if(cf_fileid !=-1)
1586 H5Fclose(cf_fileid);
1587 if(fileid !=-1)
1588 H5Fclose(fileid);
1589 string s = "unknown exception caught building HDF5 DMR";
1590 throw BESInternalFatalError(s, __FILE__, __LINE__);
1591 }
1592
1593 //dmr->print(cout);
1594
1595 // Instead of fiddling with the internal storage of the DHI object,
1596 // (by setting dhi.data[DAP4_CONSTRAINT], etc., directly) use these
1597 // methods to set the constraints. But, why? Ans: from Patrick is that
1598 // in the 'container' mode of BES each container can have a different
1599 // CE.
1600 bes_dmr_response.set_dap4_constraint(dhi);
1601 bes_dmr_response.set_dap4_function(dhi);
1602 dmr->set_factory(0);
1603
1604 BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1605 return true;
1606}
1607
1608// This function is only used when EnableCF is true.
1609bool HDF5RequestHandler::hdf5_build_dmr_with_IDs(BESDataHandlerInterface & dhi)
1610{
1611 BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
1612#if DYNAMIC_CONFIG_ENABLED
1613 load_config();
1614#endif
1615
1616 BESDEBUG("h5","Building DMR with passing file IDs. "<<endl);
1617 string filename = dhi.container->access();
1618 hid_t cf_fileid = -1;
1619
1620 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
1621 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1622 if (cf_fileid < 0){
1623 string invalid_file_msg="Could not open this HDF5 file ";
1624 invalid_file_msg +=filename;
1625 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1626 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1627 invalid_file_msg +=" distributor.";
1628 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1629 }
1630
1631 BaseTypeFactory factory;
1632 DDS dds(&factory, name_path(filename), "3.2");
1633 dds.filename(filename);
1634
1635 DAS das;
1636
1637 try {
1638
1639
1640 // This is the CF option
1641 read_cfdds( dds,filename,cf_fileid);
1642
1643 if (!dds.check_semantics()) { // DDS didn't comply with the DAP semantics
1644 dds.print(cerr);
1645 throw InternalErr(__FILE__, __LINE__,
1646 "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1647 }
1648
1649 Ancillary::read_ancillary_dds( dds, filename ) ;
1650
1651
1652 read_cfdas(das,filename,cf_fileid);
1653
1654 Ancillary::read_ancillary_das( das, filename ) ;
1655
1656 dds.transfer_attributes(&das);
1657
1659 //if(cf_fileid !=-1)
1660 // H5Fclose(cf_fileid);
1661
1662 }
1663 catch(BESError & e) {
1664 if(cf_fileid !=-1)
1665 H5Fclose(cf_fileid);
1666 BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
1667 throw;
1668 }
1669 catch(InternalErr & e) {
1670
1671 if(cf_fileid !=-1)
1672 H5Fclose(cf_fileid);
1673
1674 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1675 __FILE__, __LINE__);
1676 }
1677 catch(Error & e) {
1678
1679 if(cf_fileid !=-1)
1680 H5Fclose(cf_fileid);
1681
1682 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1683 __FILE__, __LINE__);
1684 }
1685 catch(...) {
1686
1687 if(cf_fileid !=-1)
1688 H5Fclose(cf_fileid);
1689
1690 string s = "unknown exception caught building HDF5 DataDDS";
1691 throw BESInternalFatalError(s, __FILE__, __LINE__);
1692 }
1693
1694 // Extract the DMR Response object - this holds the DMR used by the
1695 // other parts of the framework.
1696 BESResponseObject *response = dhi.response_handler->get_response_object();
1697 BESDMRResponse &bes_dmr = dynamic_cast<BESDMRResponse &>(*response);
1698
1699 // In this handler we use a different pattern since the handler specializes the DDS/DMR.
1700 // First, build the DMR adding the open handle to the HDF4 dataset, then free the DMR
1701 // the BES built and add this one. The HDF4DMR object will close the open dataset when
1702 // the BES runs the DMR's destructor.
1703
1704 DMR *dmr = bes_dmr.get_dmr();
1705 D4BaseTypeFactory MyD4TypeFactory;
1706 dmr->set_factory(&MyD4TypeFactory);
1707 dmr->build_using_dds(dds);
1708
1709 HDF5DMR *hdf5_dmr = new HDF5DMR(dmr);
1710 hdf5_dmr->setHDF5Dataset(cf_fileid);
1711 delete dmr; // The call below will make 'dmr' unreachable; delete it now to avoid a leak.
1712 bes_dmr.set_dmr(hdf5_dmr); // BESDMRResponse will delete hdf5_dmr
1713
1714 // Instead of fiddling with the internal storage of the DHI object,
1715 // (by setting dhi.data[DAP4_CONSTRAINT], etc., directly) use these
1716 // methods to set the constraints. But, why? Ans: from Patrick is that
1717 // in the 'container' mode of BES each container can have a different
1718 // CE.
1719 bes_dmr.set_dap4_constraint(dhi);
1720 bes_dmr.set_dap4_function(dhi);
1721 hdf5_dmr->set_factory(0);
1722
1723 BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1724 return true;
1725}
1726
1727bool HDF5RequestHandler::hdf5_build_help(BESDataHandlerInterface & dhi)
1728{
1729 BESResponseObject *response = dhi.response_handler->get_response_object();
1730 BESInfo *info = dynamic_cast<BESInfo *>(response);
1731 if( !info )
1732 throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
1733
1734 string add_info="Just for Test";
1735
1736 map<string,string> attrs ;
1737 attrs["name"] = MODULE_NAME ;
1738 attrs["version"] = MODULE_VERSION ;
1739 list<string> services ;
1740 BESServiceRegistry::TheRegistry()->services_handled( HDF5_NAME, services );
1741 if( services.size() > 0 )
1742 {
1743 string handles = BESUtil::implode( services, ',' ) ;
1744 attrs["handles"] = handles ;
1745 }
1746 info->begin_tag( "module", &attrs ) ;
1747 info->end_tag( "module" ) ;
1748 info->add_data(add_info);
1749
1750 return true;
1751}
1752
1753bool HDF5RequestHandler::hdf5_build_version(BESDataHandlerInterface & dhi)
1754{
1755 BESResponseObject *response = dhi.response_handler->get_response_object();
1756 BESVersionInfo *info = dynamic_cast < BESVersionInfo * >(response);
1757 if( !info )
1758 throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
1759
1760 info->add_module( MODULE_NAME, MODULE_VERSION ) ;
1761
1762 return true;
1763}
1764
1765
1766bool HDF5RequestHandler::obtain_lrd_common_cache_dirs()
1767{
1768 string lrd_config_fpath;
1769 string lrd_config_fname;
1770
1771 // Obtain DataCache path
1772 lrd_config_fpath = get_beskeys("H5.DataCachePath");
1773
1774 // Obtain the configure file name that specifics the large file configuration
1775 lrd_config_fname = get_beskeys("H5.LargeDataMemCacheFileName");
1776
1777 // If either the configure file path or fname is missing, won't add specific mem. cache dirs.
1778 if(lrd_config_fpath=="" || lrd_config_fname=="")
1779 return false;
1780
1781 // temp_line for storing info of one line in the config. file
1782 string temp_line;
1783
1784 // The full path of the configure file
1785 string mcache_config_fname = lrd_config_fpath+"/"+lrd_config_fname;
1786
1787 //ifstream mcache_config_file("example.txt");
1788 // Open the configure file
1789 ifstream mcache_config_file(mcache_config_fname.c_str());
1790
1791 // If the configuration file is not open, return false.
1792 if(mcache_config_file.is_open()==false){
1793 BESDEBUG(HDF5_NAME, prolog << "The large data memory cache configure file "<<mcache_config_fname );
1794 BESDEBUG(HDF5_NAME, prolog << " cannot be opened."<<endl);
1795 return false;
1796 }
1797
1798 // Read the configuration file line by line
1799 while(getline(mcache_config_file,temp_line)) {
1800
1801 // Only consider lines that is no less than 2 characters and the 2nd character is space.
1802 if(temp_line.size()>1 && temp_line.at(1)==' ') {
1803 char sep=' ';
1804 string subline = temp_line.substr(2);
1805 vector<string> temp_name_list;
1806
1807 // Include directories to store common latitude and longitude values
1808 if(temp_line.at(0)=='1') {
1809 HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1810 //lrd_cache_dir_list +=temp_name_list;
1811 lrd_cache_dir_list.insert(lrd_cache_dir_list.end(),temp_name_list.begin(),temp_name_list.end());
1812 }
1813 // Include directories not to store common latitude and longitude values
1814 else if(temp_line.at(0)=='0'){
1815 HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1816 //lrd_non_cache_dir_list +=temp_name_list;
1817 lrd_non_cache_dir_list.insert(lrd_non_cache_dir_list.end(),temp_name_list.begin(),temp_name_list.end());
1818 }
1819 // Include variable names that the server would like to store in the memory cache
1820 else if(temp_line.at(0)=='2') {
1821
1822 // We need to handle the space case inside a variable path
1823 // either "" or '' needs to be used to identify a var path
1824 vector<int>dq_pos;
1825 vector<int>sq_pos;
1826 for(unsigned int i = 0; i<subline.size();i++){
1827 if(subline[i]=='"') {
1828 dq_pos.push_back(i);
1829 }
1830 else if(subline[i]=='\'')
1831 sq_pos.push_back(i);
1832 }
1833 if(dq_pos.size()==0 && sq_pos.size()==0)
1834 HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1835 else if((dq_pos.size()!=0) &&(dq_pos.size()%2==0)&& sq_pos.size()==0) {
1836 unsigned int dq_index= 0;
1837 while(dq_index < dq_pos.size()){
1838 if(dq_pos[dq_index+1]>(dq_pos[dq_index]+1)) {
1839 temp_name_list.push_back
1840 (subline.substr(dq_pos[dq_index]+1,dq_pos[dq_index+1]-dq_pos[dq_index]-1));
1841 }
1842 dq_index = dq_index + 2;
1843 }
1844 }
1845 else if((sq_pos.size()!=0) &&(sq_pos.size()%2==0)&& dq_pos.size()==0) {
1846 unsigned int sq_index= 0;
1847 while(sq_index < sq_pos.size()){
1848 if(sq_pos[sq_index+1]>(sq_pos[sq_index]+1)) {
1849 temp_name_list.push_back
1850 (subline.substr(sq_pos[sq_index]+1,sq_pos[sq_index+1]-sq_pos[sq_index]-1));
1851 }
1852 sq_index = sq_index+2;
1853 }
1854 }
1855
1856 lrd_var_cache_file_list.insert(lrd_var_cache_file_list.end(),temp_name_list.begin(),temp_name_list.end());
1857 }
1858 }
1859 }
1860
1861
1862#if 0
1863
1864for(int i =0; i<lrd_cache_dir_list.size();i++)
1865cerr<<"lrd cache list is "<<lrd_cache_dir_list[i] <<endl;
1866for(int i =0; i<lrd_non_cache_dir_list.size();i++)
1867cerr<<"lrd non cache list is "<<lrd_non_cache_dir_list[i] <<endl;
1868for(int i =0; i<lrd_var_cache_file_list.size();i++)
1869cerr<<"lrd var cache file list is "<<lrd_var_cache_file_list[i] <<endl;
1870#endif
1871
1872
1873 mcache_config_file.close();
1874 if(lrd_cache_dir_list.size()==0 && lrd_non_cache_dir_list.size()==0 && lrd_var_cache_file_list.size()==0)
1875 return false;
1876 else
1877 return true;
1878}
1879
1880
1881bool HDF5RequestHandler::read_das_from_disk_cache(const string & cache_filename,DAS *das_ptr) {
1882
1883 BESDEBUG(HDF5_NAME, prolog << "Coming to read_das_from_disk_cache() " << cache_filename << endl);
1884 bool ret_value = true;
1885 FILE *md_file = NULL;
1886 md_file = fopen(cache_filename.c_str(),"rb");
1887
1888 if(NULL == md_file) {
1889 string bes_error = "An error occurred trying to open a metadata cache file " + cache_filename;
1890 throw BESInternalError( bes_error, __FILE__, __LINE__);
1891 }
1892 else {
1893
1894 int fd_md = fileno(md_file);
1895 struct flock *l_md;
1896 l_md = lock(F_RDLCK);
1897
1898 // hold a read(shared) lock to read metadata from a file.
1899 if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
1900 fclose(md_file);
1901 ostringstream oss;
1902 oss << "cache process: " << l_md->l_pid << " triggered a locking error: " << get_errno();
1903 throw BESInternalError( oss.str(), __FILE__, __LINE__);
1904 }
1905
1906 try {
1907
1908 struct stat sb;
1909 if(stat(cache_filename.c_str(),&sb) != 0) {
1910 string bes_error = "An error occurred trying to stat a metadata cache file size " + cache_filename;
1911 throw BESInternalError( bes_error, __FILE__, __LINE__);
1912
1913 }
1914
1915
1916 size_t bytes_expected_read=(size_t)sb.st_size;
1917 BESDEBUG(HDF5_NAME, prolog << "DAS Disk cache file size is " << bytes_expected_read << endl);
1918
1919 vector<char> buf;
1920 buf.resize(bytes_expected_read);
1921 size_t bytes_to_read =fread((void*)&buf[0],1,bytes_expected_read,md_file);
1922 if(bytes_to_read != bytes_expected_read)
1923 throw InternalErr(__FILE__,__LINE__,"Fail to read the data from the das cache file.");
1924
1925 char* temp_pointer =&buf[0];
1926
1927 AttrTable*at = NULL;
1928 // recursively build DAS
1929//#if 0
1930 temp_pointer = get_attr_info_from_dc(temp_pointer,das_ptr,at);
1931//#endif
1932
1933
1934 }
1935 catch(...) {
1936 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1937 fclose(md_file);
1938 throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1939 }
1940
1941 fclose(md_file);
1942 throw InternalErr(__FILE__,__LINE__,"Fail to parse a das cache file.");
1943 }
1944
1945 // Unlock the cache file
1946 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1947 fclose(md_file);
1948 throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1949 }
1950 fclose(md_file);
1951 }
1952 return ret_value;
1953
1954}
1955
1956// This fucntion will NOT be used by default. Leave here for future improvement.
1957bool HDF5RequestHandler::write_dds_to_disk_cache(const string& dds_cache_fname,DDS *dds_ptr) {
1958
1959 BESDEBUG(HDF5_NAME, prolog << "Write DDS to disk cache " << dds_cache_fname << endl);
1960 FILE *dds_file = fopen(dds_cache_fname.c_str(),"w");
1961
1962 if(NULL == dds_file) {
1963 string bes_error = "An error occurred trying to open a metadata cache file " + dds_cache_fname;
1964 throw BESInternalError( bes_error, __FILE__, __LINE__);
1965 }
1966 else {
1967
1968 int fd_md = fileno(dds_file);
1969 struct flock *l_md;
1970 l_md = lock(F_WRLCK);
1971
1972 // hold a read(shared) lock to read metadata from a file.
1973 if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
1974 fclose(dds_file);
1975 ostringstream oss;
1976 oss << "cache process: " << l_md->l_pid << " triggered a locking error: " << get_errno();
1977 throw BESInternalError( oss.str(), __FILE__, __LINE__);
1978 }
1979
1980 try {
1981 dds_ptr->print(dds_file);
1982 }
1983 catch(...) {
1984 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1985 fclose(dds_file);
1986 throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1987 }
1988
1989 fclose(dds_file);
1990 throw InternalErr(__FILE__,__LINE__,"Fail to parse a dds cache file.");
1991 }
1992
1993 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1994 fclose(dds_file);
1995 throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1996 }
1997
1998 fclose(dds_file);
1999 }
2000 return true;
2001
2002}
2003
2004// Write DAS to a binary cached file on disk.
2005bool HDF5RequestHandler::write_das_to_disk_cache(const string & das_cache_fname, DAS *das_ptr) {
2006
2007 BESDEBUG(HDF5_NAME, prolog << "Write DAS to disk cache " << das_cache_fname << endl);
2008 FILE *das_file = fopen(das_cache_fname.c_str(),"wb");
2009 if(NULL == das_file) {
2010 string bes_error = "An error occurred trying to open a metadata cache file " + das_cache_fname;
2011 throw BESInternalError( bes_error, __FILE__, __LINE__);
2012 }
2013 else {
2014 int fd_md = fileno(das_file);
2015 struct flock *l_md;
2016 l_md = lock(F_WRLCK);
2017
2018 // hold a write(exclusive) lock to write metadata to a file.
2019 if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
2020 fclose(das_file);
2021 ostringstream oss;
2022 oss << "cache process: " << l_md->l_pid << " triggered a locking error: " << get_errno();
2023 throw BESInternalError( oss.str(), __FILE__, __LINE__);
2024 }
2025
2026 try {
2027 write_das_to_file(das_ptr,das_file);
2028 }
2029 catch(...) {
2030 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
2031 fclose(das_file);
2032 throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2033 }
2034
2035 fclose(das_file);
2036 throw InternalErr(__FILE__,__LINE__,"Fail to parse a dds cache file.");
2037 }
2038
2039 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
2040 fclose(das_file);
2041 throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2042 }
2043
2044 fclose(das_file);
2045
2046 }
2047
2048 return true;
2049
2050}
2051
2052// The wrapper function to call write_das_table_to_file to generate the cache.
2053void write_das_to_file(DAS*das_ptr,FILE* das_file) {
2054
2055 // When category_flag is 2, it marks the end of the file.
2056 uint8_t category_flag = 2;
2057 AttrTable* top_table = das_ptr->get_top_level_attributes();
2058 write_das_table_to_file(top_table,das_file);
2059
2060 // Add the final ending flag for retrieving the info.
2061 fwrite((const void*)&category_flag,1,1,das_file);
2062 return;
2063
2064}
2065
2066// The main function to write DAS to a file
2067void write_das_table_to_file(AttrTable*temp_table,FILE* das_file) {
2068
2069 if(temp_table !=NULL) {
2070
2071 // 2 is the end mark of an attribute table
2072 uint8_t category_flag = 2;
2073
2074 // Loop through the whole DAS top table
2075 AttrTable::Attr_iter top_startit = temp_table->attr_begin();
2076 AttrTable::Attr_iter top_endit = temp_table->attr_end();
2077 AttrTable::Attr_iter top_it = top_startit;
2078 while(top_it !=top_endit) {
2079 AttrType atype = temp_table->get_attr_type(top_it);
2080 if(atype == Attr_unknown)
2081 throw InternalErr(__FILE__,__LINE__,"Unsupported DAS Attribute type");
2082 else if(atype!=Attr_container) {
2083 BESDEBUG(HDF5_NAME, prolog << "DAS to the disk cache, attr name is: "
2084 << temp_table->get_name(top_it) << endl);
2085 BESDEBUG(HDF5_NAME, prolog << "DAS to the disk cache, attr type is: "
2086 << temp_table->get_type(top_it) << endl);
2087 // For the debugging purpose
2088 //unsigned int num_attrs = temp_table->get_attr_num(temp_table->get_name(top_it));
2089 //cerr<<"Attribute values are "<<endl;
2090 //for (int i = 0; i <num_attrs;i++)
2091 // cerr<<(*(temp_table->get_attr_vector(temp_table->get_name(top_it))))[i]<<" ";
2092 //cerr<<endl;
2093 //write_das_attr_info(temp_table,top_it,das_file);
2094 // Write DAS attribute info to the file
2095 write_das_attr_info(temp_table,temp_table->get_name(top_it),temp_table->get_type(top_it),das_file);
2096 }
2097 else {
2098 BESDEBUG(HDF5_NAME, prolog << "DAS to the disk cache, attr container name is: "
2099 << (*top_it)->name << endl);
2100 // Write the container and then write the info. in this container
2101 AttrTable* sub_table = temp_table->get_attr_table(top_it);
2102 write_container_name_to_file(sub_table->get_name(),das_file);
2103 write_das_table_to_file(sub_table,das_file);
2104
2105 // Write the end flag
2106 fwrite((const void*)&category_flag,1,1,das_file);
2107
2108 }
2109 ++top_it;
2110 }
2111
2112 }
2113}
2114
2115// Write container name to the disk file
2116void write_container_name_to_file(const string& cont_name,FILE *das_file) {
2117
2118 // 1 marks the starting of a container
2119 uint8_t category_flag = 1;
2120 vector<char> buf;
2121 size_t bytes_to_write = cont_name.size()+sizeof(size_t)+1;
2122 buf.resize(bytes_to_write);
2123 char*temp_pointer =&buf[0];
2124 memcpy((void*)temp_pointer,(void*)&category_flag,1);
2125 temp_pointer++;
2126 temp_pointer=copy_str(temp_pointer,cont_name);
2127
2128 size_t bytes_to_be_written = fwrite((const void*)&buf[0],1,bytes_to_write,das_file);
2129 if(bytes_to_be_written != bytes_to_write)
2130 throw InternalErr(__FILE__, __LINE__,"Failed to write a DAS container name to a cache");
2131 return;
2132}
2133
2134
2135// Write DAS attribute info. to the disk cache file
2136void write_das_attr_info(AttrTable* dtp,const string& attr_name, const string & attr_type,FILE * das_file) {
2137
2138 // 0 marks the starting of a DAS attribute
2139 uint8_t category_flag = 0;
2140
2141 unsigned int num_attr_elems = dtp->get_attr_num(attr_name);
2142 vector<string> attr_values;
2143 size_t total_attr_values_size = 0;
2144 for (unsigned int i = 0; i <num_attr_elems;i++){
2145 attr_values.push_back((*(dtp->get_attr_vector(attr_name)))[i]);
2146 total_attr_values_size += attr_values[i].size();
2147 }
2148 // Need to add a flag, value as 0 to indicate the attribute.
2149 // DAS: category flag, sizeof attirubte name, attribute name, size of attribute type, attribute type
2150 size_t bytes_to_write_attr = 1 + attr_name.size() + attr_type.size() + 2* sizeof(size_t);
2151
2152 // One unsigned int to store the number of element elements i
2153 // + sizeof(size_t) * number of elements to store the number of characters for each attribute
2154 // (in DAP, every attribute is in string format)
2155 // +total size of all attribute values
2156 bytes_to_write_attr += sizeof(unsigned int) + num_attr_elems*sizeof(size_t)+total_attr_values_size;
2157
2158 vector<char>attr_buf;
2159 attr_buf.resize(bytes_to_write_attr);
2160 char* temp_attrp =&attr_buf[0];
2161
2162 // The attribute flag
2163 memcpy((void*)temp_attrp,(void*)&category_flag,1);
2164 temp_attrp++;
2165
2166 // The attribute name and type
2167 temp_attrp=copy_str(temp_attrp,attr_name);
2168 temp_attrp=copy_str(temp_attrp,attr_type);
2169
2170 // Number of elements
2171 memcpy((void*)temp_attrp,(void*)&num_attr_elems,sizeof(unsigned int));
2172 temp_attrp+=sizeof(unsigned int);
2173
2174 // All attributes
2175 for (unsigned int i = 0; i <num_attr_elems;i++)
2176 temp_attrp=copy_str(temp_attrp,(*(dtp->get_attr_vector(attr_name)))[i]);
2177
2178 size_t bytes_to_be_written = fwrite((const void*)&attr_buf[0],1,bytes_to_write_attr,das_file);
2179 if(bytes_to_be_written != bytes_to_write_attr)
2180 throw InternalErr(__FILE__, __LINE__,"Failed to write a DAS attribute to a cache");
2181
2182 return;
2183
2184}
2185
2186// Read DDS from a disk cache, this function is not used by default.
2187void HDF5RequestHandler::read_dds_from_disk_cache(BESDDSResponse* bdds, BESDataDDSResponse* data_bdds,
2188 bool build_data,const string & container_name,const string & h5_fname,
2189 const string & dds_cache_fname,const string &das_cache_fname, hid_t h5_fd,
2190 bool das_from_dc) {
2191
2192
2193 BESDEBUG(HDF5_NAME, prolog << "BEGIN dds_cache_fname: " << dds_cache_fname << endl);
2194
2195 DDS *dds;
2196 if(true == build_data)
2197 dds = data_bdds->get_dds();
2198 else
2199 dds = bdds->get_dds();
2200
2201 // write a function to pass the following with the lock.
2202 BaseTypeFactory tf;
2203 DDS tdds(&tf,name_path(h5_fname),"3.2");
2204 tdds.filename(h5_fname);
2205
2206 FILE *dds_file = fopen(dds_cache_fname.c_str(),"r");
2207 tdds.parse(dds_file);
2208 DDS* cache_dds = new DDS(tdds);
2209#if 0
2210cerr<<"before dds "<<endl;
2211dds->dump(cerr);
2212cerr<<"after dds "<<endl;
2213cerr<<"before tdds "<<endl;
2214cache_dds->dump(cerr);
2215cerr<<"after tdds "<<endl;
2216#endif
2217 if(dds != NULL)
2218 delete dds;
2219
2220 Ancillary::read_ancillary_dds( *cache_dds, h5_fname ) ;
2221
2222 add_das_to_dds(cache_dds,container_name,h5_fname,das_cache_fname,h5_fd,das_from_dc);
2223 if(true == build_data)
2224 data_bdds->set_dds(cache_dds);
2225 else
2226 bdds->set_dds(cache_dds);
2227 fclose(dds_file);
2228
2229 if (dds_cache) {
2230 // add a copy
2231 BESDEBUG(HDF5_NAME, prolog << "For memory cache, DDS added to the cache for : " << h5_fname << endl);
2232 dds_cache->add(new DDS(*cache_dds), h5_fname);
2233 }
2234
2235}
2236
2237// Add DAS to DDS.
2238void HDF5RequestHandler::add_das_to_dds(DDS *dds, const string &/*container_name*/, const string &filename,
2239 const string &das_cache_fname, hid_t h5_fd, bool das_from_dc) {
2240
2241 BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
2242
2243 // Check DAS memory cache
2244 DAS *das = 0 ;
2245 bool use_das_cache = false;
2246 if (das_cache)
2247 das = static_cast<DAS*>(das_cache->get(filename));
2248 if (das)
2249 use_das_cache = true;
2250
2251 if (true == use_das_cache) {
2252 BESDEBUG(HDF5_NAME, prolog << "DAS Cached hit for : " << filename << endl);
2253 dds->transfer_attributes(das); // no need to copy the cached DAS
2254 }
2255
2256 else {
2257
2258 das = new DAS ;
2259#if 0
2260 if (!container_name.empty())
2261 das->container_name(container_name);
2262#endif
2263 if(das_from_dc == true)
2264 read_das_from_disk_cache(das_cache_fname,das);
2265 else {
2266 // This bool is for the case, when DDS is read from a cache then we need to open the HDF5 file.
2267 bool h5_file_open = true;
2268 if(h5_fd == -1)
2269 h5_file_open = false;
2270 if (true == _usecf) {
2271 // go to the CF option
2272 if(h5_file_open == false)
2273 h5_fd = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
2274
2275 read_cfdas( *das,filename,h5_fd);
2276 if(h5_file_open == false)
2277 H5Fclose(h5_fd);
2278 }
2279 else {
2280 if(h5_file_open == false)
2281 h5_fd = get_fileid(filename.c_str());
2282 find_gloattr(h5_fd, *das);
2283 depth_first(h5_fd, "/", *das);
2284 if(h5_file_open == false)
2285 close_fileid(h5_fd);
2286 }
2287
2288 Ancillary::read_ancillary_das( *das, filename ) ;
2289
2290 if(das_cache_fname!="" && das_from_dc == false)
2291 write_das_to_disk_cache(das_cache_fname,das);
2292 }
2293
2294 dds->transfer_attributes(das);
2295
2296 if (das_cache) {
2297 // add a copy
2298 BESDEBUG(HDF5_NAME, prolog << "For memory cache, DAS added to the cache for : " << filename << endl);
2299 das_cache->add(new DAS(*das), filename);
2300 }
2301 delete das;
2302
2303 }
2304
2305}
2306
2307bool obtain_beskeys_info(const string& key, bool & has_key) {
2308
2309 bool ret_value = false;
2310 string doset ="";
2311 TheBESKeys::TheKeys()->get_value( key, doset, has_key ) ;
2312 if(has_key) {
2313 const string dosettrue ="true";
2314 const string dosetyes = "yes";
2315 doset = BESUtil::lowercase(doset) ;
2316 ret_value = (dosettrue == doset || dosetyes == doset);
2317 }
2318 return ret_value;
2319}
2320
2321#if 0
2322bool is_beskeys_set_true(const string& bes_value) {
2323
2324 const string dosettrue ="true";
2325 const string dosetyes = "yes";
2326 string doset = BESUtil::lowercase(bes_value) ;
2327 return (dosettrue == doset || dosetyes == doset);
2328
2329}
2330bool check_and_set_beskeys(const string key) {
2331
2332 bool found = false;
2333 string doset ="";
2334 const string dosettrue ="true";
2335 const string dosetyes = "yes";
2336
2337 TheBESKeys::TheKeys()->get_value( key, doset, found ) ;
2338 if( true == found ) {
2339 doset = BESUtil::lowercase( doset ) ;
2340 }
2341 BESDEBUG(HDF5_NAME, prolog << "Key: " << key << (found?(" was found. value: "+doset):" was not found.") << endl);
2342 return found && (dosettrue == doset || dosetyes == doset);
2343
2344}
2345#endif
2346
2347// get_uint_key and get_float_key are copied from the netCDF handler.
2348
2349static unsigned int get_uint_key(const string &key, unsigned int def_val)
2350{
2351 bool found = false;
2352 string doset = "";
2353
2354 TheBESKeys::TheKeys()->get_value(key, doset, found);
2355 if (true == found) {
2356 // In C++11, stoi is better.
2357 return atoi(doset.c_str()); // use better code TODO
2358 }
2359 else {
2360 return def_val;
2361 }
2362}
2363
2364static unsigned long get_ulong_key(const string &key, unsigned long def_val)
2365{
2366 bool found = false;
2367 string doset = "";
2368
2369 TheBESKeys::TheKeys()->get_value(key, doset, found);
2370 if (true == found) {
2371 // In C++11, stoull is better.
2372 return atol(doset.c_str()); // use better code TODO
2373 }
2374 else {
2375 return def_val;
2376 }
2377}
2378static float get_float_key(const string &key, float def_val)
2379{
2380 bool found = false;
2381 string doset = "";
2382
2383 TheBESKeys::TheKeys()->get_value(key, doset, found);
2384 if (true == found) {
2385 return atof(doset.c_str()); // use better code TODO
2386 }
2387 else {
2388 return def_val;
2389 }
2390}
2391
2392static string get_beskeys(const string &key) {
2393
2394 bool found = false;
2395 string ret_value ="";
2396
2397 TheBESKeys::TheKeys()->get_value( key, ret_value, found ) ;
2398 return ret_value;
2399
2400}
2401
2402// The function to copy a string to a memory buffer.
2403char* copy_str(char*temp_ptr,const string & str) {
2404
2405 size_t str_size=str.size();
2406 memcpy((void*)temp_ptr,(void*)&str_size,sizeof(size_t));
2407 temp_ptr+=sizeof(size_t);
2408 vector<char>temp_vc2(str.begin(),str.end());
2409 memcpy((void*)temp_ptr,(void*)&temp_vc2[0],str.size());
2410 temp_ptr+=str.size();
2411 return temp_ptr;
2412
2413}
2414
2415
2416// Obtain the string from a memory buffer.
2417// Note: both char* and string(as a reference) will be returned
2418// The attribute binary first stores the size of the string, then the string itself
2419char* obtain_str(char*temp_ptr,string & str) {
2420
2421 size_t oname_size = *((size_t *)temp_ptr);
2422 temp_ptr = temp_ptr + sizeof(size_t);
2423 string oname;
2424 for(unsigned int i =0; i<oname_size; i++){
2425 oname.push_back(*temp_ptr);
2426 ++temp_ptr;
2427 }
2428 str = oname;
2429 return temp_ptr;
2430
2431}
2432
2433// For our case, there are no global attributes for DAS.
2434// The global attribures are always under HDF_GLOBAL.
2435// The main function to obtain the DAS info. from the cache.
2436char* get_attr_info_from_dc(char*temp_pointer,DAS *das,AttrTable *at_par) {
2437
2438 // 3 is only for the code to come into the loop.
2439 uint8_t flag =3;
2440 while(flag !=2) {
2441 flag = *((uint8_t*)(temp_pointer));
2442 BESDEBUG(HDF5_NAME, prolog << "Build DAS from the disk cache file flag: "
2443 <<" flag = 0, attribute; flag = 1, container; flag =2; end of container;"
2444 <<" flag = 3; the initial value to get the attribute retrieval process started."
2445 <<" The flag value is "
2446 << (int)flag <<endl);
2447 temp_pointer++;
2448
2449 if(flag ==1) {
2450 string container_name;
2451 temp_pointer = obtain_str(temp_pointer,container_name);
2452 BESDEBUG(HDF5_NAME, prolog << "DAS from the disk cache, container name is " << container_name << endl);
2453
2454 // Remember the current Attribute table state
2455 AttrTable*temp_at_par = at_par;
2456 if(at_par == NULL)
2457 at_par = das->add_table(container_name, new AttrTable);
2458 else
2459 at_par = at_par->append_container(container_name);
2460
2461 temp_pointer = get_attr_info_from_dc(temp_pointer,das,at_par);
2462 // MUST resume the original state
2463 at_par = temp_at_par;
2464
2465 }
2466 else if(flag == 0) {
2467 // The attribute must have a table.
2468 if(at_par ==NULL)
2469 throw BESInternalError( "The AttrTable must exist for DAS attributes", __FILE__, __LINE__ ) ;
2470
2471 // Attribute name
2472 string attr_name;
2473 temp_pointer = obtain_str(temp_pointer,attr_name);
2474 BESDEBUG(HDF5_NAME, prolog << "DAS from the disk cache, attr name is: " << attr_name << endl);
2475
2476 // Attribute type
2477 string attr_type;
2478 temp_pointer = obtain_str(temp_pointer,attr_type);
2479 BESDEBUG(HDF5_NAME, prolog << "DAS from the disk cache, attr type is: " << attr_type << endl);
2480
2481 // Attribute values
2482 unsigned int num_values = *((unsigned int*)(temp_pointer));
2483 BESDEBUG(HDF5_NAME, prolog << "DAS from the disk cache, number of attribute values is: " << num_values << endl);
2484 temp_pointer+=sizeof(unsigned int);
2485
2486 vector <string> attr_values;
2487
2488 for(unsigned int i = 0; i<num_values; i++) {
2489 string attr_value;
2490 temp_pointer = obtain_str(temp_pointer,attr_value);
2491 attr_values.push_back(attr_value);
2492 BESDEBUG(HDF5_NAME, prolog << "DAS from the disk cache, attribute value is: " << attr_value << endl);
2493 }
2494
2495 at_par->append_attr(attr_name,attr_type,&attr_values);
2496 }
2497
2498 }
2499 return temp_pointer;
2500
2501}
2502
2503// The debugging function to get attribute info.
2504void get_attr_contents(AttrTable*temp_table) {
2505 if(temp_table !=NULL) {
2506 AttrTable::Attr_iter top_startit = temp_table->attr_begin();
2507 AttrTable::Attr_iter top_endit = temp_table->attr_end();
2508 AttrTable::Attr_iter top_it = top_startit;
2509 while(top_it !=top_endit) {
2510 AttrType atype = temp_table->get_attr_type(top_it);
2511 if(atype == Attr_unknown)
2512 cerr<<"unsupported DAS attributes" <<endl;
2513 else if(atype!=Attr_container) {
2514
2515 cerr<<"Attribute name is "<<temp_table->get_name(top_it)<<endl;
2516 cerr<<"Attribute type is "<<temp_table->get_type(top_it)<<endl;
2517 unsigned int num_attrs = temp_table->get_attr_num(temp_table->get_name(top_it));
2518 cerr<<"Attribute values are "<<endl;
2519 for (unsigned int i = 0; i <num_attrs;i++)
2520 cerr<<(*(temp_table->get_attr_vector(temp_table->get_name(top_it))))[i]<<" ";
2521 cerr<<endl;
2522 }
2523 else {
2524 cerr<<"Coming to the attribute container. "<<endl;
2525 cerr<<"container name is "<<(*top_it)->name <<endl;
2526 AttrTable* sub_table = temp_table->get_attr_table(top_it);
2527 cerr<<"container table name is "<<sub_table->get_name() <<endl;
2528 get_attr_contents(sub_table);
2529 }
2530 ++top_it;
2531 }
2532
2533 }
2534}
2535
2536
2537void HDF5RequestHandler::add_attributes(BESDataHandlerInterface &dhi) {
2538
2539
2540 BESResponseObject *response = dhi.response_handler->get_response_object();
2541 BESDataDDSResponse *bdds = dynamic_cast<BESDataDDSResponse *>(response);
2542 if (!bdds)
2543 throw BESInternalError("cast error", __FILE__, __LINE__);
2544 DDS *dds = bdds->get_dds();
2545 string container_name = bdds->get_explicit_containers() ? dhi.container->get_symbolic_name(): "";
2546 string filename = dhi.container->access();
2547 DAS* das = 0;
2548 bool das_from_mcache = false;
2549 if(das_cache) {
2550 das = static_cast<DAS*>(das_cache->get(filename));
2551 if(das) {
2552 BESDEBUG(HDF5_NAME, prolog << "DAS Cached hit for : " << filename << endl);
2553 dds->transfer_attributes(das); // no need to copy the cached DAS
2554 das_from_mcache = true;
2555 }
2556 }
2557
2558 if(false == das_from_mcache) {
2559 das = new DAS;
2560 // This looks at the 'use explicit containers' prop, and if true
2561 // sets the current container for the DAS.
2562 if (!container_name.empty()) das->container_name(container_name);
2563
2564 hid_t h5_fd =-1;
2565 if (true == _usecf) {
2566 // go to the CF option
2567 h5_fd = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
2568
2569 read_cfdas( *das,filename,h5_fd);
2570
2571 H5Fclose(h5_fd);
2572 }
2573 else {
2574 h5_fd = get_fileid(filename.c_str());
2575 find_gloattr(h5_fd, *das);
2576 depth_first(h5_fd, "/", *das);
2577 close_fileid(h5_fd);
2578 }
2579
2580
2581 Ancillary::read_ancillary_das(*das, filename);
2582
2583 dds->transfer_attributes(das);
2584
2585 // Only free the DAS if it's not added to the cache
2586 if (das_cache) {
2587 // add a copy
2588 BESDEBUG(HDF5_NAME, prolog << "DAS added to the cache for : " << filename << endl);
2589 das_cache->add(das, filename);
2590 }
2591 else {
2592 delete das;
2593 }
2594 }
2595 BESDEBUG(HDF5_NAME, prolog << "Data ACCESS in add_attributes(): set the including attribute flag to true: "<<filename << endl);
2596 bdds->set_ia_flag(true);
2597 return;
2598
2599}
2600
2601
include the entry functions to execute the handlers
std::string get_symbolic_name() const
retrieve the symbolic name for this container
Definition: BESContainer.h:221
virtual std::string access()=0
returns the true name of this container
Represents an OPeNDAP DAS DAP2 data object within the BES.
virtual void clear_container()
clear the container in the DAP response object
virtual void set_container(const std::string &cn)
set the container in the DAP response object
Holds a DDS object within the BES.
virtual void set_container(const std::string &cn)
set the container in the DAP response object
void set_dds(libdap::DDS *ddsIn)
virtual void clear_container()
clear the container in the DAP response object
libdap::DDS * get_dds()
Represents an OPeNDAP DMR DAP4 data object within the BES.
error object created from libdap error objects and can handle those errors
Definition: BESDapError.h:59
virtual void set_dap4_function(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_dap4_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
bool get_explicit_containers() const
Should containers be explicitly represented in the DD* responses?
std::string get_request_xml_base() const
Return the xml:base URL for this request.
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
void set_dds(libdap::DDS *ddsIn)
virtual void set_container(const std::string &cn)
set the container in the DAP response object
virtual void clear_container()
clear the container in the DAP response object
Structure storing information used by the BES to handle the request.
BESContainer * container
pointer to current container in this interface
Abstract exception class for the BES with basic string message.
Definition: BESError.h:58
virtual std::string get_message()
get the error message for this exception
Definition: BESError.h:99
informational response object
Definition: BESInfo.h:63
virtual void add_data(const std::string &s)
add data to this informational object. If buffering is not set then the information is output directl...
Definition: BESInfo.cc:160
exception thrown if internal error encountered
exception thrown if an internal error is found and is fatal to the BES
Represents a specific data type request handler.
virtual BESResponseObject * get_response_object()
return the current response object
Abstract base class representing a specific set of information in response to a request to the BES.
virtual void services_handled(const std::string &handler, std::list< std::string > &services)
returns the list of servies provided by the handler in question
virtual bool start(std::string name)
Definition: BESStopWatch.cc:67
static std::string lowercase(const std::string &s)
Definition: BESUtil.cc:206
static std::string implode(const std::list< std::string > &values, char delim)
Definition: BESUtil.cc:657
An in-memory cache for DapObj (DAS, DDS, ...) objects.
Definition: ObjMemCache.h:84
virtual void add(libdap::DapObj *obj, const std::string &key)
Add an object to the cache and associate it with a key.
Definition: ObjMemCache.cc:63
virtual libdap::DapObj * get(const std::string &key)
Get the cached pointer.
Definition: ObjMemCache.cc:105
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:340
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:71
Helper functions for generating DAS attributes and a function to check BES Key.
void depth_first(hid_t pid, const char *gname, DAS &das)
Definition: h5das.cc:62
void find_gloattr(hid_t file, DAS &das)
Definition: h5das.cc:487
bool breadth_first(const hid_t file_id, hid_t pid, char *gname, D4Group *par_grp, const char *fname, bool use_dimscale, vector< link_info_t > &hdf5_hls)
Definition: h5dmr.cc:321
hid_t get_fileid(const char *filename)
Definition: h5get.cc:411
void close_fileid(hid_t fid)
Definition: h5get.cc:433
The main header of the HDF5 OPeNDAP handler.