bes Updated for version 3.20.10
h5dmr.cc
Go to the documentation of this file.
1// This file is part of hdf5_handler a HDF5 file handler for the OPeNDAP
2// data server.
3
4// Copyright (c) 2007-2015 The HDF Group, Inc. and OPeNDAP, Inc.
5//
6// This is free software; you can redistribute it and/or modify it under the
7// terms of the GNU Lesser General Public License as published by the Free
8// Software Foundation; either version 2.1 of the License, or (at your
9// option) any later version.
10//
11// This software is distributed in the hope that it will be useful, but
12// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13// or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14// License for more details.
15//
16// You should have received a copy of the GNU Lesser General Public
17// License along with this library; if not, write to the Free Software
18// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19//
20// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
21// You can contact The HDF Group, Inc. at 1800 South Oak Street,
22// Suite 203, Champaign, IL 61820
23
36// the correct DAP4 DMR layout(group's variables first and then the group).
39
40#include <sstream>
41#include "config_hdf5.h"
42
43#include <libdap/InternalErr.h>
44#include <BESDebug.h>
45
46#include <libdap/mime_util.h>
47
48#include "hdf5_handler.h"
49#include "HDF5Int32.h"
50#include "HDF5UInt32.h"
51#include "HDF5UInt16.h"
52#include "HDF5Int16.h"
53#include "HDF5Byte.h"
54#include "HDF5Array.h"
55#include "HDF5Str.h"
56#include "HDF5Float32.h"
57#include "HDF5Float64.h"
58#include "HDF5Url.h"
59#include "HDF5Structure.h"
60
61// The HDF5CFUtil.h includes the utility function obtain_string_after_lastslash.
62#include "HDF5CFUtil.h"
63#include "h5dmr.h"
64
65using namespace std;
66using namespace libdap;
69
70
72static DS_t dt_inst;
73
75void map_h5_attrs_to_dap4(hid_t oid,D4Group* d4g,BaseType* d4b,Structure * d4s,int flag);
76
77#if 0
83// \param par_grp DAP4 parent group
93
94//bool depth_first(hid_t pid, char *gname, DMR & dmr, D4Group* par_grp, const char *fname)
95bool depth_first(hid_t pid, char *gname, D4Group* par_grp, const char *fname)
96{
97 BESDEBUG("h5",
98 ">depth_first() for dmr "
99 << " pid: " << pid
100 << " gname: " << gname
101 << " fname: " << fname
102 << endl);
103
105 int slinkindex = 0;
106
107 H5G_info_t g_info;
108 hsize_t nelems = 0;
109
111 if(H5Gget_info(pid,&g_info) <0) {
112 string msg =
113 "h5_dmr handler: counting hdf5 group elements error for ";
114 msg += gname;
115 throw InternalErr(__FILE__, __LINE__, msg);
116 }
117
118 nelems = g_info.nlinks;
119
120 ssize_t oname_size = 0;
121
122 // Iterate through the file to see the members of the group from the root.
123 for (hsize_t i = 0; i < nelems; i++) {
124
125 vector <char>oname;
126
127 // Query the length of object name.
128 oname_size =
129 H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,NULL,
130 (size_t)DODS_NAMELEN, H5P_DEFAULT);
131 if (oname_size <= 0) {
132 string msg = "h5_dmr handler: Error getting the size of the hdf5 object from the group: ";
133 msg += gname;
134 throw InternalErr(__FILE__, __LINE__, msg);
135 }
136
137 // Obtain the name of the object
138 oname.resize((size_t) oname_size + 1);
139
140 if (H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,&oname[0],
141 (size_t)(oname_size+1), H5P_DEFAULT) < 0){
142 string msg =
143 "h5_dmr handler: Error getting the hdf5 object name from the group: ";
144 msg += gname;
145 throw InternalErr(__FILE__, __LINE__, msg);
146 }
147
148 // Check if it is the hard link or the soft link
149 H5L_info_t linfo;
150 if (H5Lget_info(pid,&oname[0],&linfo,H5P_DEFAULT)<0) {
151 string msg = "hdf5 link name error from: ";
152 msg += gname;
153 throw InternalErr(__FILE__, __LINE__, msg);
154 }
155
156 // Information of soft links are stored as attributes
157 if(linfo.type == H5L_TYPE_SOFT) {
158 slinkindex++;
159 size_t val_size = linfo.u.val_size;
160 get_softlink(par_grp,pid,&oname[0],slinkindex,val_size);
161 //get_softlink(par_grp,pid,gname,&oname[0],slinkindex,val_size);
162 continue;
163 }
164
165 // Ignore external links
166 if(linfo.type == H5L_TYPE_EXTERNAL)
167 continue;
168
169 // Obtain the object type, such as group or dataset.
170 H5O_info_t oinfo;
171
172 if (H5OGET_INFO_BY_IDX(pid, ".", H5_INDEX_NAME, H5_ITER_NATIVE,
173 i, &oinfo, H5P_DEFAULT)<0) {
174 string msg = "h5_dmr handler: Error obtaining the info for the object";
175 msg += string(oname.begin(),oname.end());
176 throw InternalErr(__FILE__, __LINE__, msg);
177 }
178
179 H5O_type_t obj_type = oinfo.type;
180
181 switch (obj_type) {
182
183 case H5O_TYPE_GROUP:
184 {
185
186 // Obtain the full path name
187 string full_path_name =
188 string(gname) + string(oname.begin(),oname.end()-1) + "/";
189
190 BESDEBUG("h5", "=depth_first dmr ():H5G_GROUP " << full_path_name
191 << endl);
192
193 vector <char>t_fpn;
194 t_fpn.resize(full_path_name.length()+1);
195 copy(full_path_name.begin(),full_path_name.end(),t_fpn.begin());
196 t_fpn[full_path_name.length()] = '\0';
197
198 hid_t cgroup = H5Gopen(pid, &t_fpn[0],H5P_DEFAULT);
199 if (cgroup < 0){
200 throw InternalErr(__FILE__, __LINE__, "h5_dmr handler: H5Gopen() failed.");
201 }
202
203 string grp_name = string(oname.begin(),oname.end()-1);
204
205 // Check the hard link loop and break the loop if it exists.
206 string oid = get_hardlink_dmr(cgroup, full_path_name.c_str());
207 if (oid == "") {
208 try {
209 D4Group* tem_d4_cgroup = new D4Group(grp_name);
210 // Map the HDF5 cgroup attributes to DAP4 group attributes.
211 // Note the last flag of map_h5_attrs_to_dap4 must be 0 for the group attribute mapping.
212 map_h5_attrs_to_dap4(cgroup,tem_d4_cgroup,NULL,NULL,0);
213
214 // Add this new DAP4 group
215 par_grp->add_group_nocopy(tem_d4_cgroup);
216
217 // Continue searching the objects under this group
218 //depth_first(cgroup, &t_fpn[0], dmr, tem_d4_cgroup,fname);
219 depth_first(cgroup, &t_fpn[0], tem_d4_cgroup,fname);
220 }
221 catch(...) {
222 H5Gclose(cgroup);
223 throw;
224 }
225 }
226 else {
227 // This group has been visited.
228 // Add the attribute table with the attribute name as HDF5_HARDLINK.
229 // The attribute value is the name of the group when it is first visited.
230 D4Group* tem_d4_cgroup = new D4Group(string(grp_name));
231
232 // Note attr_str_c is the DAP4 attribute string datatype
233 D4Attribute *d4_hlinfo = new D4Attribute("HDF5_HARDLINK",attr_str_c);
234
235 d4_hlinfo->add_value(obj_paths.get_name(oid));
236 tem_d4_cgroup->attributes()->add_attribute_nocopy(d4_hlinfo);
237 par_grp->add_group_nocopy(tem_d4_cgroup);
238
239 }
240
241 if (H5Gclose(cgroup) < 0){
242 throw InternalErr(__FILE__, __LINE__, "Could not close the group.");
243 }
244 break;
245 }
246
247 case H5O_TYPE_DATASET:
248 {
249
250 // Obtain the absolute path of the HDF5 dataset
251 string full_path_name = string(gname) + string(oname.begin(),oname.end()-1);
252
253 // TOOOODOOOO
254 // Obtain the hdf5 dataset handle stored in the structure dt_inst.
255 // All the metadata information in the handler is stored in dt_inst.
256 // Work on this later, redundant for dmr since dataset is opened twice. KY 2015-07-01
257 // Note: depth_first is for building DMR of an HDF5 file that doesn't use dim. scale.
258 // so passing the last parameter as false.
259 get_dataset(pid, full_path_name, &dt_inst,false);
260
261 // Here we open the HDF5 dataset again to use the dataset id for dataset attributes.
262 // This is not necessary for DAP2 since DAS and DDS are separated.
263 hid_t dset_id = -1;
264 if((dset_id = H5Dopen(pid,full_path_name.c_str(),H5P_DEFAULT)) <0) {
265 string msg = "cannot open the HDF5 dataset ";
266 msg += full_path_name;
267 throw InternalErr(__FILE__, __LINE__, msg);
268 }
269
270 try {
271 read_objects(par_grp, full_path_name, fname,dset_id);
272 }
273 catch(...) {
274 H5Dclose(dset_id);
275 throw;
276 }
277 if(H5Dclose(dset_id)<0) {
278 string msg = "cannot close the HDF5 dataset ";
279 msg += full_path_name;
280 throw InternalErr(__FILE__, __LINE__, msg);
281 }
282 }
283 break;
284
285 case H5O_TYPE_NAMED_DATATYPE:
286 // ignore the named datatype
287 break;
288 default:
289 break;
290 }// switch(obj_type)
291 } // for i is 0 ... nelems
292
293 BESDEBUG("h5", "<depth_first() for dmr" << endl);
294 return true;
295}
296#endif
303// \param par_grp DAP4 parent group
316
317
318// The reason to use breadth_first is that the DMR representation needs to show the dimension names and the variables under the group first and then the group names.
319// So we use this search. In the future, we may just use the breadth_first search for all cases.??
320//bool breadth_first(hid_t pid, char *gname, DMR & dmr, D4Group* par_grp, const char *fname,bool use_dimscale)
321bool breadth_first(const hid_t file_id, hid_t pid, char *gname, D4Group* par_grp, const char *fname,bool use_dimscale,vector<link_info_t> & hdf5_hls )
322{
323 BESDEBUG("h5",
324 ">breadth_first() for dmr "
325 << " pid: " << pid
326 << " gname: " << gname
327 << " fname: " << fname
328 << endl);
329
331 int slinkindex = 0;
332
333 // Obtain the number of objects in this group
334 H5G_info_t g_info;
335 hsize_t nelems = 0;
336 if(H5Gget_info(pid,&g_info) <0) {
337 string msg =
338 "h5_dmr handler: counting hdf5 group elements error for ";
339 msg += gname;
340 throw InternalErr(__FILE__, __LINE__, msg);
341 }
342
343 nelems = g_info.nlinks;
344
345 ssize_t oname_size;
346
347 // First iterate through the HDF5 datasets under the group.
348 for (hsize_t i = 0; i < nelems; i++) {
349
350 vector <char>oname;
351
352 // Query the length of object name.
353 oname_size =
354 H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,NULL,
355 (size_t)DODS_NAMELEN, H5P_DEFAULT);
356 if (oname_size <= 0) {
357 string msg = "h5_dmr handler: Error getting the size of the hdf5 object from the group: ";
358 msg += gname;
359 throw InternalErr(__FILE__, __LINE__, msg);
360 }
361
362 // Obtain the name of the object
363 oname.resize((size_t) oname_size + 1);
364
365 if (H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,&oname[0],
366 (size_t)(oname_size+1), H5P_DEFAULT) < 0){
367 string msg =
368 "h5_dmr handler: Error getting the hdf5 object name from the group: ";
369 msg += gname;
370 throw InternalErr(__FILE__, __LINE__, msg);
371 }
372
373 // Check if it is the hard link or the soft link
374 H5L_info_t linfo;
375 if (H5Lget_info(pid,&oname[0],&linfo,H5P_DEFAULT)<0) {
376 string msg = "hdf5 link name error from: ";
377 msg += gname;
378 throw InternalErr(__FILE__, __LINE__, msg);
379 }
380
381 // Information of soft links are stored as attributes
382 if(linfo.type == H5L_TYPE_SOFT) {
383 slinkindex++;
384
385 // Size of a soft link value
386 size_t val_size = linfo.u.val_size;
387 get_softlink(par_grp,pid,&oname[0],slinkindex,val_size);
388 continue;
389 }
390
391 // Ignore external links
392 if(linfo.type == H5L_TYPE_EXTERNAL)
393 continue;
394
395 // Obtain the object type, such as group or dataset.
396 H5O_info_t oinfo;
397
398 if (H5OGET_INFO_BY_IDX(pid, ".", H5_INDEX_NAME, H5_ITER_NATIVE,
399 i, &oinfo, H5P_DEFAULT)<0) {
400 string msg = "h5_dmr handler: Error obtaining the info for the object";
401 msg += string(oname.begin(),oname.end());
402 throw InternalErr(__FILE__, __LINE__, msg);
403 }
404
405 H5O_type_t obj_type = oinfo.type;
406
407 if(H5O_TYPE_DATASET == obj_type) {
408
409 // Obtain the absolute path of the HDF5 dataset
410 string full_path_name = string(gname) + string(oname.begin(),oname.end()-1);
411
412 // TOOOODOOOO
413 // Obtain the hdf5 dataset handle stored in the structure dt_inst.
414 // All the metadata information in the handler is stored in dt_inst.
415 // Work on this later, redundant for dmr since dataset is opened twice. KY 2015-07-01
416 // Dimension scale is handled in this routine. So need to keep it. KY 2020-06-10
417 bool is_pure_dim = false;
418 get_dataset_dmr(file_id, pid, full_path_name, &dt_inst,use_dimscale,is_pure_dim,hdf5_hls);
419
420 if(false == is_pure_dim) {
421 hid_t dset_id = -1;
422 if((dset_id = H5Dopen(pid,full_path_name.c_str(),H5P_DEFAULT)) <0) {
423 string msg = "cannot open the HDF5 dataset ";
424 msg += full_path_name;
425 throw InternalErr(__FILE__, __LINE__, msg);
426 }
427
428 try {
429 read_objects(par_grp, full_path_name, fname,dset_id);
430 }
431 catch(...) {
432 H5Dclose(dset_id);
433 throw;
434 }
435 if(H5Dclose(dset_id)<0) {
436 string msg = "cannot close the HDF5 dataset ";
437 msg += full_path_name;
438 throw InternalErr(__FILE__, __LINE__, msg);
439 }
440 }
441 else {
442 //Need to add this pure dimension to the corresponding DAP4 group
443 D4Dimensions *d4_dims = par_grp->dims();
444 string d4dim_name = string(oname.begin(),oname.end()-1);
445 D4Dimension *d4_dim = d4_dims->find_dim(d4dim_name);
446 if(d4_dim == NULL) {
447 d4_dim = new D4Dimension(d4dim_name,dt_inst.nelmts);
448 d4_dims->add_dim_nocopy(d4_dim);
449 }
450 BESDEBUG("h5", "<h5dmr.cc: pure dimension: dataset name." << d4dim_name << endl);
451 if(H5Tclose(dt_inst.type)<0) {
452 throw InternalErr(__FILE__, __LINE__, "Cannot close the HDF5 datatype.");
453 }
454 }
455
456 }
457 }
458
459 // The attributes of this group. Doing this order to follow ncdump's way (variable,attribute then groups)
460 map_h5_attrs_to_dap4(pid,par_grp,NULL,NULL,0);
461
462 // Then HDF5 child groups
463 for (hsize_t i = 0; i < nelems; i++) {
464
465 vector <char>oname;
466
467 // Query the length of object name.
468 oname_size =
469 H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,NULL,
470 (size_t)DODS_NAMELEN, H5P_DEFAULT);
471 if (oname_size <= 0) {
472 string msg = "h5_dmr handler: Error getting the size of the hdf5 object from the group: ";
473 msg += gname;
474 throw InternalErr(__FILE__, __LINE__, msg);
475 }
476
477 // Obtain the name of the object
478 oname.resize((size_t) oname_size + 1);
479
480 if (H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,&oname[0],
481 (size_t)(oname_size+1), H5P_DEFAULT) < 0){
482 string msg =
483 "h5_dmr handler: Error getting the hdf5 object name from the group: ";
484 msg += gname;
485 throw InternalErr(__FILE__, __LINE__, msg);
486 }
487
488 // Check if it is the hard link or the soft link
489 H5L_info_t linfo;
490 if (H5Lget_info(pid,&oname[0],&linfo,H5P_DEFAULT)<0) {
491 string msg = "hdf5 link name error from: ";
492 msg += gname;
493 throw InternalErr(__FILE__, __LINE__, msg);
494 }
495
496 // Information of soft links are handled already, the softlinks need to be ignored, otherwise
497 // the group it links will be mapped again in the block of if obj_type is H5O_TYPE_GROUP
498 if(linfo.type == H5L_TYPE_SOFT) {
499 continue;
500 }
501
502 // Ignore external links
503 if(linfo.type == H5L_TYPE_EXTERNAL)
504 continue;
505
506 // Obtain the object type, such as group or dataset.
507 H5O_info_t oinfo;
508
509 if (H5OGET_INFO_BY_IDX(pid, ".", H5_INDEX_NAME, H5_ITER_NATIVE,
510 i, &oinfo, H5P_DEFAULT)<0) {
511 string msg = "h5_dmr handler: Error obtaining the info for the object in the breadth_first.";
512 throw InternalErr(__FILE__, __LINE__, msg);
513 }
514
515 H5O_type_t obj_type = oinfo.type;
516
517
518 if(obj_type == H5O_TYPE_GROUP) {
519
520 // Obtain the full path name
521 string full_path_name =
522 string(gname) + string(oname.begin(),oname.end()-1) + "/";
523
524 BESDEBUG("h5", "=breadth_first dmr ():H5G_GROUP " << full_path_name
525 << endl);
526
527 vector <char>t_fpn;
528 t_fpn.resize(full_path_name.length()+1);
529 copy(full_path_name.begin(),full_path_name.end(),t_fpn.begin());
530 t_fpn[full_path_name.length()] = '\0';
531
532 hid_t cgroup = H5Gopen(pid, &t_fpn[0],H5P_DEFAULT);
533 if (cgroup < 0){
534 throw InternalErr(__FILE__, __LINE__, "h5_dmr handler: H5Gopen() failed.");
535 }
536
537 string grp_name = string(oname.begin(),oname.end()-1);
538
539 // Check the hard link loop and break the loop if it exists.
540 string oid = get_hardlink_dmr(cgroup, full_path_name.c_str());
541 if (oid == "") {
542 try {
543 D4Group* tem_d4_cgroup = new D4Group(grp_name);
544
545 // Add this new DAP4 group
546 par_grp->add_group_nocopy(tem_d4_cgroup);
547
548 // Continue searching the objects under this group
549 breadth_first(file_id,cgroup, &t_fpn[0], tem_d4_cgroup,fname,use_dimscale,hdf5_hls);
550 }
551 catch(...) {
552 H5Gclose(cgroup);
553 throw;
554 }
555 }
556 else {
557 // This group has been visited.
558 // Add the attribute table with the attribute name as HDF5_HARDLINK.
559 // The attribute value is the name of the group when it is first visited.
560 D4Group* tem_d4_cgroup = new D4Group(string(grp_name));
561
562 // Note attr_str_c is the DAP4 attribute string datatype
563 D4Attribute *d4_hlinfo = new D4Attribute("HDF5_HARDLINK",attr_str_c);
564
565 d4_hlinfo->add_value(obj_paths.get_name(oid));
566 tem_d4_cgroup->attributes()->add_attribute_nocopy(d4_hlinfo);
567 par_grp->add_group_nocopy(tem_d4_cgroup);
568 }
569
570 if (H5Gclose(cgroup) < 0){
571 throw InternalErr(__FILE__, __LINE__, "Could not close the group.");
572 }
573 }// end if
574 } // for i is 0 ... nelems
575
576 BESDEBUG("h5", "<breadth_first() " << endl);
577 return true;
578}
579
595//
596void
597read_objects( D4Group * d4_grp, const string &varname, const string &filename, const hid_t dset_id)
598{
599
600 switch (H5Tget_class(dt_inst.type)) {
601
602 // HDF5 compound maps to DAP structure.
603 case H5T_COMPOUND:
604 read_objects_structure(d4_grp, varname, filename,dset_id);
605 break;
606
607 case H5T_ARRAY:
608 H5Tclose(dt_inst.type);
609 throw InternalErr(__FILE__, __LINE__, "Currently don't support accessing data of Array datatype when array datatype is not inside the compound.");
610
611 default:
612 read_objects_base_type(d4_grp,varname, filename,dset_id);
613 break;
614 }
615 // We must close the datatype obtained in the get_dataset routine since this is the end of reading DDS.
616 if(H5Tclose(dt_inst.type)<0) {
617 throw InternalErr(__FILE__, __LINE__, "Cannot close the HDF5 datatype.");
618 }
619}
620
635//
636
637//void
638//read_objects_base_type(DMR & dmr, D4Group * d4_grp,const string & varname,
639void
640read_objects_base_type(D4Group * d4_grp,const string & varname,
641 const string & filename,hid_t dset_id)
642{
643
644 // Obtain the relative path of the variable name under the leaf group
645 string newvarname = HDF5CFUtil::obtain_string_after_lastslash(varname);
646
647 // Get a base type. It should be an HDF5 atomic datatype
648 // datatype.
649 BaseType *bt = Get_bt(newvarname, varname,filename, dt_inst.type,true);
650 if (!bt) {
651 throw
652 InternalErr(__FILE__, __LINE__,
653 "Unable to convert hdf5 datatype to dods basetype");
654 }
655
656 // First deal with scalar data.
657 if (dt_inst.ndims == 0) {
658 // transform the DAP2 to DAP4 for this DAP base type and add it to d4_grp
659 bt->transform_to_dap4(d4_grp,d4_grp);
660 // Get it back - this may return null because the underlying type
661 // may have no DAP2 manifestation.
662 BaseType* new_var = d4_grp->var(bt->name());
663 if(new_var){
664 // Map the HDF5 dataset attributes to DAP4
665 map_h5_attrs_to_dap4(dset_id,NULL,new_var,NULL,1);
666 // If this variable is a hardlink, stores the HARDLINK info. as an attribute.
667 map_h5_dset_hardlink_to_d4(dset_id,varname,new_var,NULL,1);
668 }
669 delete bt;
670 bt = 0;
671 }
672 else {
673 // Next, deal with Array data. This 'else clause' runs to
674 // the end of the method.
675 HDF5Array *ar = new HDF5Array(newvarname, filename, bt);
676 delete bt; bt = 0;
677
678 // set number of elements and variable name values.
679 // This essentially stores in the struct.
680 ar->set_memneed(dt_inst.need);
681 ar->set_numdim(dt_inst.ndims);
682 ar->set_numelm((int) (dt_inst.nelmts));
683 ar->set_varpath(varname);
684
685
686 // If we have dimension names(dimension scale is used.),we will see if we can add the names.
687 int dimnames_size = 0;
688 if((unsigned int)((int)(dt_inst.dimnames.size())) != dt_inst.dimnames.size())
689 {
690 delete ar;
691 throw
692 InternalErr(__FILE__, __LINE__,
693 "number of dimensions: overflow");
694 }
695 dimnames_size = (int)(dt_inst.dimnames.size());
696//cerr<<"dimnames_size is "<<dimnames_size <<endl;
697//cerr<<"ndims is "<<dt_inst.ndims <<endl;
698
699 if(dimnames_size ==dt_inst.ndims) {
700
701 for (int dim_index = 0; dim_index < dt_inst.ndims; dim_index++) {
702 if(dt_inst.dimnames[dim_index] !="")
703 ar->append_dim(dt_inst.size[dim_index],dt_inst.dimnames[dim_index]);
704 else
705 ar->append_dim(dt_inst.size[dim_index]);
706 // D4dimension has to have a name. If no name, no D4dimension(from comments libdap4: Array.cc)
707 }
708 dt_inst.dimnames.clear();
709 }
710 else {
711 // For DAP4, no need to add dimension if no dimension name
712 for (int dim_index = 0; dim_index < dt_inst.ndims; dim_index++)
713 ar->append_dim(dt_inst.size[dim_index]);
714 }
715
716 // We need to transform dimension info. to DAP4 group
717 BaseType* new_var = NULL;
718 try {
719 new_var = ar->h5dims_transform_to_dap4(d4_grp,dt_inst.dimnames_path);
720 }
721 catch(...) {
722 delete ar;
723 throw;
724 }
725
726 // clear DAP4 dimnames_path vector
727 dt_inst.dimnames_path.clear();
728
729 // Map HDF5 dataset attributes to DAP4
730 map_h5_attrs_to_dap4(dset_id,NULL,new_var,NULL,1);
731
732 // If this is a hardlink, map the Hardlink info. as an DAP4 attribute.
733 map_h5_dset_hardlink_to_d4(dset_id,varname,new_var,NULL,1);
734#if 0
735 // Test the attribute
736 D4Attribute *test_attr = new D4Attribute("DAP4_test",attr_str_c);
737 test_attr->add_value("test_grp_attr");
738 new_var->attributes()->add_attribute_nocopy(test_attr);
739#endif
740 // Add this var to DAP4 group.
741 d4_grp->add_var_nocopy(new_var);
742 delete ar; ar = 0;
743 }
744 BESDEBUG("h5", "<read_objects_base_type(dmr)" << endl);
745
746}
747
761void
762read_objects_structure(D4Group *d4_grp, const string & varname,
763 const string & filename,hid_t dset_id)
764{
765 // Obtain the relative path of the variable name under the leaf group
766 string newvarname = HDF5CFUtil::obtain_string_after_lastslash(varname);
767
768 // Map HDF5 compound datatype to Structure
769 Structure *structure = Get_structure(newvarname, varname,filename, dt_inst.type,true);
770
771 try {
772 BESDEBUG("h5", "=read_objects_structure(): Dimension is "
773 << dt_inst.ndims << endl);
774
775 if (dt_inst.ndims != 0) { // Array of Structure
776 BESDEBUG("h5", "=read_objects_structure(): array of size " <<
777 dt_inst.nelmts << endl);
778 BESDEBUG("h5", "=read_objects_structure(): memory needed = " <<
779 dt_inst.need << endl);
780
781 // Create the Array of structure.
782 HDF5Array *ar = new HDF5Array(newvarname, filename, structure);
783 delete structure; structure = 0;
784
785
786 // These parameters are used in the data read function.
787 ar->set_memneed(dt_inst.need);
788 ar->set_numdim(dt_inst.ndims);
789 ar->set_numelm((int) (dt_inst.nelmts));
790 ar->set_length((int) (dt_inst.nelmts));
791 ar->set_varpath(varname);
792
793 // If having dimension names, add the dimension names to DAP.
794 int dimnames_size = 0;
795 if((unsigned int)((int)(dt_inst.dimnames.size())) != dt_inst.dimnames.size())
796 {
797 delete ar;
798 throw
799 InternalErr(__FILE__, __LINE__,
800 "number of dimensions: overflow");
801 }
802 dimnames_size = (int)(dt_inst.dimnames.size());
803
804
805 if(dimnames_size ==dt_inst.ndims) {
806 for (int dim_index = 0; dim_index < dt_inst.ndims; dim_index++) {
807 if(dt_inst.dimnames[dim_index] !="")
808 ar->append_dim(dt_inst.size[dim_index],dt_inst.dimnames[dim_index]);
809 else
810 ar->append_dim(dt_inst.size[dim_index]);
811 }
812 dt_inst.dimnames.clear();
813 }
814 else {
815 for (int dim_index = 0; dim_index < dt_inst.ndims; dim_index++)
816 ar->append_dim(dt_inst.size[dim_index]);
817
818 }
819
820 // We need to transform dimension info. to DAP4 group
821 BaseType* new_var = ar->h5dims_transform_to_dap4(d4_grp,dt_inst.dimnames_path);
822 dt_inst.dimnames_path.clear();
823
824 // Map HDF5 dataset attributes to DAP4
825 map_h5_attrs_to_dap4(dset_id,NULL,new_var,NULL,1);
826
827 // If this is a hardlink, map the Hardlink info. as an DAP4 attribute.
828 map_h5_dset_hardlink_to_d4(dset_id,varname,new_var,NULL,1);
829
830 // Add this var to DAP4 group
831 if(new_var)
832 d4_grp->add_var_nocopy(new_var);
833 delete ar; ar = 0;
834 }// end if
835 else {// A scalar structure
836
837 structure->set_is_dap4(true);
838 map_h5_attrs_to_dap4(dset_id,NULL,NULL,structure,2);
839 map_h5_dset_hardlink_to_d4(dset_id,varname,NULL,structure,2);
840 if(structure)
841 d4_grp->add_var_nocopy(structure);
842 }
843 } // try Structure
844 catch (...) {
845 delete structure;
846 throw;
847 }
848}
849
850
864//
865
866void map_h5_attrs_to_dap4(hid_t h5_objid,D4Group* d4g,BaseType* d4b,Structure * d4s,int flag) {
867
868 // Get the object info
869 H5O_info_t obj_info;
870 if (H5OGET_INFO(h5_objid, &obj_info) <0) {
871 string msg = "Fail to obtain the HDF5 object info. .";
872 throw InternalErr(__FILE__, __LINE__, msg);
873 }
874
875 // Obtain the number of attributes
876 int num_attr = obj_info.num_attrs;
877 if (num_attr < 0 ) {
878 string msg = "Fail to get the number of attributes for the HDF5 object. ";
879 throw InternalErr(__FILE__, __LINE__,msg);
880 }
881
882 string print_rep;
883 vector<char>temp_buf;
884
885 bool ignore_attr = false;
886 hid_t attr_id = -1;
887 for (int j = 0; j < num_attr; j++) {
888
889 // Obtain attribute information.
890 DSattr_t attr_inst;
891
892 // Ignore the attributes of which the HDF5 datatype
893 // cannot be mapped to DAP4. The ignored attribute datatypes can be found
894 // at function get_attr_info in h5get.cc.
895 attr_id = get_attr_info(h5_objid, j, true,&attr_inst, &ignore_attr);
896 if (true == ignore_attr) {
897 H5Aclose(attr_id);
898 continue;
899 }
900
901 // Get the corresponding DAP data type of the HDF5 datatype.
902 // The following line doesn't work in HDF5 1.10.
903#if 0
904 //hid_t ty_id = attr_inst.type;
905#endif
906 hid_t ty_id = H5Aget_type(attr_id);
907 if(ty_id <0) {
908 H5Aclose(attr_id);
909 throw InternalErr(__FILE__, __LINE__, "Cannot retrieve HDF5 attribute datatype successfully.");
910 }
911
912 string dap_type = get_dap_type(ty_id,true);
913
914 // Need to have DAP4 representation of the attribute type
915 D4AttributeType dap4_attr_type = daptype_strrep_to_dap4_attrtype(dap_type);
916
917 // We encounter an unsupported DAP4 attribute type.
918 if(attr_null_c == dap4_attr_type) {
919 H5Tclose(ty_id);
920 H5Aclose(attr_id);
921 throw InternalErr(__FILE__, __LINE__, "unsupported DAP4 attribute type");
922 }
923
924 string attr_name = attr_inst.name;
925 BESDEBUG("h5", "arttr_name= " << attr_name << endl);
926
927 // Create the DAP4 attribute mapped from HDF5
928 D4Attribute *d4_attr = new D4Attribute(attr_name,dap4_attr_type);
929
930 // We have to handle variable length string differently.
931 if (H5Tis_variable_str(ty_id)) {
932 write_vlen_str_attrs(attr_id,ty_id,&attr_inst,d4_attr,NULL,true);
933 }// if (H5Tis_variable_str(ty_id)
934 else {
935
936 vector<char> value;
937 value.resize(attr_inst.need + sizeof(char));
938 //value.resize(attr_inst.need);
939 BESDEBUG("h5", "arttr_inst.need=" << attr_inst.need << endl);
940
941 // Need to obtain the memtype since we still find BE data.
942 hid_t memtype = H5Tget_native_type(ty_id, H5T_DIR_ASCEND);
943 // Read HDF5 attribute data.
944 if (H5Aread(attr_id, memtype, (void *) (&value[0])) < 0) {
945 delete d4_attr;
946 throw InternalErr(__FILE__, __LINE__, "unable to read HDF5 attribute data");
947 }
948 H5Aclose(memtype);
949
950 // For scalar data, just read data once.
951 if (attr_inst.ndims == 0) {
952 for (int loc = 0; loc < (int) attr_inst.nelmts; loc++) {
953 print_rep = print_attr(ty_id, loc, &value[0]);
954 if (print_rep.c_str() != NULL) {
955 d4_attr->add_value(print_rep);
956 }
957 }
958
959 }
960 else {// The number of dimensions is > 0
961
962 // Get the attribute datatype size
963 int elesize = (int) H5Tget_size(ty_id);
964 if (elesize == 0) {
965 H5Tclose(ty_id);
966 H5Aclose(attr_id);
967 delete d4_attr;
968 throw InternalErr(__FILE__, __LINE__, "unable to get attibute size");
969 }
970
971 // Due to the implementation of print_attr, the attribute value will be
972 // written one by one.
973 char *tempvalue = &value[0];
974
975 // Write this value. the "loc" can always be set to 0 since
976 // tempvalue will be moved to the next value.
977 for( hsize_t temp_index = 0; temp_index < attr_inst.nelmts; temp_index ++) {
978 //print_rep = print_attr(ty_id, 0, (void*)&value[0]);
979 print_rep = print_attr(ty_id, 0, tempvalue);
980 if (print_rep.c_str() != NULL) {
981
982 BESDEBUG("h5", "print_rep= " << print_rep << endl);
983
984 d4_attr->add_value(print_rep);
985 tempvalue = tempvalue + elesize;
986 BESDEBUG("h5",
987 "tempvalue= " << tempvalue
988 << "elesize=" << elesize
989 << endl);
990
991 }
992 else {
993 H5Tclose(ty_id);
994 H5Aclose(attr_id);
995 delete d4_attr;
996 throw InternalErr(__FILE__, __LINE__, "unable to convert attibute value to DAP");
997 }
998 }//for(hsize_t temp_index=0; .....
999 } // if attr_inst.ndims != 0
1000 }
1001 if(H5Tclose(ty_id) < 0) {
1002 H5Aclose(attr_id);
1003 delete d4_attr;
1004 throw InternalErr(__FILE__, __LINE__, "unable to close HDF5 type id");
1005 }
1006 if (H5Aclose(attr_id) < 0) {
1007 delete d4_attr;
1008 throw InternalErr(__FILE__, __LINE__, "unable to close attibute id");
1009 }
1010
1011 if(0 == flag) // D4group
1012 d4g->attributes()->add_attribute_nocopy(d4_attr);
1013 else if (1 == flag) // HDF5 dataset with atomic datatypes
1014 d4b->attributes()->add_attribute_nocopy(d4_attr);
1015 else if ( 2 == flag) // HDF5 dataset with compound datatype
1016 d4s->attributes()->add_attribute_nocopy(d4_attr);
1017 else {
1018 stringstream sflag;
1019 sflag << flag;
1020 string msg ="The add_dap4_attr flag has to be either 0,1 or 2.";
1021 msg+="The current flag is "+sflag.str();
1022 delete d4_attr;
1023 throw InternalErr(__FILE__, __LINE__, msg);
1024 }
1025 } // for (int j = 0; j < num_attr; j++)
1026
1027 return;
1028}
1029
1043
1044
1045void map_h5_dset_hardlink_to_d4(hid_t h5_dsetid,const string & full_path, BaseType* d4b,Structure * d4s,int flag) {
1046
1047 // Obtain the unique object number info. If no hardlinks, empty string will return.
1048 string oid = get_hardlink_dmr(h5_dsetid, full_path);
1049
1050 // Find that this is a hardlink,add the hardlink info to a DAP4 attribute.
1051 if(false == oid.empty()) {
1052
1053 D4Attribute *d4_hlinfo = new D4Attribute("HDF5_HARDLINK",attr_str_c);
1054 d4_hlinfo->add_value(obj_paths.get_name(oid));
1055
1056 if (1 == flag)
1057 d4b->attributes()->add_attribute_nocopy(d4_hlinfo);
1058 else if ( 2 == flag)
1059 d4s->attributes()->add_attribute_nocopy(d4_hlinfo);
1060 else
1061 delete d4_hlinfo;
1062 }
1063
1064}
1065
1078void get_softlink(D4Group* par_grp, hid_t h5obj_id, const string & oname, int index, size_t val_size)
1079{
1080 BESDEBUG("h5", "dap4 >get_softlink():" << oname << endl);
1081
1082 ostringstream oss;
1083 oss << string("HDF5_SOFTLINK");
1084 oss << "_";
1085 oss << index;
1086 string temp_varname = oss.str();
1087
1088
1089 BESDEBUG("h5", "dap4->get_softlink():" << temp_varname << endl);
1090 D4Attribute *d4_slinfo = new D4Attribute;
1091 d4_slinfo->set_name(temp_varname);
1092
1093 // Make the type as a container
1094 d4_slinfo->set_type(attr_container_c);
1095
1096 string softlink_name = "linkname";
1097
1098 D4Attribute *softlink_src = new D4Attribute(softlink_name,attr_str_c);
1099 softlink_src->add_value(oname);
1100
1101 d4_slinfo->attributes()->add_attribute_nocopy(softlink_src);
1102 string softlink_value_name ="LINKTARGET";
1103
1104 // Get the link target information. We always return the link value in a string format.
1105 D4Attribute *softlink_tgt = 0;
1106
1107 try {
1108 vector<char> buf;
1109 buf.resize(val_size + 1);
1110
1111 // get link target name
1112 if (H5Lget_val(h5obj_id, oname.c_str(), (void*) &buf[0], val_size + 1, H5P_DEFAULT) < 0) {
1113 throw InternalErr(__FILE__, __LINE__, "unable to get link value");
1114 }
1115 softlink_tgt = new D4Attribute(softlink_value_name, attr_str_c);
1116 string link_target_name = string(buf.begin(), buf.end());
1117 softlink_tgt->add_value(link_target_name);
1118
1119 d4_slinfo->attributes()->add_attribute_nocopy(softlink_tgt);
1120 }
1121 catch (...) {
1122 delete softlink_tgt;
1123 throw;
1124 }
1125
1126 par_grp->attributes()->add_attribute_nocopy(d4_slinfo);
1127}
1128
1129
1142string get_hardlink_dmr( hid_t h5obj_id, const string & oname) {
1143
1144 BESDEBUG("h5", "dap4->get_hardlink_dmr():" << oname << endl);
1145
1146 // Get the object info
1147 H5O_info_t obj_info;
1148 if (H5OGET_INFO(h5obj_id, &obj_info) <0) {
1149 throw InternalErr(__FILE__, __LINE__, "H5OGET_INFO() failed.");
1150 }
1151
1152 // If the reference count is greater than 1,that means
1153 // hard links are found. return the original object name this
1154 // hard link points to.
1155
1156 if (obj_info.rc >1) {
1157
1158 string objno;
1159
1160#if (H5_VERS_MAJOR == 1 && ((H5_VERS_MINOR == 12) || (H5_VERS_MINOR == 13)))
1161 char *obj_tok_str = NULL;
1162 if(H5Otoken_to_str(h5obj_id, &(obj_info.token), &obj_tok_str) <0) {
1163 throw InternalErr(__FILE__, __LINE__, "H5Otoken_to_str failed.");
1164 }
1165 objno.assign(obj_tok_str,obj_tok_str+strlen(obj_tok_str));
1166 H5free_memory(obj_tok_str);
1167
1168#else
1169 ostringstream oss;
1170 oss << hex << obj_info.addr;
1171 objno = oss.str();
1172#endif
1173
1174 BESDEBUG("h5", "dap4->get_hardlink_dmr() objno=" << objno << endl);
1175
1176 // Add this hard link to the map.
1177 // obj_paths is a global variable defined at the beginning of this file.
1178 // it is essentially a id to obj name map. See HDF5PathFinder.h.
1179 if (!obj_paths.add(objno, oname)) {
1180 return objno;
1181 }
1182 else {
1183 return "";
1184 }
1185 }
1186 else {
1187 return "";
1188 }
1189
1190}
A class for handling all types of array in HDF5 for the default option.
This class provides a way to map HDF5 byte to DAP Byte for the default option.
This file includes several helper functions for translating HDF5 to CF-compliant.
A class for mapping HDF5 32-bit float to DAP for the default option.
A class for mapping HDF5 64-bit float to DAP for the default option.
A class for HDF5 signed 16 bit integer type.
This class provides a way to map HDF5 32 bit integer to DAP Int32 for the default option.
This class that translates HDF5 string into DAP string for the default option.
This class converts HDF5 compound type into DAP structure for the default option.
This class provides a way to map unsigned HDF5 16 bit integer to DAP UInt16 for the default option.
This class provides a way to map unsigned HDF5 32 bit integer to DAP UInt32.
This class generates DAP URL type for the default option.
void set_numdim(int ndims)
remembers number of dimensions of this array.
Definition: HDF5Array.cc:1737
void set_numelm(int nelms)
remembers number of elements in this array.
Definition: HDF5Array.cc:1741
void set_memneed(size_t need)
remembers memory size needed.
Definition: HDF5Array.cc:1733
std::string get_name(std::string id)
bool add(std::string id, const std::string name)
void depth_first(hid_t pid, const char *gname, DAS &das)
Definition: h5das.cc:62
void read_objects(DAS &das, const string &varname, hid_t oid, int num_attr)
Definition: h5das.cc:295
void read_objects_base_type(DDS &dds_table, const string &varname, const string &filename)
Definition: h5dds.cc:257
void read_objects_structure(DDS &dds_table, const string &varname, const string &filename)
Definition: h5dds.cc:311
void map_h5_attrs_to_dap4(hid_t oid, D4Group *d4g, BaseType *d4b, Structure *d4s, int flag)
A function that map HDF5 attributes to DAP4.
Definition: h5dmr.cc:866
HDF5PathFinder obj_paths
A variable for remembering visited paths to break cyclic HDF5 groups.
Definition: h5dmr.cc:68
bool breadth_first(const hid_t file_id, hid_t pid, char *gname, D4Group *par_grp, const char *fname, bool use_dimscale, vector< link_info_t > &hdf5_hls)
Definition: h5dmr.cc:321
void get_softlink(D4Group *par_grp, hid_t h5obj_id, const string &oname, int index, size_t val_size)
Definition: h5dmr.cc:1078
Data structure and retrieval processing header for the default option.
string print_attr(hid_t type, int loc, void *sm_buf)
Definition: h5get.cc:863
string get_dap_type(hid_t type, bool is_dap4)
Definition: h5get.cc:291
void get_dataset(hid_t pid, const string &dname, DS_t *dt_inst_ptr)
Definition: h5get.cc:452
hid_t get_attr_info(hid_t dset, int index, bool is_dap4, DSattr_t *attr_inst_ptr, bool *ignore_attr_ptr)
Definition: h5get.cc:90
The main header of the HDF5 OPeNDAP handler.
const int DODS_NAMELEN
Maximum length of variable or attribute name(default option only).
Definition: hdf5_handler.h:65
A structure for DDS generation.
Definition: hdf5_handler.h:71
hsize_t nelmts
Number of elements.
Definition: hdf5_handler.h:89
hsize_t need
Space needed.
Definition: hdf5_handler.h:91
hid_t type
HDF5 data set id.
Definition: hdf5_handler.h:79
int size[DODS_MAX_RANK]
Size of each dimension.
Definition: hdf5_handler.h:85
int ndims
HDF5 data space id.
Definition: hdf5_handler.h:83
A structure for DAS generation.
Definition: hdf5_handler.h:94
char name[DODS_NAMELEN]
Name of HDF5 group or dataset.
Definition: hdf5_handler.h:96
int ndims
Number of dimensions.
Definition: hdf5_handler.h:100
hsize_t nelmts
Number of elements.
Definition: hdf5_handler.h:104
hsize_t need
Memory space needed to hold nelmts type.
Definition: hdf5_handler.h:106