bes Updated for version 3.20.13
HDFEOS5CF.cc
Go to the documentation of this file.
1// This file is part of the hdf5_handler implementing for the CF-compliant
2// Copyright (c) 2011-2016 The HDF Group, Inc. and OPeNDAP, Inc.
3//
4// This is free software; you can redistribute it and/or modify it under the
5// terms of the GNU Lesser General Public License as published by the Free
6// Software Foundation; either version 2.1 of the License, or (at your
7// option) any later version.
8//
9// This software is distributed in the hope that it will be useful, but
10// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
11// or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
12// License for more details.
13//
14// You should have received a copy of the GNU Lesser General Public
15// License along with this library; if not, write to the Free Software
16// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17//
18// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
19// You can contact The HDF Group, Inc. at 1800 South Oak Street,
20// Suite 203, Champaign, IL 61820
21
36
37#include "HDF5CF.h"
38#include "HDF5RequestHandler.h"
39#include "h5cfdaputil.h"
40#include "BESDebug.h"
41
42using namespace std;
43using namespace libdap;
44using namespace HDF5CF;
45
46// A constructor of EOS5CVar
47EOS5CVar::EOS5CVar(Var*var)
48{
49
50 newname = var->newname;
51 name = var->name;
52 fullpath = var->fullpath;
53 rank = var->rank;
54 total_elems = var->total_elems;
55 zero_storage_size = var->zero_storage_size;
56 dtype = var->dtype;
57 unsupported_attr_dtype = var->unsupported_attr_dtype;
58 unsupported_dspace = var->unsupported_dspace;
59 coord_attr_add_path = false;
60
61 for (auto ira = var->attrs.begin(); ira != var->attrs.end(); ++ira) {
62 auto attr = new Attribute();
63 attr->name = (*ira)->name;
64 attr->newname = (*ira)->newname;
65 attr->dtype = (*ira)->dtype;
66 attr->count = (*ira)->count;
67 attr->strsize = (*ira)->strsize;
68 attr->fstrsize = (*ira)->fstrsize;
69 attr->value = (*ira)->value;
70 attrs.push_back(attr);
71 }
72
73 for (auto ird = var->dims.begin(); ird != var->dims.end(); ++ird) {
74 Dimension *dim = new Dimension((*ird)->size);
75 dim->name = (*ird)->name;
76 dim->newname = (*ird)->newname;
77 dim->unlimited_dim = (*ird)->unlimited_dim;
78 dims.push_back(dim);
79 }
80
81 // For the coordinate variable specific fields, we just fill in the default one in the ctr
82 // If needed, the caller of this function should fill in those information after calling this function.
83 eos_type = OTHERVARS;
84 is_2dlatlon = false;
85 point_lower = 0.0;
86 point_upper = 0.0;
87 point_left = 0.0;
88 point_right = 0.0;
89 xdimsize = 0;
90 ydimsize = 0;
91 eos5_pixelreg = HE5_HDFE_CENTER;
92 eos5_origin = HE5_HDFE_GD_UL;
93 eos5_projcode = HE5_GCTP_GEO;
94 zone = -1;
95 sphere = 0;
96 std::fill_n(param, 13, 0);
97
98}
99
100//This method will effectively remove any dimnames like
101// ???/XDim or ???/YDim from the dimension name set.
102// Use this function in caution.
103void EOS5CFGrid::Update_Dimnamelist()
104{
105
106 BESDEBUG("h5", "coming to Update_Dimnamelist" <<endl);
107
108 // If I put both "XDim" and "YDim" into one for loop, Mac g++ compiler
109 // gives segmentation fault, which doesn't make sense.
110 // I simply split them into two loops. It doesn't affect performance much.
111 // KY 2012-2-14
112 for (auto it = this->vardimnames.begin(); it != this->vardimnames.end(); ++it) {
113 string xydimname_candidate = HDF5CFUtil::obtain_string_after_lastslash(*it);
114 if ("XDim" == xydimname_candidate) {
115 this->vardimnames.erase(*it);
116 break;
117 }
118 }
119
120 for (auto it = this->vardimnames.begin(); it != this->vardimnames.end(); ++it) {
121 string xydimname_candidate = HDF5CFUtil::obtain_string_after_lastslash(*it);
122 if ("YDim" == xydimname_candidate) {
123 this->vardimnames.erase(*it);
124 break;
125 }
126 }
127
128}
129
130// A destructor of EOS5File
131EOS5File::~EOS5File()
132{
133 for (vector<EOS5CVar *>::const_iterator i = this->cvars.begin(); i != this->cvars.end(); ++i)
134 delete *i;
135
136 for (vector<EOS5CFGrid *>::const_iterator i = this->eos5cfgrids.begin(); i != this->eos5cfgrids.end(); ++i)
137 delete *i;
138
139 for (vector<EOS5CFSwath *>::const_iterator i = this->eos5cfswaths.begin(); i != this->eos5cfswaths.end(); ++i)
140 delete *i;
141
142 for (vector<EOS5CFZa *>::const_iterator i = this->eos5cfzas.begin(); i != this->eos5cfzas.end(); ++i)
143 delete *i;
144
145}
146
147// Helper function to make the name follow the CF conventions.
148string EOS5File::get_CF_string(string s)
149{
150
151 // We need to remove the first "/" from the full name.
152 if (s[0] != '/')
153 return File::get_CF_string(s);
154 else {
155 s.erase(0, 1);
156 return File::get_CF_string(s);
157 }
158}
159
160// Retrieve the HDF5 information for HDF-EOS5
161void EOS5File::Retrieve_H5_Info(const char *file_fullpath, hid_t file_id, bool /*include_attr*/)
162{
163 // Since we need to check the attribute info in order to determine if the file is augmented to netCDF-4,
164 // we need to retrieve the attribute info also.
165 File::Retrieve_H5_Info(file_fullpath, file_id, true);
166}
167
169{
170
171 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ++ircv) {
172
173 // When the coordinate variables exist in the file, retrieve the attribute values.
174 if ((CV_EXIST == (*ircv)->cvartype) || (CV_MODIFY == (*ircv)->cvartype)) {
175 for (auto ira = (*ircv)->attrs.begin(); ira != (*ircv)->attrs.end(); ++ira)
176 Retrieve_H5_Attr_Value(*ira, (*ircv)->fullpath);
177
178 }
179 }
180
181}
182
183// Retrieve the attribute values for the HDF-EOS5
185{
186
188 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ++ircv) {
189
190 // When the coordinate variables exist in the file, retrieve the attribute values.
191 if ((CV_EXIST == (*ircv)->cvartype) || (CV_MODIFY == (*ircv)->cvartype)) {
192 for (auto ira = (*ircv)->attrs.begin(); ira != (*ircv)->attrs.end(); ++ira)
193 Retrieve_H5_Attr_Value(*ira, (*ircv)->fullpath);
194
195 }
196 }
197}
198
199// Adjust attribute value
200void EOS5File::Adjust_H5_Attr_Value(Attribute *attr)
201{
202 // For future usage.
203
204}
205
206// Handle unsupported datatype
208{
209
210 if (true == check_ignored) {
211 Gen_Unsupported_Dtype_Info(include_attr);
212 }
213
214 File::Handle_Unsupported_Dtype(include_attr);
215 Handle_EOS5_Unsupported_Dtype(include_attr);
216}
217
218// Handle EOS5 unsupported datatype,add EOS5 coordinate variables
219void EOS5File::Handle_EOS5_Unsupported_Dtype(bool include_attr)
220{
221
222 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end();) {
223 if (true == include_attr) {
224 for (auto ira = (*ircv)->attrs.begin(); ira != (*ircv)->attrs.end();) {
225 H5DataType temp_dtype = (*ira)->getType();
226 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
227 delete (*ira);
228 ira = (*ircv)->attrs.erase(ira);
229 }
230 else {
231 ++ira;
232
233 }
234 }
235 }
236
237 H5DataType temp_dtype = (*ircv)->getType();
238 if (!HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
239 delete (*ircv);
240 ircv = this->cvars.erase(ircv);
241 }
242 else {
243 ++ircv;
244 }
245 }
246}
247
248// Generate unsupported datatype information
249void EOS5File::Gen_Unsupported_Dtype_Info(bool include_attr)
250{
251
252 if (true == include_attr) {
253
254 File::Gen_Group_Unsupported_Dtype_Info();
255 File::Gen_Var_Unsupported_Dtype_Info();
256 Gen_VarAttr_Unsupported_Dtype_Info();
257
258 }
259
260}
261
262// Generate variable attribute datatype info.
263void EOS5File::Gen_VarAttr_Unsupported_Dtype_Info()
264{
265
266 // Dimension scale info for general variables
267 Gen_DimScale_VarAttr_Unsupported_Dtype_Info();
268
269 // HDF-EOS5 variable attribute unsupported datatype
270 Gen_EOS5_VarAttr_Unsupported_Dtype_Info();
271
272}
273
274void EOS5File::Gen_EOS5_VarAttr_Unsupported_Dtype_Info()
275{
276
277 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
278 // If the attribute REFERENCE_LIST comes with the attribut CLASS, the
279 // attribute REFERENCE_LIST is okay to ignore. No need to report.
280 bool is_ignored = ignored_dimscale_ref_list((*irv));
281 if (false == (*irv)->attrs.empty()) {
282 //if (true == (*irv)->unsupported_attr_dtype) {
283 for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
284 H5DataType temp_dtype = (*ira)->getType();
285 // TODO: check why 64-bit integer is included.
286 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4) || (temp_dtype == H5INT64) ||(temp_dtype == H5UINT64)) {
287 // "DIMENSION_LIST" is okay to ignore and "REFERENCE_LIST"
288 // is okay to ignore if the variable has another attribute
289 // CLASS="DIMENSION_SCALE"
290 if (("DIMENSION_LIST" != (*ira)->name)
291 && ("REFERENCE_LIST" != (*ira)->name || true == is_ignored))
292 this->add_ignored_info_attrs(false, (*irv)->fullpath, (*ira)->name);
293 }
294 }
295 //}
296 }
297 }
298}
299
300// Handle unsupported data space.
302{
303
304 // Generate unsupported info.
305 if (true == check_ignored) {
306 Gen_Unsupported_Dspace_Info();
307 }
308
310 Handle_EOS5_Unsupported_Dspace(include_attr);
311
312}
313
314// Handle EOS5 unsupported data space.
315void EOS5File::Handle_EOS5_Unsupported_Dspace(bool include_attr)
316{
317
318 if (true == this->unsupported_var_dspace) {
319 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end();) {
320 if (true == (*ircv)->unsupported_dspace) {
321 delete (*ircv);
322 ircv = this->cvars.erase(ircv);
323 }
324 else {
325 ++ircv;
326 }
327 }
328 }
329
330 if (true == include_attr) {
331 if (true == this->unsupported_var_attr_dspace) {
332 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ++ircv) {
333 if (false == (*ircv)->attrs.empty()) {
334 if (true == (*ircv)->unsupported_attr_dspace) {
335 for (auto ira = (*ircv)->attrs.begin(); ira != (*ircv)->attrs.end();) {
336 if (0 == (*ira)->count) {
337 delete (*ira);
338 ira = (*ircv)->attrs.erase(ira);
339 }
340 else {
341 ++ira;
342 }
343 }
344 }
345 }
346 }
347 }
348 }
349}
350
351// Generating unsupported data space.
352void EOS5File::Gen_Unsupported_Dspace_Info()
353{
354
355 File::Gen_Unsupported_Dspace_Info();
356
357}
358
359// Handle other unsupported EOS5 information
361{
362
363 remove_netCDF_internal_attributes(include_attr);
364#if 0
365 if(true == include_attr) {
366 for (auto irv = this->vars.begin();
367 irv != this->vars.end(); ++irv) {
368 for (auto ira = (*irv)->attrs.begin();
369 ira != (*irv)->attrs.end();) {
370 if((*ira)->name == "CLASS") {
371 string class_value = Retrieve_Str_Attr_Value(*ira,(*irv)->fullpath);
372
373 // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
374 // "DIMENSION_SCALE", which is 15.
375 if (0 == class_value.compare(0,15,"DIMENSION_SCALE")) {
376 delete((*ira));
377 ira = (*irv)->attrs.erase(ira);
378 }
379#if 0
380 else if(1) {// Add a BES key,also delete
381
382 }
383#endif
384 else {
385 ++ira;
386 }
387 }
388 //else if((*ira)->name == "NAME" && 1) {// Add a BES Key later if necessary
389 else if((*ira)->name == "NAME") {// Add a BES Key
390 string name_value = Retrieve_Str_Attr_Value(*ira,(*irv)->fullpath);
391 if( 0 == name_value.compare(0,(*irv)->name.size(),(*irv)->name)) {
392 delete((*ira));
393 ira =(*irv)->attrs.erase(ira);
394 }
395 else {
396 string netcdf_dim_mark= "This is a netCDF dimension but not a netCDF variable";
397 if( 0 == name_value.compare(0,netcdf_dim_mark.size(),netcdf_dim_mark)) {
398 delete((*ira));
399 ira =(*irv)->attrs.erase(ira);
400 }
401 else {
402 ++ira;
403 }
404 }
405
406 }
407 else if((*ira)->name == "_Netcdf4Dimid") {
408 delete((*ira));
409 ira =(*irv)->attrs.erase(ira);
410 }
411
412 else {
413 ++ira;
414 }
415 }
416 }
417#endif
418 if(true == include_attr) {
419 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
420 for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end();) {
421 if((*ira)->name == "CLASS") {
422 string class_value = Retrieve_Str_Attr_Value(*ira,(*irv)->fullpath);
423
424 // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
425 // "DIMENSION_SCALE", which is 15.
426 if (0 == class_value.compare(0,15,"DIMENSION_SCALE")) {
427 delete(*ira);
428 ira = (*irv)->attrs.erase(ira);
429 // Add another block to set a key
430 }
431 else {
432 ++ira;
433 }
434 }
435 else if((*ira)->name == "NAME") {// Add a BES Key later
436 delete(*ira);
437 ira=(*irv)->attrs.erase(ira);
438 //"NAME" attribute causes the file netCDF-4 failed.
439#if 0
440 string name_value = Retrieve_Str_Attr_Value(*ira,(*irv)->fullpath);
441 if( 0 == name_value.compare(0,(*irv)->name.size(),(*irv)->name)) {
442 delete(*ira);
443 ira =(*irv)->attrs.erase(ira);
444 }
445 else {
446 string netcdf_dim_mark= "This is a netCDF dimension but not a netCDF variable";
447 if( 0 == name_value.compare(0,netcdf_dim_mark.size(),netcdf_dim_mark)) {
448 delete(*ira);
449 ira =(*irv)->attrs.erase(ira);
450 }
451 else {
452 ++ira;
453 }
454 }
455#endif
456 }
457 else if((*ira)->name == "_Netcdf4Dimid") {
458 delete(*ira);
459 ira =(*irv)->attrs.erase(ira);
460 }
461
462 else {
463 ++ira;
464 }
465 }
466 }
467 }
468
469
470 // We cannot use the general routine from the base class since
471 // the information of ignored ECS metadata variables is transferred
472 // to DAS. The ignored ECS metadata variables should not be reported.
473 //File::Handle_Unsupported_Others(include_attr);
474 if (true == this->check_ignored && true == include_attr) {
475
476 // netCDF Java lifts the string size restriction for attributes. So comment out for the time being. KY 2018/08/10
477 if (true == HDF5RequestHandler::get_drop_long_string()) {
478#if 0
479 for (auto ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira) {
480 if (H5FSTRING == (*ira)->dtype || H5VSTRING == (*ira)->dtype) {
481 if ((*ira)->getBufSize() > NC_JAVA_STR_SIZE_LIMIT) {
482 this->add_ignored_droplongstr_hdr();
483 this->add_ignored_grp_longstr_info("/", (*ira)->name);
484 }
485 }
486 }
487
488 for (auto irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
489 for (auto ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end(); ++ira) {
490 if (H5FSTRING == (*ira)->dtype || H5VSTRING == (*ira)->dtype) {
491 if ((*ira)->getBufSize() > NC_JAVA_STR_SIZE_LIMIT) {
492 this->add_ignored_droplongstr_hdr();
493 this->add_ignored_grp_longstr_info((*irg)->path, (*ira)->name);
494 }
495 }
496
497 }
498 }
499#endif
500 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
501 if (true == Check_DropLongStr((*irv), nullptr)) {
502 string ecsmeta_grp = "/HDFEOS INFORMATION";
503 // Ignored ECS metadata should not be reported.
504 if ((*irv)->fullpath.find(ecsmeta_grp) != 0
505 || ((*irv)->fullpath.rfind("/") != ecsmeta_grp.size())) {
506 this->add_ignored_droplongstr_hdr();
507 this->add_ignored_var_longstr_info((*irv), nullptr);
508 }
509 }
510#if 0
511 for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
512 if (true == Check_DropLongStr((*irv), (*ira))) {
513 this->add_ignored_droplongstr_hdr();
514 this->add_ignored_var_longstr_info((*irv), (*ira));
515 }
516 }
517#endif
518 }
519#if 0
520 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
521 for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
522 if (true == Check_DropLongStr((*irv), (*ira))) {
523 this->add_ignored_droplongstr_hdr();
524 this->add_ignored_var_longstr_info((*irv), (*ira));
525 }
526 }
527 }
528#endif
529 }
530 }
531
532 if (false == this->have_ignored) this->add_no_ignored_info();
533
534}
535
536// Adjust HDF-EOS5 dimension info.
538{
539
540 BESDEBUG("h5", "coming to Adjust_EOS5Dim_Info" <<endl);
541
542 // Condense redundant XDim, YDim in the grid/swath/za dimension list
543 for (unsigned int i = 0; i < strmeta_info->swath_list.size(); ++i) {
544 HE5Swath& he5s = strmeta_info->swath_list.at(i);
545
546 Adjust_EOS5Dim_List(he5s.dim_list);
547
548 // Correct the possible wrong dimension size,this only happens for the unlimited dimension,
549 // WE JUST NEED TO CORRECT the EOS group dimension size.
550 // STEPS:
551 // 1. Merge SWATH data_var_list and geo_var_list
552 // Function parameters will be the object dim. list(he5s.dim_list), EOS5Type(SWATH,GRID...) and varlist
553 // Need to use Obtain_Var_EOS5Type_GroupName to find var's group name and Get_Var_EOS5_Type(var) to find
554 // Var's EOS5Type.
555 // After checking group and type, check "if(he5v.name == var->name)" and change the he5v dim. size to var size.
556 if(this->have_udim == true) {
557 vector<HE5Var> svlist = he5s.geo_var_list;
558 svlist.insert(svlist.end(),he5s.data_var_list.begin(),he5s.data_var_list.end());
559 // Only apply when the unlimited dimension is found!! So we don't have to go over this for every file.
560 Adjust_EOS5DimSize_List(he5s.dim_list,svlist,SWATH,he5s.name);
561 }
562
563 for (unsigned int j = 0; j < he5s.geo_var_list.size(); ++j) {
564 Adjust_EOS5VarDim_Info((he5s.geo_var_list)[j].dim_list, he5s.dim_list, he5s.name, SWATH);
565 }
566 for (unsigned int j = 0; j < he5s.data_var_list.size(); ++j) {
567 Adjust_EOS5VarDim_Info((he5s.data_var_list)[j].dim_list, he5s.dim_list, he5s.name, SWATH);
568 }
569 }
570
571 for (unsigned int i = 0; i < strmeta_info->grid_list.size(); ++i) {
572
573 HE5Grid& he5g = strmeta_info->grid_list.at(i);
574
575 Adjust_EOS5Dim_List(he5g.dim_list);
576
577 // Correct possible wrong dimension size in the eosdim list.
578 if(this->have_udim == true) {
579 // Only apply when the unlimited dimension is found!! So we don't have to go over this for every file.
580 Adjust_EOS5DimSize_List(he5g.dim_list,he5g.data_var_list,GRID,he5g.name);
581 }
582
583 for (unsigned int j = 0; j < he5g.data_var_list.size(); ++j) {
584 Adjust_EOS5VarDim_Info((he5g.data_var_list)[j].dim_list, he5g.dim_list, he5g.name, GRID);
585 }
586 }
587
588 for (unsigned int i = 0; i < strmeta_info->za_list.size(); ++i) {
589 HE5Za& he5z = strmeta_info->za_list.at(i);
590
591 Adjust_EOS5Dim_List(he5z.dim_list);
592
593 // Correct possible wrong dimension size in the eosdim list.
594 if(this->have_udim == true) {
595 // Only apply when the unlimited dimension is found!! So we don't have to go over this for every file.
596 Adjust_EOS5DimSize_List(he5z.dim_list,he5z.data_var_list,ZA,he5z.name);
597 }
598
599 for (unsigned int j = 0; j < he5z.data_var_list.size(); ++j) {
600 Adjust_EOS5VarDim_Info((he5z.data_var_list)[j].dim_list, he5z.dim_list, he5z.name, ZA);
601 }
602 }
603}
604
605// Adjust HDF-EOS5 dimension list.
606void EOS5File::Adjust_EOS5Dim_List(vector<HE5Dim>& groupdimlist)
607{
608
609 BESDEBUG("h5", "Coming to Adjust_EOS5Dim_List"<<endl);
610
611 // The negative dimension sizes are found in some HDF-EOS5 files.
612 // We need to remove them.
613 Remove_NegativeSizeDims(groupdimlist);
614
615 // Condense redundant XDim, YDim in the grid/swath/za dimension list
616 Condense_EOS5Dim_List(groupdimlist);
617
618}
619
620// The negative dimension sizes are found in some HDF-EOS5 files.
621// We need to remove them.
622void EOS5File::Remove_NegativeSizeDims(vector<HE5Dim>& groupdimlist)
623{
624
625 BESDEBUG("h5", "Coming to Remove_NegativeSizeDims" <<endl);
626
627 // We find one product has dimension with name: Unlimited, size: -1; this dimension
628 // will not be used by any variables. The "Unlimited" dimension is useful for extended
629 // datasets when data is written. It is not useful for data accessing as far as I know.
630 // So we will remove it from the list.
631 // This algoritm will also remove any dimension with size <=0. KY 2011-1-14
632 // Note: Unlimited dimension is supported by the handler but not by using this "Unlimited" name.
633 // For the unlimited dimension support, check class Dimension and function Retrieve_H5_VarDim.
634 for (auto id = groupdimlist.begin(); id != groupdimlist.end();) {
635 if ((*id).size <= 0) {
636 id = groupdimlist.erase(id);
637 }
638 else {
639 ++id;
640 }
641 }
642}
643
644// Condense redundant XDim, YDim in the grid/swath/za dimension list
645// Some products use Xdim rather XDim, Ydim rather than Ydim.
646// This is significant for grids. We need to make them "XDim" and "YDim".
647// See comments of function Adjust_EOS5VarDim_Info for the reason.
648void EOS5File::Condense_EOS5Dim_List(vector<HE5Dim>& groupdimlist)
649{
650
651 BESDEBUG("h5", "Coming to Condense_EOS5Dim_List"<<endl);
652 set<int> xdimsizes;
653 set<int> ydimsizes;
654 pair<set<int>::iterator, bool> setret;
655 vector<HE5Dim>::iterator id;
656
657 for (id = groupdimlist.begin(); id != groupdimlist.end();) {
658 if ("XDim" == (*id).name || "Xdim" == (*id).name) {
659 setret = xdimsizes.insert((*id).size);
660 if (false == setret.second) {
661 id = groupdimlist.erase(id);
662 }
663 else if ("Xdim" == (*id).name) {
664 (*id).name = "XDim";
665 ++id;
666 }
667 else {
668 ++id;
669 }
670
671 }
672 else {
673 ++id;
674 }
675 }
676
677 for (id = groupdimlist.begin(); id != groupdimlist.end();) {
678 if ("YDim" == (*id).name || "Ydim" == (*id).name) {
679 setret = ydimsizes.insert((*id).size);
680 if (false == setret.second) {
681 id = groupdimlist.erase(id);
682 }
683 else if ("Ydim" == (*id).name) {
684 (*id).name = "YDim";
685 ++id;
686 }
687 else {
688 ++id;
689 }
690 }
691 else {
692 ++id;
693 }
694 }
695}
696
697void EOS5File:: Adjust_EOS5DimSize_List(vector<HE5Dim>& eos5objdimlist,const vector<HE5Var> & eos5objvarlist,
698 const EOS5Type eos5type, const string & eos5objname)
699{
700
701 set<string>updated_dimlist;
702 pair<set<string>::iterator,bool> set_insert_ret;
703
704 for(unsigned int i = 0; i<eos5objvarlist.size();i++) {
705 HE5Var he5v = eos5objvarlist.at(i);
706 for(unsigned int j = 0; j<he5v.dim_list.size();j++) {
707 HE5Dim he5d = he5v.dim_list.at(j);
708 set_insert_ret = updated_dimlist.insert(he5d.name);
709 if(set_insert_ret.second == true) {
710 // Find out the index of this dimension in eos5objdimlist
711 unsigned int objdimlist_index = 9999;
712 bool has_objdimlist_index = false;
713 for(unsigned int k = 0; k <eos5objdimlist.size();k++) {
714 if(eos5objdimlist[k].name == he5d.name) {
715 objdimlist_index = k;
716 has_objdimlist_index = true;
717 break;
718 }
719 }
720 if(has_objdimlist_index == false)
721 throw2("Cannot find the dimension in the EOS5 object dimension list for the dimension ", he5d.name);
722 for (vector<Var *>::const_iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
723
724 EOS5Type vartype = Get_Var_EOS5_Type((*irv));
725 // Compare the EOS5 object type: SWATH,GRID or ZA
726 // eos5objvarlist only stores the variable name, not the path. So we have to ensure the path matches.
727 if(vartype == eos5type) {
728 string var_eos5gname = Obtain_Var_EOS5Type_GroupName((*irv),vartype);
729 // Compare the EOS5 object name
730 // Now we need to match the var name from eos5objvarlist with the var name.
731 if(var_eos5gname == eos5objname) {
732 if((*irv)->name == he5v.name) {
733 if (he5v.dim_list.size() != (*irv)->dims.size())
734 throw2("Number of dimensions don't match with the structmetadata for variable ", (*irv)->name);
735 // Change dimension size
736 (eos5objdimlist[objdimlist_index]).size = ((*irv)->dims[j])->size;
737 break;
738 }
739
740 }
741 }
742 }
743 }
744
745 }
746 // Don't need to go over every var, just find enough.
747 if(updated_dimlist.size() == eos5objdimlist.size())// Finish updating the eos5objdimlist
748 break;
749 }
750#if 0
751for(unsigned int k = 0; k <eos5objdimlist.size();k++) {
752 cerr<<"eos5 obj dim name is "<<eos5objdimlist[k].name << " Size is "<< eos5objdimlist[k].size << endl;
753}
754#endif
755}
756
757
758// Adjust HDF-EOS5 Variable,dimension information.
759void EOS5File::Adjust_EOS5VarDim_Info(vector<HE5Dim>& vardimlist, vector<HE5Dim>& groupdimlist,
760 const string & eos5_obj_name, EOS5Type eos5type)
761{
762
763 BESDEBUG("h5", "Coming to Adjust_EOS5VarDim_Info"<<endl);
764 set<string> dimnamelist;
765 pair<set<string>::iterator, bool> setret;
766
767 // For EOS5 Grids: Dimension names XDim and YDim are predefined.
768 // Even the data producers make a mistake to define "xdim", "ydim" etc in the grid
769 // dimension name list, the variable will still pick up "XDim" and "YDim" as their
770 // dimension names So we assume that 'xdim", "ydim" etc will never appear in the
771 // variable name list.
772 for (unsigned int i = 0; i < vardimlist.size(); ++i) {
773
774 HE5Dim& he5d = vardimlist.at(i);
775 bool dim_in_groupdimlist = false;
776 for (unsigned int j = 0; j < groupdimlist.size(); ++j) {
777 HE5Dim he5gd = groupdimlist.at(j);
778 if (he5gd.name == he5d.name) {
779 he5d.size = he5gd.size;
780 dim_in_groupdimlist = true;
781 break;
782 }
783 }
784
785 if (false == dim_in_groupdimlist)
786 throw2("The EOS5 group dimension name list doesn't include the dimension ", he5d.name);
787
788 // Some variables have data like float foo[nlevel= 10][nlevel= 10],need to make the dimname unique
789 // to ensure the coordinate variables to be generated correctly.
790 //
791 setret = dimnamelist.insert(he5d.name);
792 if (false == setret.second) {
793 int clash_index = 1;
794 string temp_clashname = he5d.name + '_';
795 HDF5CFUtil::gen_unique_name(temp_clashname, dimnamelist, clash_index);
796
797 string ori_dimname = he5d.name;
798
799 he5d.name = temp_clashname;
800
801 // We have to add this dim. to this dim. list if this dim doesn't exist in the dim. list.
802 bool dim_exist = false;
803 for (unsigned int j = 0; j < groupdimlist.size(); ++j) {
804 if (he5d.name == groupdimlist[j].name && he5d.size == groupdimlist[j].size) {
805 dim_exist = true;
806 break;
807 }
808 }
809
810 // Add the new dim. to the dim. list
811 if (false == dim_exist) {
812 ori_dimname = eos5_obj_name + "/" + ori_dimname;
813 string dup_dimname = eos5_obj_name + "/" + he5d.name;
814 if (GRID == eos5type) {
815 ori_dimname = "/GRIDS/" + ori_dimname;
816 dup_dimname = "/GRIDS/" + dup_dimname;
817 }
818 else if (SWATH == eos5type) {
819 ori_dimname = "/SWATHS/" + ori_dimname;
820 dup_dimname = "/SWATHS/" + dup_dimname;
821 }
822 else if (ZA == eos5type) {
823 ori_dimname = "/ZAS/" + ori_dimname;
824 dup_dimname = "/ZAS/" + dup_dimname;
825 }
826
827 // Need to remember the dimname and dupdimname relation in case the situation happens at other variables.
828 dimname_to_dupdimnamelist.insert(pair<string, string>(ori_dimname, dup_dimname));
829 groupdimlist.push_back(he5d);
830 }
831
832 } //end of if(false == setret.second)
833 } // end of for (unsigned int i = 0; i <vardimlist.size(); ++i)
834
835}
836
837// Add EOS5 FIle information
838void EOS5File::Add_EOS5File_Info(HE5Parser * strmeta_info, bool grids_mllcv)
839{
840
841 BESDEBUG("h5", "Coming to Add_EOS5File_Info"<<endl);
842 string fslash_str = "/";
843 string grid_str = "/GRIDS/";
844 string swath_str = "/SWATHS/";
845 string za_str = "/ZAS/";
846
847 // Assign the original number of grids. These number will be useful
848 // to generate the final DAP object names for grids/swaths/zas that don't have coordinate
849 // variables. For example, OMI level 2G product has latitude and longitude with 3-D arrays.
850 // There is no way to make the lat/lon become CF coordinate variables. To still follow the
851 // HDF-EOS5 object name conventions, the original number of grid is expected.
852 // Since this happens only for grids, we just keep the original number for grids now.
853 this->orig_num_grids = (int)(strmeta_info->grid_list.size());
854
855 //
856 for (unsigned int i = 0; i < strmeta_info->grid_list.size(); i++) {
857 HE5Grid he5g = strmeta_info->grid_list.at(i);
858 auto eos5grid = new EOS5CFGrid();
859 eos5grid->name = he5g.name;
860 eos5grid->dimnames.resize(he5g.dim_list.size());
861
862 for (unsigned int j = 0; j < he5g.dim_list.size(); j++) {
863
864 HE5Dim he5d = he5g.dim_list.at(j);
865 if ("XDim" == he5d.name) eos5grid->xdimsize = he5d.size;
866 if ("YDim" == he5d.name) eos5grid->ydimsize = he5d.size;
867
868 // Here we add the grid name connecting with "/" to
869 // adjust the dim names to assure the uniqueness of
870 // the dimension names for multiple grids.
871 // For single grid, we don't have to do that.
872 // However, considering the rare case that one
873 // can have one grid, one swath and one za, the dimnames
874 // without using the group names may cause the name clashings.
875 // so still add the group path.
876 string unique_dimname = grid_str + he5g.name + fslash_str + he5d.name;
877
878 (eos5grid->dimnames)[j] = unique_dimname;
879
880 pair<map<hsize_t, string>::iterator, bool> mapret1;
881 mapret1 = eos5grid->dimsizes_to_dimnames.insert(pair<hsize_t, string>((hsize_t) he5d.size, unique_dimname));
882
883 // Create the dimname to dimsize map. This will be used to create the missing coordinate
884 // variables. Based on our understanding, dimension names should be unique for
885 // grid/swath/zonal average. We will throw an error if we find the same dimension name used.
886 pair<map<string, hsize_t>::iterator, bool> mapret2;
887 mapret2 = eos5grid->dimnames_to_dimsizes.insert(pair<string, hsize_t>(unique_dimname, (hsize_t) he5d.size));
888 if (false == mapret2.second)
889 throw5("The dimension name ", unique_dimname, " with the dimension size ", he5d.size, "is not unique");
890
891 } // "for (int j=0; j <he5g.dim_list.size(); j++)"
892
893 // Check if having Latitude/Longitude. We will use those Latitude and Longitude as CVs if possible.
894 EOS5SwathGrid_Set_LatLon_Flags(eos5grid, he5g.data_var_list);
895
896 // Using map for possible the third-D CVs.
897 map<string, string> dnames_to_1dvnames;
898 EOS5Handle_nonlatlon_dimcvars(he5g.data_var_list, GRID, he5g.name, dnames_to_1dvnames);
899 eos5grid->dnames_to_1dvnames = dnames_to_1dvnames;
900 eos5grid->point_lower = he5g.point_lower;
901 eos5grid->point_upper = he5g.point_upper;
902 eos5grid->point_left = he5g.point_left;
903 eos5grid->point_right = he5g.point_right;
904
905 eos5grid->eos5_pixelreg = he5g.pixelregistration;
906 eos5grid->eos5_origin = he5g.gridorigin;
907 eos5grid->eos5_projcode = he5g.projection;
908
909 for (unsigned int k = 0; k < 13; k++)
910 eos5grid->param[k] = he5g.param[k];
911 eos5grid->zone = he5g.zone;
912 eos5grid->sphere = he5g.sphere;
913
914 this->eos5cfgrids.push_back(eos5grid);
915
916 } // "for(int i=0; i < strmeta_info->grid_list.size(); i++)"
917
918 // Adding this here seems a hack.
919 this->grids_multi_latloncvs = grids_mllcv;
920
921 // Second Swath
922 for (unsigned int i = 0; i < strmeta_info->swath_list.size(); i++) {
923
924 HE5Swath he5s = strmeta_info->swath_list.at(i);
925 auto eos5swath = new EOS5CFSwath();
926 eos5swath->name = he5s.name;
927 eos5swath->dimnames.resize(he5s.dim_list.size());
928
929 for (unsigned int j = 0; j < he5s.dim_list.size(); j++) {
930
931 HE5Dim he5d = he5s.dim_list.at(j);
932
933 // Here we add the swath name connecting with "/" to
934 // adjust the dim names to assure the uniqueness of
935 // the dimension names for multiple swaths.
936 // For single swath, we don't have to do that.
937 // However, considering the rare case that one
938 // can have one grid, one swath and one za, the dimnames
939 // without using the group names may cause the name clashings.
940 // so still add the group path.
941 string unique_dimname = swath_str + he5s.name + fslash_str + he5d.name;
942 (eos5swath->dimnames)[j] = unique_dimname;
943
944 // Create the dimsize to dimname map for those variables missing dimension names.
945 // Note: For different dimnames sharing the same dimsizes, we only pick up the first one.
946 pair<map<hsize_t, string>::iterator, bool> mapret1;
947 mapret1 = eos5swath->dimsizes_to_dimnames.insert(
948 pair<hsize_t, string>((hsize_t) he5d.size, unique_dimname));
949
950 // Create the dimname to dimsize map. This will be used to create the missing coordinate
951 // variables. Based on our understanding, dimension names should be unique for
952 // grid/swath/zonal average. We will throw an error if we find the same dimension name used.
953 pair<map<string, hsize_t>::iterator, bool> mapret2;
954 mapret2 = eos5swath->dimnames_to_dimsizes.insert(
955 pair<string, hsize_t>(unique_dimname, (hsize_t) he5d.size));
956 if (false == mapret2.second)
957 throw5("The dimension name ", unique_dimname, " with the dimension size ", he5d.size, "is not unique");
958
959 } // "for (int j=0; j <he5s.dim_list.size(); j++)"
960
961 // Check if having Latitude/Longitude.
962 EOS5SwathGrid_Set_LatLon_Flags(eos5swath, he5s.geo_var_list);
963
964 // Using map for possible the third-D CVs.
965 map<string, string> dnames_to_geo1dvnames;
966 EOS5Handle_nonlatlon_dimcvars(he5s.geo_var_list, SWATH, he5s.name, dnames_to_geo1dvnames);
967 eos5swath->dnames_to_geo1dvnames = dnames_to_geo1dvnames;
968 this->eos5cfswaths.push_back(eos5swath);
969 } // "for (int i=0; i < strmeta_info->swath_list.size(); i++)"
970
971 // Third Zonal average
972 for (unsigned int i = 0; i < strmeta_info->za_list.size(); i++) {
973
974 HE5Za he5z = strmeta_info->za_list.at(i);
975
976 auto eos5za = new EOS5CFZa();
977 eos5za->name = he5z.name;
978 eos5za->dimnames.resize(he5z.dim_list.size());
979
980 for (unsigned int j = 0; j < he5z.dim_list.size(); j++) {
981
982 HE5Dim he5d = he5z.dim_list.at(j);
983
984 // Here we add the grid name connecting with "/" to
985 // adjust the dim names to assure the uniqueness of
986 // the dimension names for multiple grids.
987 // For single grid, we don't have to do that.
988 string unique_dimname = za_str + he5z.name + fslash_str + he5d.name;
989 (eos5za->dimnames)[j] = unique_dimname;
990 pair<map<hsize_t, string>::iterator, bool> mapret1;
991 mapret1 = eos5za->dimsizes_to_dimnames.insert(pair<hsize_t, string>((hsize_t) he5d.size, unique_dimname));
992
993 // Create the dimname to dimsize map. This will be used to create the missing coordinate
994 // variables. Based on our understanding, dimension names should be unique for
995 // grid/swath/zonal average. We will throw an error if we find the same dimension name used.
996 pair<map<string, hsize_t>::iterator, bool> mapret2;
997 mapret2 = eos5za->dimnames_to_dimsizes.insert(pair<string, hsize_t>(unique_dimname, (hsize_t) he5d.size));
998 if (false == mapret2.second)
999 throw5("The dimension name ", unique_dimname, " with the dimension size ", he5d.size, "is not unique");
1000
1001 } // "for (int j=0; j <he5z.dim_list.size(); j++) "
1002
1003 // Using map for possible the third-D CVs.
1004 map<string, string> dnames_to_1dvnames;
1005 EOS5Handle_nonlatlon_dimcvars(he5z.data_var_list, ZA, he5z.name, dnames_to_1dvnames);
1006 eos5za->dnames_to_1dvnames = dnames_to_1dvnames;
1007 this->eos5cfzas.push_back(eos5za);
1008 } // "for(int i=0; i < strmeta_info->za_list.size(); i++)"
1009
1010// Debugging info,leave it here. They are very useful.
1011#if 0
1012 for (auto irg = this->eos5cfgrids.begin();
1013 irg != this->eos5cfgrids.end(); ++irg) {
1014
1015 cerr<<"grid name "<<(*irg)->name <<endl;
1016 cerr<<"eos5_pixelreg"<<(*irg)->eos5_pixelreg <<endl;
1017 cerr<<"eos5_origin"<<(*irg)->eos5_pixelreg <<endl;
1018 cerr<<"point_lower "<<(*irg)->point_lower <<endl;
1019 cerr<<"xdimsize "<<(*irg)->xdimsize <<endl;
1020
1021 if((*irg)->has_g2dlatlon) cerr<<"has g2dlatlon"<<endl;
1022 if((*irg)->has_2dlatlon) cerr<<"has 2dlatlon"<<endl;
1023 if((*irg)->has_1dlatlon) cerr<<"has 1dlatlon"<<endl;
1024 if((*irg)->has_nolatlon) cerr<<"has no latlon" <<endl;
1025 if(this->grids_multi_latloncvs) cerr<<"having multiple lat/lon from structmeta" <<endl;
1026 else cerr<<"no multiple lat/lon from structmeta" <<endl;
1027
1028// Dimension names
1029 "h5","number of dimensions "<<(*irg)->dimnames.size() <<endl;
1030 for (auto irv = (*irg)->dimnames.begin();
1031 irv != (*irg)->dimnames.end(); ++irv)
1032 cerr<<"dim names" <<*irv <<endl;
1033
1034// mapping size to name
1035 for (auto im1 = (*irg)->dimsizes_to_dimnames.begin();
1036 im1 !=(*irg)->dimsizes_to_dimnames.end();++im1) {
1037 cerr<<"size to name "<< (int)((*im1).first) <<"=> "<<(*im1).second <<endl;
1038 }
1039
1040// mapping dime names to 1d varname
1041 for (auto im2 = (*irg)->dnames_to_1dvnames.begin();
1042 im2 !=(*irg)->dnames_to_1dvnames.end();++im2) {
1043 cerr<<"dimanme to 1d var name "<< (*im2).first <<"=> "<<(*im2).second <<endl;
1044 }
1045 }
1046
1047//Swath
1048 for (auto irg = this->eos5cfswaths.begin();
1049 irg != this->eos5cfswaths.end(); ++irg) {
1050
1051 cerr<<"swath name "<<(*irg)->name <<endl;
1052 if((*irg)->has_nolatlon) cerr<<"has no latlon" <<endl;
1053 if((*irg)->has_1dlatlon) cerr<<"has 1dlatlon"<<endl;
1054 if((*irg)->has_2dlatlon) cerr<<"has 2dlatlon"<<endl;
1055
1056// Dimension names
1057 for (auto irv = (*irg)->dimnames.begin();
1058 irv != (*irg)->dimnames.end(); ++irv)
1059 cerr<<"dim names" <<*irv <<endl;
1060
1061// mapping size to name
1062 for (auto im1 = (*irg)->dimsizes_to_dimnames.begin();
1063 im1 !=(*irg)->dimsizes_to_dimnames.end();++im1) {
1064 cerr<<"size to name "<< (int)((*im1).first) <<"=> "<<(*im1).second <<endl;
1065 }
1066
1067// mapping dime names to 1d varname
1068 for (auto im2 = (*irg)->dnames_to_geo1dvnames.begin();
1069 im2 !=(*irg)->dnames_to_geo1dvnames.end();++im2) {
1070 cerr<<"dimname to 1d varname "<< (*im2).first <<"=> "<<(*im2).second <<endl;
1071 }
1072 }
1073
1074 for (auto irg = this->eos5cfzas.begin();
1075 irg != this->eos5cfzas.end(); ++irg) {
1076
1077 cerr<<"za name now"<<(*irg)->name <<endl;
1078
1079// Dimension names
1080 for (auto irv = (*irg)->dimnames.begin();
1081 irv != (*irg)->dimnames.end(); ++irv)
1082 cerr<<"dim names" <<*irv <<endl;
1083
1084// mapping size to name
1085 for (auto im1 = (*irg)->dimsizes_to_dimnames.begin();
1086 im1 !=(*irg)->dimsizes_to_dimnames.end();++im1) {
1087 cerr<<"size to name "<< (int)((*im1).first) <<"=> "<<(*im1).second <<endl;
1088 }
1089
1090// mapping dime names to 1d varname
1091 for (auto im2 = (*irg)->dnames_to_1dvnames.begin();
1092 im2 !=(*irg)->dnames_to_1dvnames.end();++im2) {
1093 cerr<<"dimname to 1d varname "<< (*im2).first <<"=> "<<(*im2).second <<endl;
1094 }
1095 }
1096#endif
1097
1098}
1099
1100// Check if EOS5 Swath and Grid hold Latitude and Longitude fields.
1101template<class T>
1102void EOS5File::EOS5SwathGrid_Set_LatLon_Flags(T* eos5gridswath, vector<HE5Var> &eos5varlist)
1103{
1104
1105 BESDEBUG("h5", "Coming to EOS5SwathGrid_Set_LatLon_Flags"<<endl);
1106 bool find_lat = false;
1107 bool find_lon = false;
1108 bool has_1dlat = false;
1109 bool has_1dlon = false;
1110 bool has_2dlat = false;
1111 string lat_xdimname;
1112 string lat_ydimname;
1113 string lon_xdimname;
1114 string lon_ydimname;
1115 bool has_2dlon = false;
1116 bool has_g2dlat = false;
1117 bool has_g2dlon = false;
1118
1119 for (unsigned int i = 0; i < eos5varlist.size(); ++i) {
1120 HE5Var he5v = eos5varlist.at(i);
1121 if ("Latitude" == he5v.name) {
1122 find_lat = true;
1123 auto num_dims = (int)(he5v.dim_list.size());
1124 if (1 == num_dims)
1125 has_1dlat = true;
1126 else if (2 == num_dims) {
1127 lat_ydimname = (he5v.dim_list)[0].name;
1128 lat_xdimname = (he5v.dim_list)[1].name;
1129 has_2dlat = true;
1130 }
1131 else if (num_dims > 2)
1132 has_g2dlat = true;
1133 else
1134 throw1("The number of dimension should not be 0 for grids or swaths");
1135 } // "if ("Latitude" == he5v.name)"
1136
1137 if ("Longitude" == he5v.name) {
1138 find_lon = true;
1139 auto num_dims = (int)(he5v.dim_list.size());
1140 if (1 == num_dims)
1141 has_1dlon = true;
1142 else if (2 == num_dims) {
1143 lon_ydimname = (he5v.dim_list)[0].name;
1144 lon_xdimname = (he5v.dim_list)[1].name;
1145 has_2dlon = true;
1146 }
1147 else if (num_dims > 2)
1148 has_g2dlon = true;
1149 else
1150 throw1("The number of dimension should not be 0 for grids or swaths");
1151 } // "if ("Longitude" == he5v.name)"
1152
1153 if (true == find_lat && true == find_lon) {
1154 if (true == has_1dlat && true == has_1dlon) eos5gridswath->has_1dlatlon = true;
1155
1156 // Make sure we have lat[YDIM][XDIM] and lon[YDIM][XDIM]
1157 if (true == has_2dlat && true == has_2dlon && lat_ydimname == lon_ydimname && lat_xdimname == lon_xdimname)
1158 eos5gridswath->has_2dlatlon = true;
1159
1160 if (true == has_g2dlat && true == has_g2dlon) eos5gridswath->has_g2dlatlon = true;
1161
1162 eos5gridswath->has_nolatlon = false;
1163 break;
1164 } // "if (true == find_lat && true == find_lon) "
1165 } // "for (unsigned int i = 0; i < eos5varlist.size(); ++i)"
1166}
1167
1168// This function builds up the map from dimension names to coordinate variables
1169// for non-latitude and longitude fields.
1170void EOS5File::EOS5Handle_nonlatlon_dimcvars(vector<HE5Var> & eos5varlist, EOS5Type eos5type, string groupname,
1171 map<string, string>& dnamesgeo1dvnames)
1172{
1173
1174 BESDEBUG("h5", "Coming to EOS5Handle_nonlatlon_dimcvars"<<endl);
1175
1176 set<string> nocvdimnames;
1177 string grid_str = "/GRIDS/";
1178 string xdim_str = "XDim";
1179 string ydim_str = "YDim";
1180 string fslash_str = "/";
1181 string eos5typestr;
1182
1183 if (GRID == eos5type) {
1184 string xdimname = grid_str + groupname + fslash_str + xdim_str;
1185 nocvdimnames.insert(xdimname);
1186 string ydimname = grid_str + groupname + fslash_str + ydim_str;
1187 nocvdimnames.insert(ydimname);
1188 eos5typestr = "/GRIDS/";
1189 }
1190 else if (SWATH == eos5type)
1191 eos5typestr = "/SWATHS/";
1192 else if (ZA == eos5type)
1193 eos5typestr = "/ZAS/";
1194 else
1195 throw1("Unsupported HDF-EOS5 type, this type is not swath, grid or zonal average");
1196
1197 // This assumption is pretty bold. It says: Any 1-D var that has a unique dim. name
1198 // in the var list is a 3rd-dim cv. We need to review this as time goes on. KY 2017-10-19
1199 pair<map<string, string>::iterator, bool> mapret;
1200 for (unsigned int i = 0; i < eos5varlist.size(); ++i) {
1201 HE5Var he5v = eos5varlist.at(i);
1202 if (1 == he5v.dim_list.size()) {
1203 HE5Dim he5d = he5v.dim_list.at(0);
1204 string dimname;
1205 dimname = eos5typestr + groupname + fslash_str + he5d.name;
1206 string varname; // using the new var name format
1207 varname = eos5typestr + groupname + fslash_str + he5v.name;
1208 mapret = dnamesgeo1dvnames.insert(pair<string, string>(dimname, varname));
1209
1210 // If another geo field already shares the same dimname, we need to
1211 // disqualify this geofield as the coordinate variable since it is not
1212 // unique anymore.
1213 if (false == mapret.second) nocvdimnames.insert(dimname);
1214 }
1215 }
1216
1217 // Manage the coordinate variables. We only want to leave fields that uniquely hold
1218 // the dimension name to be the possible cv candidate.
1219 for (auto itset = nocvdimnames.begin(); itset != nocvdimnames.end(); ++itset)
1220 dnamesgeo1dvnames.erase(*itset);
1221}
1222
1223// Adjust variable names after obtain the parsing information.
1225{
1226
1227 BESDEBUG("h5", "Coming to Adjust_Var_NewName_After_Parsing"<<endl);
1228 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1229 Obtain_Var_NewName(*irv);
1230 }
1231}
1232
1233void EOS5File::Obtain_Var_NewName(Var *var)
1234{
1235
1236 BESDEBUG("h5", "Coming to Obtain_Var_NewName"<<endl);
1237 string fslash_str = "/";
1238 string eos5typestr = "";
1239
1240 EOS5Type vartype = Get_Var_EOS5_Type(var);
1241
1242 // Actually the newname is used to check if the we have the existing
1243 // third dimension coordinate variable. To avoid the check of
1244 // fullpath again, we will make newname to have the unique information
1245 // in the path to identify the objects(Essentially "HDFEOS" is removed).
1246 switch (vartype) {
1247 case GRID: {
1248 eos5typestr = "/GRIDS/";
1249 string eos5_groupname = Obtain_Var_EOS5Type_GroupName(var, vartype);
1250#if 0
1251 // var->newname = ((1 == num_grids)?var->name:
1252 // eos5typestr + eos5_groupname + fslash_str + var->name);
1253#endif
1254 var->newname = eos5typestr + eos5_groupname + fslash_str + var->name;
1255 }
1256 break;
1257
1258 case SWATH: {
1259 eos5typestr = "/SWATHS/";
1260 string eos5_groupname = Obtain_Var_EOS5Type_GroupName(var, vartype);
1261#if 0
1262 // var->newname = ((1 == num_swaths)?var->name:
1263 // eos5typestr + eos5_groupname + fslash_str + var->name);
1264#endif
1265 var->newname = eos5typestr + eos5_groupname + fslash_str + var->name;
1266 }
1267 break;
1268 case ZA: {
1269 eos5typestr = "/ZAS/";
1270 string eos5_groupname = Obtain_Var_EOS5Type_GroupName(var, vartype);
1271#if 0
1272 // var->newname = ((1 == num_zas)?var->name:
1273 // eos5typestr + eos5_groupname + fslash_str + var->name);
1274#endif
1275 var->newname = eos5typestr + eos5_groupname + fslash_str + var->name;
1276 }
1277 break;
1278 case OTHERVARS: {
1279 string eos5infopath = "/HDFEOS INFORMATION";
1280 if (var->fullpath.size() > eos5infopath.size()) {
1281 if (eos5infopath == var->fullpath.substr(0, eos5infopath.size())) var->newname = var->name;
1282 }
1283 else
1284 var->newname = var->fullpath;
1285 }
1286 break;
1287 default:
1288 throw1("Non-supported EOS type");
1289 }
1290}
1291
1292// Get the HDF-EOS5 type: The type is either grids, swaths or zonal average
1293EOS5Type EOS5File::Get_Var_EOS5_Type(Var* var)
1294{
1295
1296 BESDEBUG("h5", "Coming to Get_Var_EOS5_Type"<<endl);
1297
1298 string EOS5GRIDPATH = "/HDFEOS/GRIDS";
1299 string EOS5SWATHPATH = "/HDFEOS/SWATHS";
1300 string EOS5ZAPATH = "/HDFEOS/ZAS";
1301
1302 if (var->fullpath.size() >= EOS5GRIDPATH.size()) {
1303 if (EOS5GRIDPATH == var->fullpath.substr(0, EOS5GRIDPATH.size())) return GRID;
1304 }
1305 if (var->fullpath.size() >= EOS5SWATHPATH.size()) {
1306 if (EOS5SWATHPATH == var->fullpath.substr(0, EOS5SWATHPATH.size())) return SWATH;
1307 }
1308 if (var->fullpath.size() >= EOS5ZAPATH.size()) {
1309 if (EOS5ZAPATH == var->fullpath.substr(0, EOS5ZAPATH.size())) return ZA;
1310 }
1311 return OTHERVARS;
1312
1313}
1314
1315// Add dimension information from the parseing info.
1317{
1318
1319 BESDEBUG("h5", "Coming to Add_Dim_Name"<<endl);
1320 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1321 Obtain_Var_Dims(*irv, strmeta_info);
1322#if 0
1323 for (auto ird = (*irv)->dims.begin();
1324 ird != (*irv)->dims.end();++ird) {
1325 cerr<<"dim name right after change "<<(*ird)->newname <<endl;
1326 }
1327#endif
1328
1329 }
1330}
1331
1332// CHECK if finding the same variables from the parser.
1333bool EOS5File::Obtain_Var_Dims(Var *var, HE5Parser * strmeta_info)
1334{
1335
1336 BESDEBUG("h5", "Coming to Obtain_Var_Dims"<<endl);
1337 string varname_from_parser = "";
1338 EOS5Type vartype = Get_Var_EOS5_Type(var);
1339
1340 if (GRID == vartype) {
1341
1342 auto num_grids = (int)(strmeta_info->grid_list.size());
1343
1344 for (int i = 0; i < num_grids; ++i) {
1345 HE5Grid he5g = strmeta_info->grid_list.at(i);
1346 if (he5g.name == Obtain_Var_EOS5Type_GroupName(var, vartype)) {
1347 EOS5CFGrid *eos5cfgrid = (this->eos5cfgrids)[i];
1348 bool var_is_parsed = Set_Var_Dims(eos5cfgrid, var, he5g.data_var_list, he5g.name, num_grids, GRID);
1349 if (false == var_is_parsed) {
1350 map<hsize_t, string> dimsizes_to_dimnames = eos5cfgrid->dimsizes_to_dimnames;
1351 // Check if this grid includes data fields(variables) that don't have any dimension names.
1352 // This rarely happens. But we do find one NASA Aura product that has this problem. Although
1353 // this has been fixed, we should anticipiate that the similar problem may happen in the future.
1354 // So check here to avoid the potential problems. KY 2012-1-9
1355 Set_NonParse_Var_Dims(eos5cfgrid, var, dimsizes_to_dimnames, num_grids, vartype);
1356 }
1357 }
1358 }
1359
1360 }
1361 else if (SWATH == vartype) {
1362
1363 auto num_swaths = (int)(strmeta_info->swath_list.size());
1364
1365 for (int i = 0; i < num_swaths; ++i) {
1366
1367 HE5Swath he5s = strmeta_info->swath_list.at(i);
1368
1369 if (he5s.name == Obtain_Var_EOS5Type_GroupName(var, vartype)) {
1370
1371 EOS5CFSwath *eos5cfswath = (this->eos5cfswaths)[i];
1372
1373 bool var_is_parsed = true;
1374 int swath_fieldtype_flag = Check_EOS5Swath_FieldType(var);
1375 if (1 == swath_fieldtype_flag)
1376 var_is_parsed = Set_Var_Dims(eos5cfswath, var, he5s.geo_var_list, he5s.name, num_swaths, SWATH);
1377 else if (0 == swath_fieldtype_flag)
1378 var_is_parsed = Set_Var_Dims(eos5cfswath, var, he5s.data_var_list, he5s.name, num_swaths, SWATH);
1379 else
1380 // Neither Geo nor Data(For example, added by the augmentation tool)
1381 var_is_parsed = false;
1382
1383 if (false == var_is_parsed) {
1384 map<hsize_t, string> dimsizes_to_dimnames = eos5cfswath->dimsizes_to_dimnames;
1385 Set_NonParse_Var_Dims(eos5cfswath, var, dimsizes_to_dimnames, num_swaths, vartype);
1386 }
1387 } // end of inner if
1388 } // end of for
1389 } // end of else if
1390
1391 else if (ZA == vartype) {
1392
1393 auto num_zas = (int)(strmeta_info->za_list.size());
1394
1395 for (int i = 0; i < num_zas; ++i) {
1396 HE5Za he5z = strmeta_info->za_list.at(i);
1397 if (he5z.name == Obtain_Var_EOS5Type_GroupName(var, vartype)) {
1398 EOS5CFZa *eos5cfza = (this->eos5cfzas)[i];
1399 bool var_is_parsed = Set_Var_Dims(eos5cfza, var, he5z.data_var_list, he5z.name, num_zas, ZA);
1400 if (false == var_is_parsed) {
1401 map<hsize_t, string> dimsizes_to_dimnames = eos5cfza->dimsizes_to_dimnames;
1402 Set_NonParse_Var_Dims(eos5cfza, var, dimsizes_to_dimnames, num_zas, vartype);
1403 }
1404 }
1405 }
1406 }
1407 return false;
1408}
1409
1410// Set dimension info.(dimension names and sizes) to variables.
1411template<class T>
1412bool EOS5File::Set_Var_Dims(T* eos5data, Var *var, vector<HE5Var> &he5var, const string& groupname, int num_groups,
1413 EOS5Type eos5type)
1414{
1415
1416 BESDEBUG("h5", "Coming to Set_Var_Dims"<<endl);
1417
1418 bool is_parsed = false;
1419 string eos5typestr = "";
1420 string fslash_str = "/";
1421
1422 if (GRID == eos5type)
1423 eos5typestr = "/GRIDS/";
1424 else if (SWATH == eos5type)
1425 eos5typestr = "/SWATHS/";
1426 else if (ZA == eos5type)
1427 eos5typestr = "/ZAS/";
1428 else
1429 throw1("Unsupported HDF-EOS5 type, this type is not swath, grid or zonal average");
1430
1431 for (unsigned int i = 0; i < he5var.size(); i++) {
1432
1433 HE5Var he5v = he5var.at(i);
1434
1435 if (he5v.name == var->name) {
1436 if (he5v.dim_list.size() != var->dims.size())
1437 throw2("Number of dimensions don't match with the structmetadata for variable ", var->name);
1438 is_parsed = true;
1439
1440 // Some variables have the same dim. names shared. For examples, we
1441 // see variables that have int foo[nlevels][nlevels]. To generate the CVs,
1442 // we have to make the dimension name unique for one variable. So we will
1443 // change the dimension names. The variable for the same example will be
1444 // int foo[nlevels][nlevels_1]. Note this is not required by CF conventions.
1445 // This is simply due to the missing of the third coordinate variable for some
1446 // NASA products. Another way is to totally ignore this kind of variables which
1447 // we will wait for users' responses.
1448
1449 // Here is the killer, if different dim. names share the same size,
1450 // Currently there are no ways to know which dimension name is corresponding to
1451 // which size. HDF-EOS model gives too much freedom to users. The DimList in
1452 // the StructMetadata doesn't reflect the order at all. See two example files
1453 // CH4 in TES-Aura_L3-CH4_r0000010410_F01_07.he5 and NO2DayColumn in
1454 // HIRDLS-Aura_L3SCOL_v06-00-00-c02_2005d022-2008d077.he5.
1455 // Fortunately it seems that it doesn't matter for us to make the mapping from
1456 // dimension names to coordinate variables.
1457 // KY 2012-1-10
1458
1459 // Dimension list of some OMI level 2 products doesn't include all dimension name and size
1460 // pairs. For example, Latitude[1644][60]. We have no way to find the dimension name of
1461 // the dimension with the size of 1644. The dimension name list of the variable also
1462 // includes the wrong dimension name. In this case, a dimension with the dimension size =1
1463 // is allocated in the latitude's dimension list. The latest version still has this bug.
1464 // To serve this kind of files, we create a fakedim name for the unmatched size.
1465 // KY 2012-1-13
1466
1467 set<hsize_t> dimsize_have_name_set;
1468 pair<set<hsize_t>::iterator, bool> setret1;
1469 set<string> thisvar_dimname_set;
1470 pair<set<string>::iterator, bool> setret2;
1471
1472 for (unsigned int j = 0; j < he5v.dim_list.size(); j++) {
1473 HE5Dim he5d = he5v.dim_list.at(j);
1474 for (auto ird = var->dims.begin(); ird != var->dims.end(); ++ird) {
1475
1476 if ((hsize_t) (he5d.size) == (*ird)->size) {
1477 // This will assure that the same size dims be assigned to different dims
1478 if ("" == (*ird)->name) {
1479 string dimname_candidate = eos5typestr + groupname + fslash_str + he5d.name;
1480 setret2 = thisvar_dimname_set.insert(dimname_candidate);
1481 if (true == setret2.second) {
1482 (*ird)->name = dimname_candidate;
1483 // Should check in the future if the newname may cause potential inconsistency. KY:2012-3-9
1484 (*ird)->newname = (num_groups == 1) ? he5d.name : (*ird)->name;
1485 eos5data->vardimnames.insert((*ird)->name);
1486 // Since there is no way to figure out the unlimited dimension info. of an individual variable
1487 // from the dimension list. Here we just provide the dimnames to unlimited dimension mapping
1488 // based on the variable mapping. KY 2016-02-18
1489 eos5data->dimnames_to_unlimited[(*ird)->name] = (*ird)->unlimited_dim;
1490 }
1491 }
1492 }
1493 }
1494 } // for (unsigned int j=0; j<he5v.dim_list.size();j++)
1495
1496 // We have to go through the dimension list of this variable again to assure that every dimension has a name.
1497 // This is how that FakeDim is added. We still need it just in case. KY 2017-10-19
1498 for (auto ird = var->dims.begin(); ird != var->dims.end(); ++ird) {
1499 if ("" == (*ird)->name)
1500 Create_Unique_DimName(eos5data, thisvar_dimname_set, *ird, num_groups, eos5type);
1501 }
1502 } // "if (he5v.name == var->name) "
1503 } // "for (unsigned int i = 0; i < he5var.size(); i++)"
1504 return is_parsed;
1505}
1506
1507// Create unique dimension names. Se the comments below.
1508template<class T>
1509void EOS5File::Create_Unique_DimName(T*eos5data, set<string>& thisvar_dimname_set, Dimension *dim, int num_groups,
1510 EOS5Type eos5type)
1511{
1512
1513 BESDEBUG("h5", "Coming to Create_Unique_DimName"<<endl);
1514 map<hsize_t, string>::iterator itmap1;
1515 map<string, hsize_t>::iterator itmap2;
1516 pair<set<string>::iterator, bool> setret2;
1517 itmap1 = (eos5data->dimsizes_to_dimnames).find(dim->size);
1518
1519 // Even if we find this dimension matches the dimsizes_to_dimnames map, we have to check if the dimension
1520 // name has been used for this size. This is to make sure each dimension has a unique name in a variable.
1521 // For example, float foo[100][100] can be float foo[nlevels = 100][nlevels_1 = 100].
1522 // Step 1: Check if there is a dimension name that matches the size
1523
1524 if (itmap1 != (eos5data->dimsizes_to_dimnames).end()) {
1525 string dimname_candidate = (eos5data->dimsizes_to_dimnames)[dim->size];
1526
1527 // First check local var dimname set
1528 setret2 = thisvar_dimname_set.insert(dimname_candidate);
1529
1530 if (false == setret2.second) {
1531
1532 // Will see if other dimension names have this size
1533 bool match_some_dimname = Check_All_DimNames(eos5data, dimname_candidate, dim->size);
1534
1535 if (false == match_some_dimname) {
1536
1537 // dimname_candidate is updated.
1538 Get_Unique_Name(eos5data->vardimnames, dimname_candidate);
1539 thisvar_dimname_set.insert(dimname_candidate);
1540
1541 // Finally generate a new dimension(new dim. name with a size);Update all information
1542 Insert_One_NameSizeMap_Element2(eos5data->dimnames_to_dimsizes, eos5data->dimnames_to_unlimited,
1543 dimname_candidate, dim->size, dim->unlimited_dim);
1544 eos5data->dimsizes_to_dimnames.insert(pair<hsize_t, string>(dim->size, dimname_candidate));
1545 eos5data->dimnames.push_back(dimname_candidate);
1546 }
1547 }
1548
1549 // The final dimname_candidate(may be updated) should be assigned to the name of this dimension
1550 dim->name = dimname_candidate;
1551 if (num_groups > 1)
1552 dim->newname = dim->name;
1553 else {
1554 string dname = HDF5CFUtil::obtain_string_after_lastslash(dim->name);
1555 if ("" == dname)
1556 throw3("The dimension name ", dim->name, " of the variable is not right");
1557 else
1558 dim->newname = dname;
1559 }
1560 }
1561
1562 else { // No dimension names match or close to march this dimension name, we will create a fakedim.
1563 // Check Add_One_FakeDim_Name in HDF5CF.cc Fakedimname must be as a string reference.
1564 string Fakedimname = Create_Unique_FakeDimName(eos5data, eos5type);
1565 thisvar_dimname_set.insert(Fakedimname);
1566
1567 // Finally generate a new dimension(new dim. name with a size);Update all information
1568 Insert_One_NameSizeMap_Element2(eos5data->dimnames_to_dimsizes, eos5data->dimnames_to_unlimited, Fakedimname,
1569 dim->size, dim->unlimited_dim);
1570 eos5data->dimsizes_to_dimnames.insert(pair<hsize_t, string>(dim->size, Fakedimname));
1571 eos5data->dimnames.push_back(Fakedimname);
1572 dim->name = Fakedimname;
1573 if (num_groups > 1)
1574 dim->newname = dim->name;
1575 else {
1576 string dname = HDF5CFUtil::obtain_string_after_lastslash(dim->name);
1577 if ("" == dname)
1578 throw3("The dimension name ", dim->name, " of the variable is not right");
1579 else
1580 dim->newname = dname;
1581 }
1582 }
1583}
1584
1585// Check all dim. names to see if this dim. size is used by another dim. name.
1586template<class T>
1587bool EOS5File::Check_All_DimNames(T* eos5data, string& dimname, hsize_t dimsize)
1588{
1589
1590 BESDEBUG("h5", "Coming to Check_All_DimNames"<<endl);
1591 bool ret_flag = false;
1592 for (map<string, hsize_t>::iterator im = eos5data->dimnames_to_dimsizes.begin();
1593 im != eos5data->dimnames_to_dimsizes.end(); ++im) {
1594 // dimname must not be the same one since the same one is rejected.
1595 if (dimsize == (*im).second && dimname != (*im).first) {
1596 dimname = (*im).first;
1597 ret_flag = true;
1598 break;
1599 }
1600 }
1601 return ret_flag;
1602}
1603
1604// Get a unique name.
1605void EOS5File::Get_Unique_Name(set<string> & nameset, string& dimname_candidate)
1606{
1607
1608 BESDEBUG("h5", "Coming to Get_Unique_Name"<<endl);
1609 int clash_index = 1;
1610 string temp_clashname = dimname_candidate + '_';
1611 HDF5CFUtil::gen_unique_name(temp_clashname, nameset, clash_index);
1612 dimname_candidate = temp_clashname;
1613}
1614
1615// We may need to generate a unique "fake" dim. name for dimensions that don't have any dimension names.
1616template<class T>
1617string EOS5File::Create_Unique_FakeDimName(T*eos5data, EOS5Type eos5type)
1618{
1619
1620 BESDEBUG("h5", "Coming to Create_Unique_FakeDimName"<<endl);
1621 string fslash_str = "/";
1622 string eos5typestr;
1623 if (GRID == eos5type)
1624 eos5typestr = "/GRIDS/";
1625 else if (SWATH == eos5type)
1626 eos5typestr = "/SWATHS/";
1627 else if (ZA == eos5type)
1628 eos5typestr = "/ZAS/";
1629 else
1630 throw1("Unsupported HDF-EOS5 type, this type is not swath, grid or zonal average");
1631
1632 stringstream sfakedimindex;
1633 sfakedimindex << eos5data->addeddimindex;
1634 string fakedimstr = "FakeDim";
1635 string added_dimname = eos5typestr + eos5data->name + fslash_str + fakedimstr + sfakedimindex.str();
1636
1637 pair<set<string>::iterator, bool> setret;
1638 setret = eos5data->vardimnames.insert(added_dimname);
1639 if (false == setret.second) Get_Unique_Name(eos5data->vardimnames, added_dimname);
1640 eos5data->addeddimindex = eos5data->addeddimindex + 1;
1641 return added_dimname;
1642}
1643
1644// Obtain the group name this variable belongs.
1645string EOS5File::Obtain_Var_EOS5Type_GroupName(const Var*var, EOS5Type eos5type) const
1646{
1647
1648 BESDEBUG("h5", "Coming to Obtain_Var_EOS5Type_GroupName"<<endl);
1649 string EOS5GRIDPATH = "/HDFEOS/GRIDS";
1650 string EOS5SWATHPATH = "/HDFEOS/SWATHS";
1651 string EOS5ZAPATH = "/HDFEOS/ZAS";
1652 size_t eostypename_start_pos = 0;
1653 size_t eostypename_end_pos;
1654 string groupname;
1655
1656 // The fullpath is like "HDFEOS/GRIDS/Temp/Data Fields/etc
1657 // To get "Temp", we obtain the position of "T" and the position of "p"
1658 // and then generate a substr.
1659
1660 if (GRID == eos5type)
1661 eostypename_start_pos = EOS5GRIDPATH.size() + 1;
1662 else if (SWATH == eos5type)
1663 eostypename_start_pos = EOS5SWATHPATH.size() + 1;
1664 else if (ZA == eos5type)
1665 eostypename_start_pos = EOS5ZAPATH.size() + 1;
1666 else
1667 throw2("Non supported eos5 type for var ", var->fullpath);
1668
1669 eostypename_end_pos = var->fullpath.find('/', eostypename_start_pos) - 1;
1670 groupname = var->fullpath.substr(eostypename_start_pos, eostypename_end_pos - eostypename_start_pos + 1);
1671
1672 BESDEBUG("h5", "In Obtain_Var_EOS5Type_GroupName(), the groupname is "<<groupname << endl);
1673
1674 return groupname;
1675}
1676
1677// Check whether this field belongs to "Geolocation Fields" or "Data Fields"
1678int EOS5File::Check_EOS5Swath_FieldType(const Var*var) const
1679{
1680
1681 string geofield_relative_path = "/Geolocation Fields/" + var->name;
1682 string datafield_relative_path = "/Data Fields/" + var->name;
1683
1684 int tempflag = -1;
1685
1686 if (var->fullpath.size() > datafield_relative_path.size()) {
1687 size_t field_pos_in_full_path = var->fullpath.size() - datafield_relative_path.size();
1688 if (var->fullpath.rfind(datafield_relative_path, field_pos_in_full_path) != string::npos) tempflag = 0;
1689 }
1690
1691 if (tempflag != 0 && (var->fullpath.size() > geofield_relative_path.size())) {
1692 size_t field_pos_in_full_path = var->fullpath.size() - geofield_relative_path.size();
1693 if (var->fullpath.rfind(geofield_relative_path, field_pos_in_full_path) != string::npos) tempflag = 1;
1694 }
1695 return tempflag;
1696}
1697
1698// An error will be thrown if we find a dimension size that doesn't match any dimension name
1699// in this EOS5 group.
1700template<class T>
1701void EOS5File::Set_NonParse_Var_Dims(T*eos5data, Var* var, const map<hsize_t, string>& /*dimsizes_to_dimnames*/,
1702 int num_groups, EOS5Type eos5type)
1703{
1704
1705 BESDEBUG("h5", "Coming to Set_NonParse_Var_Dims"<<endl);
1706 map<hsize_t, string>::iterator itmap;
1707 set<string> thisvar_dimname_set;
1708
1709 for (auto ird = var->dims.begin(); ird != var->dims.end(); ++ird) {
1710 if ("" == (*ird)->name)
1711 Create_Unique_DimName(eos5data, thisvar_dimname_set, *ird, num_groups, eos5type);
1712 else
1713 throw5("The dimension name ", (*ird)->name, " of the variable ", var->name, " is not right");
1714 }
1715}
1716
1717// Aura files don't use the CF attribute names for bunch of attributes. We need to make it right.
1719{
1720
1721 BESDEBUG("h5", "Coming to Check_Aura_Product_Status"<<endl);
1722 // Aura files will put an attribute called InStrumentName under /HDFEOS/ADDITIONAL/FILE_ATTRIBUTES
1723 // We just need to check that attribute.
1724 string eos5_fattr_group_name = "/HDFEOS/ADDITIONAL/FILE_ATTRIBUTES";
1725 string instrument_attr_name = "InstrumentName";
1726
1727 // Check if this file is an aura file
1728 for (auto irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1729 if (eos5_fattr_group_name == (*irg)->path) {
1730 for (auto ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end(); ++ira) {
1731 if (instrument_attr_name == (*ira)->name) {
1732 Retrieve_H5_Attr_Value(*ira, (*irg)->path);
1733 string attr_value((*ira)->value.begin(), (*ira)->value.end());
1734 if ("OMI" == attr_value) {
1735 this->isaura = true;
1736 this->aura_name = OMI;
1737 }
1738 else if ("MLS Aura" == attr_value) {
1739 this->isaura = true;
1740 this->aura_name = MLS;
1741 }
1742 else if ("TES" == attr_value) {
1743 this->isaura = true;
1744 this->aura_name = TES;
1745 }
1746 else if ("HIRDLS" == attr_value) {
1747 this->isaura = true;
1748 this->aura_name = HIRDLS;
1749 }
1750 break;
1751 }
1752 }
1753 }
1754 }
1755
1756 // Assign EOS5 to CF MAP values for Aura files
1757 if (true == this->isaura) {
1758 eos5_to_cf_attr_map["FillValue"] = "_FillValue";
1759 eos5_to_cf_attr_map["MissingValue"] = "missing_value";
1760 eos5_to_cf_attr_map["Units"] = "units";
1761 eos5_to_cf_attr_map["Offset"] = "add_offset";
1762 eos5_to_cf_attr_map["ScaleFactor"] = "scale_factor";
1763 eos5_to_cf_attr_map["ValidRange"] = "valid_range";
1764 eos5_to_cf_attr_map["Title"] = "title";
1765 }
1766
1767}
1768
1769// Handle Coordinate variables
1771{
1772
1773 BESDEBUG("h5", "Coming to Handle_CVar()"<<endl);
1774
1775 // If this file is augmented.
1776 bool is_augmented = Check_Augmentation_Status();
1777
1778#if 0
1779 if(is_augmented) cerr<<"The file is augmented "<<endl;
1780 else cerr<<"The file is not augmented "<<endl;
1781#endif
1782
1783 // Handle coordinate variables for grids.
1784 if (this->eos5cfgrids.empty() == false)
1785 Handle_Grid_CVar(is_augmented);
1786 if (this->eos5cfswaths.empty() == false)
1787 Handle_Swath_CVar(is_augmented);
1788 if (this->eos5cfzas.empty() == false)
1789 Handle_Za_CVar(is_augmented);
1790
1791#if 0
1792 for (auto irv = this->cvars.begin();
1793 irv != this->cvars.end(); irv++) {
1794 cerr<<"EOS5CVar name "<<(*irv)->name <<endl;
1795 cerr<<"EOS5CVar dimension name "<< (*irv)->cfdimname <<endl;
1796 cerr<<"EOS5CVar new name "<<(*irv)->newname <<endl;
1797 cerr<<"EOS5CVar type is "<<(*irv)->cvartype <<endl;
1798//cerr<<"EOS5CVar dtype is "<<(*irv)->dtype <<endl;
1799 }
1800#endif
1801
1802}
1803
1804// Handle Grid Coordinate variables
1805void EOS5File::Handle_Grid_CVar(bool is_augmented)
1806{
1807
1808 BESDEBUG("h5", "Coming to Handle_Grid_CVar"<<endl);
1809 if (true == is_augmented) {
1810 // Create latitude/longitude based on the first XDim and YDim
1811 Handle_Augmented_Grid_CVar();
1812 }
1813 else {
1814 Remove_MultiDim_LatLon_EOS5CFGrid();
1815 // If the grid size is 0, it must be a Grid file that cannot be handled
1816 // with the CF option, simply return with handling any coordinate variables.
1817 if (this->eos5cfgrids.empty()) return;
1818 if (1 == this->eos5cfgrids.size())
1819 Handle_Single_Nonaugment_Grid_CVar((this->eos5cfgrids)[0]);
1820 else
1821 Handle_Multi_Nonaugment_Grid_CVar();
1822 }
1823}
1824
1825// Check if this file is augmented. The current augmentation tool will
1826// add extra variables for every EOS5 object. This function will check
1827// if that is the case.
1828bool EOS5File::Check_Augmentation_Status()
1829{
1830
1831 BESDEBUG("h5", "Coming to Check_Augmentation_Status()"<<endl);
1832 bool aug_status = false;
1833 int num_aug_eos5grp = 0;
1834
1835 for (auto irg = this->eos5cfgrids.begin(); irg != this->eos5cfgrids.end(); ++irg) {
1836 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1837 bool is_augmented = Check_Augmented_Var_Candidate(*irg, *irv, GRID);
1838 if (true == is_augmented) {
1839 num_aug_eos5grp++;
1840 break;
1841 }
1842 }
1843 }
1844
1845 for (auto irg = this->eos5cfswaths.begin(); irg != this->eos5cfswaths.end(); ++irg) {
1846 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1847 bool is_augmented = Check_Augmented_Var_Candidate(*irg, *irv, SWATH);
1848 if (true == is_augmented) {
1849 num_aug_eos5grp++;
1850 break;
1851 }
1852
1853 }
1854 }
1855
1856 for (auto irg = this->eos5cfzas.begin(); irg != this->eos5cfzas.end(); ++irg) {
1857 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1858 bool is_augmented = Check_Augmented_Var_Candidate(*irg, *irv, ZA);
1859 if (true == is_augmented) {
1860 num_aug_eos5grp++;
1861 break;
1862 }
1863 }
1864 }
1865
1866 int total_num_eos5grp = this->eos5cfgrids.size() + this->eos5cfswaths.size() + this->eos5cfzas.size();
1867
1868#if 0
1869//cerr<< "total_num_eos5grp "<<total_num_eos5grp <<endl;
1870//"h5","num_aug_eos5grp "<< num_aug_eos5grp <<endl;
1871#endif
1872
1873 if (num_aug_eos5grp == total_num_eos5grp) aug_status = true;
1874 return aug_status;
1875
1876}
1877
1878// This method is not used. Still keep it now since it may be useful in the future. KY 2012-3-09
1879// Don't remove the #if 0 #endif block.
1880#if 0
1881bool EOS5File::Check_Augmented_Var_Attrs(Var *var) {
1882
1883 // We will check whether the attribute "CLASS" and the attribute "REFERENCE_LIST" exist.
1884 // For the attribute "CLASS", we would like to check if the value is "DIMENSION_SCALE".
1885 bool has_dimscale_class = false;
1886 bool has_reflist = false;
1887 for (auto ira = var->attrs.begin(); ira != var->attrs.end(); ++ira) {
1888 if ("CLASS" == (*ira)->name) {
1889 Retrieve_H5_Attr_Value(*ira,var->fullpath);
1890 string class_value((*ira)->value.begin(),(*ira)->value.end());
1891 if ("DIMENSION_SCALE"==class_value)
1892 has_dimscale_class = true;
1893 }
1894
1895 if ("REFERENCE_LIST" == (*ira)->name)
1896 has_reflist = true;
1897 if (true == has_reflist && true == has_dimscale_class)
1898 break;
1899 }
1900
1901 if (true == has_reflist && true == has_dimscale_class)
1902 return true;
1903 else
1904 return false;
1905
1906}
1907#endif
1908
1909// Check if the variable candidate exists for the augmented case.
1910// The augmented variables have path like /HDFEOS/GRIDS/HIRDLS/nTimes
1911// The general HDF-EOS5 variables have path like /HDFEOS/GRIDS/HIRDLS/Data Fields/Times.
1912// So if we find the var name is the same as the string stripped from /HDFEOS/GRIDS/HIRDLS,
1913// then this file is augmented.
1914// Hope that no other hybrid-HDFEOS5 files fall to this category.
1915template<class T>
1916bool EOS5File::Check_Augmented_Var_Candidate(T *eos5data, Var *var, EOS5Type eos5type)
1917{
1918
1919 BESDEBUG("h5", "Coming to Check_Augmented_Var_Candidate"<<endl);
1920 bool augmented_var = false;
1921
1922 string EOS5DATAPATH = "";
1923 if (GRID == eos5type)
1924 EOS5DATAPATH = "/HDFEOS/GRIDS/";
1925 else if (ZA == eos5type)
1926 EOS5DATAPATH = "/HDFEOS/ZAS/";
1927 else if (SWATH == eos5type)
1928 EOS5DATAPATH = "/HDFEOS/SWATHS/";
1929 else
1930 throw1("Non supported EOS5 type");
1931
1932 string fslash_str = "/";
1933 string THIS_EOS5DATAPATH = EOS5DATAPATH + eos5data->name + fslash_str;
1934
1935 // Match the EOS5 type
1936 if (eos5type == Get_Var_EOS5_Type(var)) {
1937 string var_eos5data_name = Obtain_Var_EOS5Type_GroupName(var, eos5type);
1938 // Match the EOS5 group name
1939 if (var_eos5data_name == eos5data->name) {
1940 if (var->fullpath.size() > THIS_EOS5DATAPATH.size()) {
1941 // Obtain the var name from the full path
1942 string var_path_after_eos5dataname = var->fullpath.substr(THIS_EOS5DATAPATH.size());
1943 // Match the variable name
1944 if (var_path_after_eos5dataname == var->name) augmented_var = true;
1945 }
1946 }
1947 }
1948
1949 return augmented_var;
1950
1951}
1952
1953// Handle augmented grid coordinate variables.
1954void EOS5File::Handle_Augmented_Grid_CVar()
1955{
1956 BESDEBUG("h5", "Coming to Handle_Augmented_Grid_CVar()"<<endl);
1957 for (auto irv = this->eos5cfgrids.begin(); irv != this->eos5cfgrids.end(); ++irv)
1958 Handle_Single_Augment_CVar(*irv, GRID);
1959}
1960
1961// Handle the coordinate variables for the single HDF-EOS5 objects(grid,swath,zonal average) for an augmented file
1962template<class T>
1963void EOS5File::Handle_Single_Augment_CVar(T* cfeos5data, EOS5Type eos5type)
1964{
1965
1966 BESDEBUG("h5", "Coming to Handle_Single_Augment_CVar()"<<endl);
1967 set<string> tempvardimnamelist;
1968 tempvardimnamelist = cfeos5data->vardimnames;
1969 set<string>::iterator its;
1970 for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its) {
1971 for (auto irv = this->vars.begin(); irv != this->vars.end();) {
1972
1973 bool is_augmented = Check_Augmented_Var_Candidate(cfeos5data, *irv, eos5type);
1974
1975 if (true == is_augmented) {
1976
1977 // Since we have already checked if this file is augmented or not, we can safely
1978 // compare the dimension name with the var name now.
1979 string tempdimname = HDF5CFUtil::obtain_string_after_lastslash(*its);
1980
1981 // The added variable name is always the same as the dimension name.
1982 if (tempdimname == (*irv)->name) {
1983
1984 //Find it, create a coordinate variable.
1985 auto EOS5cvar = new EOS5CVar(*irv);
1986
1987 // Still keep the original dimension name to avoid the nameclashing when
1988 // one grid and one swath and one za occur in the same file
1989 EOS5cvar->cfdimname = *its;
1990 EOS5cvar->cvartype = CV_EXIST;
1991 EOS5cvar->eos_type = eos5type;
1992
1993 // Save this cv to the cv vector
1994 this->cvars.push_back(EOS5cvar);
1995
1996 // Remove this var from the var vector since it becomes a cv.
1997 delete (*irv);
1998 irv = this->vars.erase(irv);
1999 }
2000 else {
2001 ++irv;
2002 }
2003 } // "if (true == is_augmented)"
2004 else {
2005 ++irv;
2006 }
2007 } // end of for (auto irv = this->vars.begin();....
2008 } // end of for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its)
2009
2010 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
2011 its = tempvardimnamelist.find((*irv)->cfdimname);
2012 if (its != tempvardimnamelist.end()) tempvardimnamelist.erase(its);
2013 }
2014
2015 if (false == tempvardimnamelist.empty())
2016 throw1("Augmented files still need to provide more coordinate variables");
2017}
2018
2019//Currently we remove HDF-EOS5 grid if we find the latitude/longitude is >2D. This is a big question mark
2020// given some data producers just don't follow the HDF-EOS5 specification to generate the latitude/longitude.
2021// KY 2016-07-12
2022void EOS5File::Remove_MultiDim_LatLon_EOS5CFGrid()
2023{
2024
2025 BESDEBUG("h5", "Coming to Remove_MultiDim_LatLon_EOS5CFGrid()"<<endl);
2026 for (auto irg = this->eos5cfgrids.begin(); irg != this->eos5cfgrids.end();) {
2027
2028 // If number of dimension latitude/longitude is >=2, no cooridnate variables will be generated.
2029 // We will simply remove this grid from the vector eos5cfgrids.
2030 // In the future, we may consider supporting 2D latlon. KY 2012-1-17
2031 // I just find that new OMI level 3 data provide 2D lat/lon for geographic projection data.
2032 // The 2D lat/lon can be condensed to 1D lat/lon, which is the same calculated by the calculation of
2033 // the projection. So I don't remove this OMI grid from the grid list. KY 2012-2-9
2034 // However, I do remove the "Longitude" and "Latitude" fields since "Latitude" and "Longitude"
2035 // can be calculated.
2036
2037 bool irg_erase = false;
2038
2039 if (true == (*irg)->has_2dlatlon) {
2040
2041 if ((true == this->isaura) && (OMI == this->aura_name) && (HE5_GCTP_GEO == (*irg)->eos5_projcode))
2042
2043 { // We need to remove the redundant latitude and longitude fields
2044
2045 string EOS5GRIDPATH = "/HDFEOS/GRIDS/";
2046 string fslash_str = "/";
2047 string THIS_EOS5GRIDPATH = EOS5GRIDPATH + (*irg)->name + fslash_str;
2048 int catch_latlon = 0;
2049
2050 for (auto irv = this->vars.begin(); (irv != this->vars.end()) && (catch_latlon != 2);
2051 ) {
2052 if (GRID == Get_Var_EOS5_Type(*irv) && ((*irv)->fullpath.size() > THIS_EOS5GRIDPATH.size())) {
2053
2054 string var_grid_name = Obtain_Var_EOS5Type_GroupName(*irv, GRID);
2055 if (var_grid_name == (*irg)->name) {
2056 if (("Longitude" == (*irv)->name) || ("Latitude" == (*irv)->name)) {
2057 catch_latlon++;
2058 // Remove this var from the var vector since it becomes a cv.
2059 delete (*irv);
2060 irv = this->vars.erase(irv);
2061 }
2062 else {
2063 ++irv;
2064 }
2065 }
2066 else {
2067 ++irv;
2068 }
2069 }
2070 else {
2071 ++irv;
2072 }
2073 } // "for (auto irv = this->vars.begin() ..."
2074 if (2 == catch_latlon) {
2075 (*irg)->has_nolatlon = true;
2076 (*irg)->has_2dlatlon = false;
2077 }
2078
2079 } // "if ((true == this->isaura) ..."
2080 else { // remove this grid from the eos5cfgrids list.
2081 delete (*irg);
2082 irg = this->eos5cfgrids.erase(irg);
2083 irg_erase = true;
2084 }
2085 } // "if (true == (*irg) ..."
2086
2087 if (false == irg_erase) {
2088 ++irg;
2089 }
2090
2091 } // "for (vector <EOS5CFGrid *>::iterator irg = this->eos5cfgrids.begin() ..."
2092
2093 // Also remove >2d latlon grids.
2094 for (auto irg = this->eos5cfgrids.begin(); irg != this->eos5cfgrids.end();) {
2095
2096 if (true == (*irg)->has_g2dlatlon) {
2097 delete (*irg);
2098 irg = this->eos5cfgrids.erase(irg);
2099 }
2100 else {
2101 ++irg;
2102 }
2103 }
2104}
2105
2106// Handle single nonaugmented grid coordinate variables.
2107void EOS5File::Handle_Single_Nonaugment_Grid_CVar(EOS5CFGrid* cfgrid)
2108{
2109
2110 BESDEBUG("h5", "Coming to Handle_Single_Nonaugment_Grid_CVar()"<<endl);
2111 set<string> tempvardimnamelist;
2112 tempvardimnamelist = cfgrid->vardimnames;
2113
2114 // Handle Latitude and longitude
2115 bool use_own_latlon = false;
2116 if (true == cfgrid->has_1dlatlon)
2117 use_own_latlon = Handle_Single_Nonaugment_Grid_CVar_OwnLatLon(cfgrid, tempvardimnamelist);
2118#if 0
2119 if(use_own_latlon) "h5","using 1D latlon"<<endl;
2120 else "h5","use_own_latlon is false "<<endl;
2121#endif
2122
2123 if (false == use_own_latlon) {
2124 bool use_eos5_latlon = false;
2125 use_eos5_latlon = Handle_Single_Nonaugment_Grid_CVar_EOS5LatLon(cfgrid, tempvardimnamelist);
2126
2127 // If we cannot obtain lat/lon from the HDF-EOS5 library, no need to create other CVs. Simply return.
2128 if (false == use_eos5_latlon) return;
2129 }
2130
2131 // Else handling non-latlon grids
2132 Handle_NonLatLon_Grid_CVar(cfgrid, tempvardimnamelist);
2133
2134}
2135
2136// Handle single nonaugmented grid coordinate variables with its own lat/lon
2137bool EOS5File::Handle_Single_Nonaugment_Grid_CVar_OwnLatLon(const EOS5CFGrid *cfgrid, set<string>& tempvardimnamelist)
2138
2139{
2140
2141 BESDEBUG("h5", "Coming to Handle_Single_Nonaugment_Grid_CVar_OwnLatLon()"<<endl);
2142 set<string>::iterator its;
2143 string EOS5GRIDPATH = "/HDFEOS/GRIDS/";
2144 string fslash_str = "/";
2145 string THIS_EOS5GRIDPATH = EOS5GRIDPATH + cfgrid->name + fslash_str;
2146
2147 // Handle latitude and longitude
2148 bool find_latydim = false;
2149 bool find_lonxdim = false;
2150
2151 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2152 if (GRID == Get_Var_EOS5_Type(*irv) && ((*irv)->fullpath.size() > THIS_EOS5GRIDPATH.size())) {
2153
2154 string var_grid_name = Obtain_Var_EOS5Type_GroupName(*irv, GRID);
2155 if ((var_grid_name == cfgrid->name) && ((*irv)->name == "Latitude")) {
2156
2157 string tempdimname = (((*irv)->dims)[0])->name;
2158
2159 if ("YDim" == HDF5CFUtil::obtain_string_after_lastslash(tempdimname)) {
2160 //Find it, create a coordinate variable.
2161 auto EOS5cvar = new EOS5CVar(*irv);
2162
2163 // Still keep the original dimension name to avoid the nameclashing when
2164 // one grid and one swath and one za occur in the same file
2165 EOS5cvar->cfdimname = tempdimname;
2166 EOS5cvar->cvartype = CV_EXIST;
2167 EOS5cvar->eos_type = GRID;
2168
2169 // Save this cv to the cv vector
2170 this->cvars.push_back(EOS5cvar);
2171
2172 // Remove this var from the var vector since it becomes a cv.
2173 delete (*irv);
2174 this->vars.erase(irv);
2175
2176 // No need to remove back the iterator since it will go out of the loop.
2177 find_latydim = true;
2178 break;
2179 } // <if ("YDim" == HDF5CFUtil::obtain_string_after_lastslash(tempdimname))>
2180 } // <if ((var_grid_name == cfgrid->name) && ((*irv)->name == "Latitude"))>
2181 } // <if (GRID == Get_Var_EOS5_Type(*irv) ...>
2182 } // <for (auto irv = this->vars.begin() ...>
2183
2184 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2185
2186 if (GRID == Get_Var_EOS5_Type(*irv) && ((*irv)->fullpath.size() > THIS_EOS5GRIDPATH.size())) {
2187
2188 string var_grid_name = Obtain_Var_EOS5Type_GroupName(*irv, GRID);
2189
2190 if ((var_grid_name == cfgrid->name) && ((*irv)->name == "Longitude")) {
2191
2192 string tempdimname = (((*irv)->dims)[0])->name;
2193
2194 if ("XDim" == HDF5CFUtil::obtain_string_after_lastslash(tempdimname)) {
2195 //Find it, create a coordinate variable.
2196 auto EOS5cvar = new EOS5CVar(*irv);
2197
2198 // Still keep the original dimension name to avoid the nameclashing when
2199 // one grid and one swath and one za occur in the same file
2200 EOS5cvar->cfdimname = tempdimname;
2201 EOS5cvar->cvartype = CV_EXIST;
2202 EOS5cvar->eos_type = GRID;
2203
2204 // Save this cv to the cv vector
2205 this->cvars.push_back(EOS5cvar);
2206
2207 // Remove this var from the var vector since it becomes a cv.
2208 delete (*irv);
2209 this->vars.erase(irv);
2210 find_lonxdim = true;
2211 break;
2212 } // <if ("XDim" == HDF5CFUtil::obtain_string_after_lastslash(tempdimname))>
2213 } // "if ((var_grid_name == cfgrid->name) && ((*irv)->name == "Longitude"))"
2214 } // <if (GRID == Get_Var_EOS5_Type(*irv) ...>
2215 } // for (auto irv = this->vars.begin() ...
2216
2217 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
2218 its = tempvardimnamelist.find((*irv)->cfdimname);
2219 if (its != tempvardimnamelist.end()) tempvardimnamelist.erase(its);
2220
2221 }
2222
2223#if 0
2224 for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its)
2225 "h5","tempvardim "<<*its <<endl;
2226#endif
2227
2228 return (find_latydim == true && find_lonxdim == true);
2229}
2230
2231// Handle single non-augmented grid latitude/longitude coordinate variables.
2232bool EOS5File::Handle_Single_Nonaugment_Grid_CVar_EOS5LatLon(const EOS5CFGrid *cfgrid, set<string>& tempvardimnamelist)
2233
2234{
2235
2236 BESDEBUG("h5", "Coming to Handle_Single_Nonaugment_Grid_CVar_EOS5LatLon()"<<endl);
2237
2238 // Handle latitude and longitude
2239 bool find_ydim = false;
2240 bool find_xdim = false;
2241 set<string>::iterator its;
2242
2243#if 0
2244 for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its)
2245 cerr<<"dim names "<<(*its) <<endl;
2246#endif
2247
2248 string ydim_full_path;
2249 string xdim_full_path;
2250
2251 if (cfgrid->eos5_projcode != HE5_GCTP_GEO) {
2252
2253 for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its) {
2254 if ("YDim" == HDF5CFUtil::obtain_string_after_lastslash(*its))
2255 ydim_full_path = *its;
2256 else if ("XDim" == HDF5CFUtil::obtain_string_after_lastslash(*its)) xdim_full_path = *its;
2257 }
2258 }
2259
2260
2261 for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end();) {
2262 if ("YDim" == HDF5CFUtil::obtain_string_after_lastslash(*its)) {
2263
2264 // Create EOS5 Latitude CV
2265 auto EOS5cvar = new EOS5CVar();
2266 EOS5cvar->name = "lat";
2267 Create_Added_Var_NewName_FullPath(GRID, cfgrid->name, EOS5cvar->name, EOS5cvar->newname,
2268 EOS5cvar->fullpath);
2269
2270 if (cfgrid->eos5_projcode != HE5_GCTP_GEO) {
2271 EOS5cvar->rank = 2;
2272 EOS5cvar->dtype = H5FLOAT64;
2273 }
2274 else {
2275 EOS5cvar->rank = 1;
2276 EOS5cvar->dtype = H5FLOAT32;
2277 }
2278
2279 auto eos5cvar_dim = new Dimension((hsize_t) cfgrid->ydimsize);
2280 eos5cvar_dim->name = *its;
2281 eos5cvar_dim->newname = (this->eos5cfgrids.size() == 1) ? "YDim" : *its;
2282 EOS5cvar->dims.push_back(eos5cvar_dim);
2283 EOS5cvar->cfdimname = eos5cvar_dim->name;
2284
2285 if (EOS5cvar->rank == 2) {
2286
2287 eos5cvar_dim = new Dimension((hsize_t) cfgrid->xdimsize);
2288 eos5cvar_dim->name = xdim_full_path;
2289 eos5cvar_dim->newname = (this->eos5cfgrids.size() == 1) ? "XDim" : xdim_full_path;
2290 EOS5cvar->dims.push_back(eos5cvar_dim);
2291
2292 }
2293 EOS5cvar->cvartype = CV_LAT_MISS;
2294 EOS5cvar->eos_type = GRID;
2295 EOS5cvar->xdimsize = cfgrid->xdimsize;
2296 EOS5cvar->ydimsize = cfgrid->ydimsize;
2297
2298 //Special parameters for EOS5 Grid
2299 EOS5cvar->point_lower = cfgrid->point_lower;
2300 EOS5cvar->point_upper = cfgrid->point_upper;
2301 EOS5cvar->point_left = cfgrid->point_left;
2302 EOS5cvar->point_right = cfgrid->point_right;
2303 EOS5cvar->eos5_pixelreg = cfgrid->eos5_pixelreg;
2304 EOS5cvar->eos5_origin = cfgrid->eos5_origin;
2305 EOS5cvar->eos5_projcode = cfgrid->eos5_projcode;
2306
2307 for (unsigned int k = 0; k < 13; k++)
2308 EOS5cvar->param[k] = cfgrid->param[k];
2309
2310 EOS5cvar->zone = cfgrid->zone;
2311 EOS5cvar->sphere = cfgrid->sphere;
2312
2313 // Save this cv to the cv vector
2314 this->cvars.push_back(EOS5cvar);
2315 // erase the dimension name from the dimension name set
2316
2317 // This is the right way to make its platform-independent.
2318 tempvardimnamelist.erase(its++);
2319 find_ydim = true;
2320
2321 } // <if ("YDim" == HDF5CFUtil::obtain_string_after_lastslash(*its))>
2322 else if ("XDim" == HDF5CFUtil::obtain_string_after_lastslash(*its)) {
2323
2324 // Create EOS5 Latitude CV
2325 auto EOS5cvar = new EOS5CVar();
2326 EOS5cvar->name = "lon";
2327 Create_Added_Var_NewName_FullPath(GRID, cfgrid->name, EOS5cvar->name, EOS5cvar->newname,
2328 EOS5cvar->fullpath);
2329#if 0
2330 //EOS5cvar->newname = EOS5cvar->name;
2331 //EOS5cvar->fullpath = EOS5cvar->name;
2332#endif
2333 if (cfgrid->eos5_projcode != HE5_GCTP_GEO) {
2334 EOS5cvar->rank = 2;
2335 EOS5cvar->dtype = H5FLOAT64;
2336 }
2337 else {
2338 EOS5cvar->rank = 1;
2339 EOS5cvar->dtype = H5FLOAT32;
2340 }
2341
2342 Dimension* eos5cvar_dim = nullptr;
2343 if (EOS5cvar->rank == 2) {
2344 eos5cvar_dim = new Dimension((hsize_t) cfgrid->ydimsize);
2345#if 0
2346 //eos5cvar_dim->name = EOS5cvar->name;
2347#endif
2348 eos5cvar_dim->name = ydim_full_path;
2349 eos5cvar_dim->newname = (this->eos5cfgrids.size() == 1) ? "YDim" : ydim_full_path;
2350 EOS5cvar->dims.push_back(eos5cvar_dim);
2351 }
2352
2353 eos5cvar_dim = new Dimension((hsize_t) cfgrid->xdimsize);
2354#if 0
2355 //eos5cvar_dim->name = EOS5cvar->name;
2356#endif
2357 eos5cvar_dim->name = *its;
2358 eos5cvar_dim->newname = (this->eos5cfgrids.size() == 1) ? "XDim" : *its;
2359 EOS5cvar->dims.push_back(eos5cvar_dim);
2360 EOS5cvar->cfdimname = eos5cvar_dim->name;
2361
2362 EOS5cvar->cvartype = CV_LON_MISS;
2363 EOS5cvar->eos_type = GRID;
2364 EOS5cvar->xdimsize = cfgrid->xdimsize;
2365 EOS5cvar->ydimsize = cfgrid->ydimsize;
2366
2367 //Special parameters for EOS5 Grid
2368 EOS5cvar->point_lower = cfgrid->point_lower;
2369 EOS5cvar->point_upper = cfgrid->point_upper;
2370 EOS5cvar->point_left = cfgrid->point_left;
2371 EOS5cvar->point_right = cfgrid->point_right;
2372 EOS5cvar->eos5_pixelreg = cfgrid->eos5_pixelreg;
2373 EOS5cvar->eos5_origin = cfgrid->eos5_origin;
2374 EOS5cvar->eos5_projcode = cfgrid->eos5_projcode;
2375 for (unsigned int k = 0; k < 13; k++)
2376 EOS5cvar->param[k] = cfgrid->param[k];
2377 EOS5cvar->zone = cfgrid->zone;
2378 EOS5cvar->sphere = cfgrid->sphere;
2379
2380 // Save this cv to the cv vector
2381 this->cvars.push_back(EOS5cvar);
2382
2383 // erase the dimension name from the dimension name set,platform independent way.
2384 tempvardimnamelist.erase(its++);
2385 find_xdim = true;
2386
2387 } // "else if ("XDim" == HDF5CFUtil::obtain_string_after_lastslash(*its))"
2388 else
2389 ++its;
2390 if (true == find_xdim && true == find_ydim) break;
2391 } // <for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its)>
2392
2393 return (true == find_xdim && true == find_ydim);
2394}
2395
2396// Handle non-latitude/longitude grid coordinate variables.
2397void EOS5File::Handle_NonLatLon_Grid_CVar(EOS5CFGrid *cfgrid, set<string>& tempvardimnamelist)
2398{
2399
2400 // First check if we have existing coordinate variable
2401 set<string>::iterator its;
2402 auto num_dimnames = (int)(tempvardimnamelist.size());
2403 bool has_dimnames = true;
2404
2405 for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its) {
2406 if (cfgrid->dnames_to_1dvnames.find(*its) != cfgrid->dnames_to_1dvnames.end()) {
2407 for (auto irv = this->vars.begin(); has_dimnames && (irv != this->vars.end());) {
2408 // We need to check if this var is a grid and use "newname"
2409 // of var to check the dnames_to_1dvnames since it is
2410 // possible to have name clashings for the "name" of a var.
2411 if (GRID == Get_Var_EOS5_Type(*irv) && (*irv)->newname == (cfgrid->dnames_to_1dvnames)[*its]) {
2412
2413 //Find it, create a coordinate variable.
2414 auto EOS5cvar = new EOS5CVar(*irv);
2415
2416 // Still keep the original dimension name to avoid the nameclashing when
2417 // one grid and one swath and one za occur in the same file
2418 EOS5cvar->cfdimname = *its;
2419 EOS5cvar->cvartype = CV_EXIST;
2420 EOS5cvar->eos_type = GRID;
2421
2422 // Save this cv to the cv vector
2423 this->cvars.push_back(EOS5cvar);
2424
2425 // Remove this var from the var vector since it becomes a cv.
2426 delete (*irv);
2427 irv = this->vars.erase(irv);
2428 num_dimnames--;
2429 if (0 == num_dimnames) has_dimnames = false;
2430 } // if (GRID == Get_Var_EOS5_Type(*irv) ...
2431 else {
2432 ++irv;
2433 }
2434 } // for (auto irv = this->vars.begin(); ...
2435 } // if (cfgrid->dnames_to_1dvnames.find(*its) !=cfgrid->dnames_to_1dvnames.end())
2436 } // for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its)
2437
2438 // Remove the dimension name that finds the cooresponding variables from the tempvardimlist.
2439 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
2440 its = tempvardimnamelist.find((*irv)->cfdimname);
2441 if (its != tempvardimnamelist.end()) tempvardimnamelist.erase(its);
2442 }
2443
2444 // Second: Some dimension names still need to find CVs, create the missing CVs
2445 for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its) {
2446
2447 auto EOS5cvar = new EOS5CVar();
2448 Create_Missing_CV(cfgrid, EOS5cvar, *its, GRID, this->eos5cfgrids.size());
2449 this->cvars.push_back(EOS5cvar);
2450
2451 }
2452}
2453
2454// Handle none-augmented grid coordinate variables for mutliple grids.
2455void EOS5File::Handle_Multi_Nonaugment_Grid_CVar()
2456{
2457
2458 BESDEBUG("h5", "Coming to Handle_Multi_nonaugment_Grid_CVar()"<<endl);
2459
2460 // If the multiple grids don't share the same lat/lon according to the parameters
2461 // We then assume that each single grid has its own lat/lon, just loop through each grid.
2462 if (true == this->grids_multi_latloncvs) {
2463 for (auto irv = this->eos5cfgrids.begin(); irv != this->eos5cfgrids.end(); ++irv)
2464 Handle_Single_Nonaugment_Grid_CVar(*irv);
2465 }
2466
2467 // We would like to check if lat/lon pairs provide for all grids
2468 // If lat/lon pairs are provided for all grids, then we ASSUME that
2469 // all grids share the same lat/lon values. This is what happened with
2470 // Aura grids. We only apply this to Aura files.They provide a lat/lon pair for each grid. We will observe
2471 // if this assumption is true for the future products.
2472 // If lat/lon pairs are not provided for all grids, we assume that each grid
2473 // may still have its unique lat/lon.
2474 else {
2475 int num_1dlatlon_pairs = 0;
2476 for (auto irv = this->eos5cfgrids.begin(); irv != this->eos5cfgrids.end(); ++irv)
2477 if (true == (*irv)->has_1dlatlon) num_1dlatlon_pairs++;
2478
2479 bool use_eos5_latlon = false;
2480 if ((0 == num_1dlatlon_pairs)
2481 || ((num_1dlatlon_pairs == (int) (this->eos5cfgrids.size())) && (true == this->isaura))) {
2482 set<string> tempvardimnamelist = ((this->eos5cfgrids)[0])->vardimnames;
2483 if (0 == num_1dlatlon_pairs) {
2484 use_eos5_latlon = Handle_Single_Nonaugment_Grid_CVar_EOS5LatLon((this->eos5cfgrids)[0],
2485 tempvardimnamelist);
2486
2487 if (false == use_eos5_latlon) return;
2488 }
2489
2490 else {
2491 // One lat/lon for all grids
2492 bool use_own_latlon = false;
2493 use_own_latlon = Handle_Single_Nonaugment_Grid_CVar_OwnLatLon((this->eos5cfgrids)[0],
2494 tempvardimnamelist);
2495 if (false == use_own_latlon) {
2496 use_eos5_latlon = Handle_Single_Nonaugment_Grid_CVar_EOS5LatLon((this->eos5cfgrids)[0],
2497 tempvardimnamelist);
2498 if (false == use_eos5_latlon) return;
2499 }
2500 }
2501
2502 // We need to handle the first grid differently since it will include "XDim" and "YDim".
2503 Handle_NonLatLon_Grid_CVar((this->eos5cfgrids)[0], tempvardimnamelist);
2504
2505 // Updating the dimension name sets for other grids
2506 for (unsigned j = 1; j < this->eos5cfgrids.size(); j++)
2507 (this->eos5cfgrids)[j]->Update_Dimnamelist();
2508
2509 // Adjusting the XDim and YDim dimension names for all vars
2510 Adjust_EOS5GridDimNames((this->eos5cfgrids)[0]);
2511
2512 // Now we can safely handle the rest grids
2513 for (unsigned j = 1; j < this->eos5cfgrids.size(); j++) {
2514 tempvardimnamelist = (this->eos5cfgrids)[j]->vardimnames;
2515 Handle_NonLatLon_Grid_CVar((this->eos5cfgrids)[j], tempvardimnamelist);
2516 tempvardimnamelist.clear();
2517 }
2518 } // if (( 0 == num_1dlatlon_pairs) || .....
2519 // No unique lat/lon, just loop through.
2520 else {
2521
2522 this->grids_multi_latloncvs = true;
2523 for (auto irv = this->eos5cfgrids.begin(); irv != this->eos5cfgrids.end(); ++irv)
2524 Handle_Single_Nonaugment_Grid_CVar(*irv);
2525 }
2526 }
2527}
2528
2529// Adjust the HDF-EOS5 grid dimension names for XDim and YDim, we need to remember the grid path
2530// Note this function is used under the assumption that only one lat/lon pair is used for all grids.
2531// This is the case for Aura.
2532void EOS5File::Adjust_EOS5GridDimNames(EOS5CFGrid *cfgrid)
2533{
2534
2535 BESDEBUG("h5", "Coming to Adjust_EOS5GridDimNames()"<<endl);
2536 string xdimname;
2537 string ydimname;
2538 bool find_xdim = false;
2539 bool find_ydim = false;
2540
2541 for (auto it = cfgrid->vardimnames.begin(); it != cfgrid->vardimnames.end(); ++it) {
2542 string xydimname_candidate = HDF5CFUtil::obtain_string_after_lastslash(*it);
2543 if ("XDim" == xydimname_candidate) {
2544 find_xdim = true;
2545 xdimname = *it;
2546 }
2547 else if ("YDim" == xydimname_candidate) {
2548 find_ydim = true;
2549 ydimname = *it;
2550 }
2551 if (find_xdim && find_ydim) break;
2552 } // for (auto it = cfgrid->vardimnames.begin() ...
2553
2554 if (false == find_xdim || false == find_ydim)
2555 throw2("Cannot find Dimension name that includes XDim or YDim in the grid ", cfgrid->name);
2556
2557 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2558 if (GRID == Get_Var_EOS5_Type(*irv)) {
2559 for (auto id = (*irv)->dims.begin(); id != (*irv)->dims.end(); ++id) {
2560 string xydimname_candidate = HDF5CFUtil::obtain_string_after_lastslash((*id)->name);
2561 if ("XDim" == xydimname_candidate)
2562 (*id)->name = xdimname;
2563 else if ("YDim" == xydimname_candidate) (*id)->name = ydimname;
2564 }
2565 }
2566 }
2567}
2568
2569// Handle Swath Coordinate variables.
2570void EOS5File::Handle_Swath_CVar(bool isaugmented)
2571{
2572
2573 BESDEBUG("h5", "Coming to Handle_Swath_CVar()"<<endl);
2574 // In this version, we will not use the augmented option for coordinate variables of swath
2575 // since MLS products don't use the recent version of the augmentation tool to allocate their
2576 // coordinate variables.
2577 for (auto irs = this->eos5cfswaths.begin(); irs != this->eos5cfswaths.end();) {
2578 if ((*irs)->has_1dlatlon) {
2579 Handle_Single_1DLatLon_Swath_CVar(*irs, isaugmented);
2580 ++irs;
2581 }
2582 else if ((*irs)->has_2dlatlon) {
2583 Handle_Single_2DLatLon_Swath_CVar(*irs, isaugmented);
2584 ++irs;
2585 }
2586 // If number of dimension latitude/longitude is >2 or no lat/lon,
2587 // no cooridnate variables will be generated.
2588 // We will simply remove this swath from the vector eos5cfswaths.
2589 // In the future, we may consider supporting non "Latitude", "Longitude" naming swaths.
2590 // KY 2011-1-20
2591 else {
2592 delete (*irs);
2593 irs = this->eos5cfswaths.erase(irs);
2594 }
2595 }
2596}
2597
2598// Handle single 1D LatLon Swath Coordinate variables.
2599void EOS5File::Handle_Single_1DLatLon_Swath_CVar(EOS5CFSwath *cfswath, bool is_augmented)
2600{
2601
2602 BESDEBUG("h5", "Coming to Handle_Single_1DLatLon_Swath_CVar"<<endl);
2603 // For 1DLatLon, we will use latitude as the coordinate variable
2604 set<string> tempvardimnamelist = cfswath->vardimnames;
2605 string EOS5SWATHPATH = "/HDFEOS/SWATHS/";
2606 string fslash_str = "/";
2607 string THIS_EOS5SWATHPATH = EOS5SWATHPATH + cfswath->name + fslash_str;
2608#if 0
2609 for (auto its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its)
2610 cerr<<"Dimension name befor latitude " << *its << endl;
2611#endif
2612
2613 // Find latitude and assign to the coordinate variable
2614 // (*irv)->fullpath.size() > THIS_EOS5SWATHPATH.size() is necessary to handle the augmented variables.
2615 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2616 if (SWATH == Get_Var_EOS5_Type(*irv) && ((*irv)->fullpath.size() > THIS_EOS5SWATHPATH.size())) {
2617
2618 string var_swath_name = Obtain_Var_EOS5Type_GroupName(*irv, SWATH);
2619 if ((var_swath_name == cfswath->name) && ((*irv)->name == "Latitude")) {
2620
2621 //Find it, create a coordinate variable.
2622 auto EOS5cvar = new EOS5CVar(*irv);
2623
2624 // Still keep the original dimension name to avoid the nameclashing when
2625 // one grid and one swath and one za occur in the same file
2626 EOS5cvar->cfdimname = ((*irv)->dims)[0]->name;
2627 EOS5cvar->cvartype = CV_EXIST;
2628 EOS5cvar->eos_type = SWATH;
2629
2630 // Save this cv to the cv vector
2631 this->cvars.push_back(EOS5cvar);
2632
2633 // Remove this var from the var vector since it becomes a cv.
2634 delete (*irv);
2635 this->vars.erase(irv);
2636 break;
2637 } // if ((var_swath_name == cfswath->name) && ...
2638 } // if (SWATH == Get_Var_EOS5_Type(*irv) &&
2639 } // for (auto irv = this->vars.begin() ...
2640
2641 // Finish this variable, remove it from the list.
2642
2643 bool find_lat_dim = false;
2644 for (auto its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its) {
2645
2646 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
2647 if (((*irv)->name == "Latitude") && (*irv)->cfdimname == (*its)) {
2648 tempvardimnamelist.erase(its);
2649 find_lat_dim = true;
2650 break;
2651 }
2652 }
2653
2654 if (true == find_lat_dim) break;
2655 }
2656
2657#if 0
2658 for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its)
2659 cerr<<"Dimension name afte latitude " << *its << endl;
2660#endif
2661
2662 Handle_NonLatLon_Swath_CVar(cfswath, tempvardimnamelist);
2663
2664 // Remove the added variables during the augmentation process
2665 if (true == is_augmented) {
2666 for (auto irv = this->vars.begin(); irv != this->vars.end();) {
2667
2668 if (SWATH == Get_Var_EOS5_Type(*irv)) {
2669#if 0
2670 string my_swath_short_path = (*irv)->fullpath.substr(EOS5SWATHPATH.size());
2671 size_t first_fslash_pos = my_swath_short_path.find_first_of("/");
2672 string my_swath_name = my_swath_short_path.substr(0,first_fslash_pos);
2673#endif
2674 // Need to find the swath for this variable
2675 string my_swath_name = Obtain_Var_EOS5Type_GroupName(*irv, SWATH);
2676
2677 if (my_swath_name == cfswath->name) {
2678 string var_path_after_swathname = (*irv)->fullpath.substr(THIS_EOS5SWATHPATH.size());
2679 if (var_path_after_swathname == (*irv)->name) {
2680 delete (*irv);
2681 irv = this->vars.erase(irv);
2682 }
2683 else {
2684 ++irv;
2685 }
2686 }
2687 else {
2688 ++irv;
2689 }
2690 }
2691 else {
2692 ++irv;
2693 }
2694 } // end of for loop
2695 } // if (true == is_augmented)
2696}
2697
2698// Handle Single 2D lat/lon Coordinate variables for Swath
2699void EOS5File::Handle_Single_2DLatLon_Swath_CVar(EOS5CFSwath *cfswath, bool is_augmented)
2700{
2701
2702 BESDEBUG("h5", "Coming to Handle_Single_2DLatLon_Swath_CVar()"<<endl);
2703 // For 2DLatLon, we will use both latitude and longitude as the coordinate variables
2704 set<string>::iterator its;
2705 set<string> tempvardimnamelist = cfswath->vardimnames;
2706 string EOS5SWATHPATH = "/HDFEOS/SWATHS/";
2707 string fslash_str = "/";
2708 string THIS_EOS5SWATHPATH = EOS5SWATHPATH + cfswath->name + fslash_str;
2709 bool find_lat = false;
2710 bool find_lon = false;
2711
2712#if 0
2713 for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its)
2714 cerr<<"Dimension name befor latitude " << *its << endl;
2715#endif
2716
2717 // Find latitude and assign to the coordinate variable
2718 for (auto irv = this->vars.begin(); irv != this->vars.end();) {
2719 if (SWATH == Get_Var_EOS5_Type(*irv) && ((*irv)->fullpath.size() > THIS_EOS5SWATHPATH.size())) {
2720 string var_swath_name = Obtain_Var_EOS5Type_GroupName(*irv, SWATH);
2721 if ((var_swath_name == cfswath->name) && ((*irv)->name == "Latitude")) {
2722
2723 //Find it, create a coordinate variable.
2724 auto EOS5cvar = new EOS5CVar(*irv);
2725
2726 // Still keep the original dimension name to avoid the nameclashing when
2727 // one grid and one swath and one za occur in the same file
2728 EOS5cvar->cfdimname = ((*irv)->dims)[0]->name;
2729 EOS5cvar->cvartype = CV_EXIST;
2730 EOS5cvar->eos_type = SWATH;
2731 EOS5cvar->is_2dlatlon = true;
2732
2733 // Save this cv to the cv vector
2734 this->cvars.push_back(EOS5cvar);
2735
2736 // Remove this var from the var vector since it becomes a cv.
2737 delete (*irv);
2738 irv = this->vars.erase(irv);
2739 find_lat = true;
2740 }
2741 else if ((var_swath_name == cfswath->name) && ((*irv)->name == "Longitude")) {
2742
2743 //Find it, create a coordinate variable.
2744 auto EOS5cvar = new EOS5CVar(*irv);
2745
2746 // Still keep the original dimension name to avoid the nameclashing when
2747 // one grid and one swath and one za occur in the same file
2748 EOS5cvar->cfdimname = ((*irv)->dims)[1]->name;
2749 EOS5cvar->cvartype = CV_EXIST;
2750 EOS5cvar->eos_type = SWATH;
2751 EOS5cvar->is_2dlatlon = true;
2752
2753 // Save this cv to the cv vector
2754 this->cvars.push_back(EOS5cvar);
2755
2756 // Remove this var from the var vector since it becomes a cv.
2757 delete (*irv);
2758 irv = this->vars.erase(irv);
2759 find_lon = true;
2760
2761 }
2762 else {
2763 ++irv;
2764 }
2765 } // if (SWATH == Get_Var_EOS5_Type(*irv) && ...
2766 else {
2767 ++irv;
2768 }
2769
2770 if (true == find_lat && true == find_lon) break;
2771 } // for (auto irv = this->vars.begin();
2772
2773 // Remove the dim. of latitude
2774 find_lat = false;
2775 for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its) {
2776 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
2777 if (((*irv)->name == "Latitude") && (*irv)->cfdimname == (*its)) {
2778 tempvardimnamelist.erase(its);
2779 find_lat = true;
2780 break;
2781 }
2782 }
2783
2784 if (true == find_lat) break;
2785 }
2786
2787 // Remove the dim. of longitude
2788 find_lon = false;
2789 for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its) {
2790
2791 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
2792
2793 if (((*irv)->name == "Longitude") && (*irv)->cfdimname == (*its)) {
2794 tempvardimnamelist.erase(its);
2795 find_lon = true;
2796 break;
2797 }
2798 }
2799
2800 if (true == find_lon) break;
2801 }
2802
2803#if 0
2804 for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its)
2805 cerr<<"Dimension name afte latitude " << *its << endl;
2806#endif
2807
2808 Handle_NonLatLon_Swath_CVar(cfswath, tempvardimnamelist);
2809
2810 // Remove the added variables during the augmentation process
2811 // For Swath, we don't want to keep the augmented files. This is because
2812 // some aura files assign the dimensional scale as zero.
2813 // We will actively check the new NASA HDF-EOS5 products and will
2814 // revise the following section as needed. KY 2012-03-09
2815 if (true == is_augmented) {
2816 for (auto irv = this->vars.begin(); irv != this->vars.end();) {
2817
2818 if (SWATH == Get_Var_EOS5_Type(*irv)) {
2819
2820 string my_swath_name = Obtain_Var_EOS5Type_GroupName(*irv, SWATH);
2821 if (my_swath_name == cfswath->name) {
2822 string var_path_after_swathname = (*irv)->fullpath.substr(THIS_EOS5SWATHPATH.size());
2823 if (var_path_after_swathname == (*irv)->name) {
2824 delete (*irv);
2825 irv = this->vars.erase(irv);
2826 }
2827 else {
2828 ++irv;
2829 }
2830 }
2831 else {
2832 ++irv;
2833 }
2834 }
2835 else {
2836 ++irv;
2837 }
2838 }
2839 }
2840}
2841
2842// Handle non-lat/lon Swath coordinate variables.
2843void EOS5File::Handle_NonLatLon_Swath_CVar(EOS5CFSwath *cfswath, set<string>& tempvardimnamelist)
2844{
2845
2846 BESDEBUG("h5", "Coming to Handle_NonLatLon_Swath_CVar()"<<endl);
2847 // First check if we have existing coordinate variable
2848 set<string>::iterator its;
2849 auto num_dimnames = (int)(tempvardimnamelist.size());
2850 bool has_dimnames = true;
2851 for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its) {
2852 if (cfswath->dnames_to_geo1dvnames.find(*its) != cfswath->dnames_to_geo1dvnames.end()) {
2853 for (auto irv = this->vars.begin(); has_dimnames && (irv != this->vars.end());) {
2854
2855 // We need to check if this var is a swath and use "newname"
2856 // of var to check the dnames_to_1dvnames since it is
2857 // possible to have name clashings for the "name" of a var.
2858 if (SWATH == Get_Var_EOS5_Type(*irv) && (*irv)->newname == (cfswath->dnames_to_geo1dvnames)[*its]) {
2859
2860 //Find it, create a coordinate variable.
2861 auto EOS5cvar = new EOS5CVar(*irv);
2862
2863 // Still keep the original dimension name to avoid the nameclashing when
2864 // one grid and one swath and one za occur in the same file
2865 EOS5cvar->cfdimname = *its;
2866 EOS5cvar->cvartype = CV_EXIST;
2867 EOS5cvar->eos_type = SWATH;
2868
2869 // Save this cv to the cv vector
2870 this->cvars.push_back(EOS5cvar);
2871
2872 // Remove this var from the var vector since it becomes a cv.
2873 delete (*irv);
2874 irv = this->vars.erase(irv);
2875 num_dimnames--;
2876 if (0 == num_dimnames) has_dimnames = false;
2877 }
2878 else {
2879 ++irv;
2880 }
2881 } // for (auto irv = this->vars.begin(); ...
2882 } // if (cfswath->dnames_to_geo1dvnames.find(*its) ....
2883 } // for (its = tempvardimnamelist.begin()...
2884
2885 // Remove the dimension name that finds the cooresponding variables from the tempvardimlist.
2886 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
2887 its = tempvardimnamelist.find((*irv)->cfdimname);
2888 if (its != tempvardimnamelist.end()) tempvardimnamelist.erase(its);
2889 }
2890
2891 // Check if some attributes have CV information for some special products
2892 // Currently TES needs to be handled carefully
2893 Handle_Special_NonLatLon_Swath_CVar(cfswath, tempvardimnamelist);
2894
2895 // Remove the dimension name that finds the cooresponding variables from the tempvardimlist.
2896 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
2897 its = tempvardimnamelist.find((*irv)->cfdimname);
2898 if (its != tempvardimnamelist.end()) tempvardimnamelist.erase(its);
2899 }
2900
2901 // Second: Some dimension names still need to find CVs, create the missing CVs
2902 for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its) {
2903
2904 auto EOS5cvar = new EOS5CVar();
2905 Create_Missing_CV(cfswath, EOS5cvar, *its, SWATH, this->eos5cfswaths.size());
2906 this->cvars.push_back(EOS5cvar);
2907
2908 }
2909}
2910
2911// Handle special non-lat/lon coordinate variables for swath.
2912void EOS5File::Handle_Special_NonLatLon_Swath_CVar(EOS5CFSwath *cfswath, set<string>& tempvardimnamelist)
2913
2914{
2915
2916 BESDEBUG("h5", "Handle_Special_NonLatLon_Swath_CVar()"<<endl);
2917 // We have no choice but hard-code this one.
2918 // TES swath puts "Pressure" as the VerticalCoordinate but doesn't provide "Pressure" values.
2919 // Moreover, the number of pressure level(66) is one less than the total number of corresponding dimension size(67)
2920 // most probably due to the missing pressure level on the ground. To make the handler visualize some
2921 // TES variables and to follow the general physical sense. We have to add a pressure level by linear interpolation.
2922 // KY 2012-1-27
2923 if (true == this->isaura && TES == this->aura_name) {
2924
2925 string eos5_swath_group_name = "/HDFEOS/SWATHS/" + cfswath->name;
2926 string eos5_vc_attr_name = "VerticalCoordinate";
2927 string eos5_pre_attr_name = "Pressure";
2928 bool has_vc_attr = false;
2929 Group *vc_group = nullptr;
2930
2931 // 1. Check if having the "VerticalCoordinate" attribute in this swath and the attribute is "Pressure".
2932 for (auto irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
2933 if (eos5_swath_group_name == (*irg)->path) {
2934 for (auto ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end(); ++ira) {
2935 if (eos5_vc_attr_name == (*ira)->name) {
2936 Retrieve_H5_Attr_Value(*ira, (*irg)->path);
2937 string attr_value((*ira)->value.begin(), (*ira)->value.end());
2938 if (eos5_pre_attr_name == attr_value) {
2939 has_vc_attr = true;
2940 vc_group = *irg;
2941 break;
2942 }
2943 }
2944 } // for (vector<Attribute *>:: iterator ira =(*irg)->attrs.begin(); ...
2945 if (true == has_vc_attr) break;
2946 } // if (eos5_swath_group_name ==(*irg)->path)
2947 } // for (auto irg = this->groups.begin(); ...
2948
2949 // 2. Check if having the "Pressure" attribute and if the attribute size is 1 less than
2950 // the dimension size of "nLevels". If yes,
2951 // add one pressure value by using the nearest neighbor value. This value should be the first value
2952 // of the "Pressure" attribute.
2953 // Another special part of the TES file is that dimension name nLevels is used twice in some variables
2954 // float foo[...][nLevels][nLevels]. To make the variable visualized by tools, the dimension name
2955 // needs to be changed and the coordinate variable needs to separately created. Note this is not
2956 // against CF conventions. However, the popular tools are not happy with the duplicate dimension names
2957 // in a variable.
2958 // Though may not cover 100% cases, searching the string after the last forward slash and see if
2959 // it contains nLevels should catch 99% memebers of the "nLevels" family. We will then create the
2960 // corresponding coordinate variables.
2961
2962 // 2.1. Check if we have the dimension name called "nLevels" for this swath
2963 if (true == has_vc_attr) {
2964 string dimname_candidate = "/SWATHS/" + cfswath->name + "/nLevels";
2965 for (auto it = tempvardimnamelist.begin(); it != tempvardimnamelist.end(); ++it) {
2966 if ((*it).find(dimname_candidate) != string::npos) {
2967 hsize_t dimsize_candidate = 0;
2968 if ((cfswath->dimnames_to_dimsizes).find(*it) != (cfswath->dimnames_to_dimsizes).end())
2969 dimsize_candidate = cfswath->dimnames_to_dimsizes[*it];
2970 else
2971 throw2("Cannot find the dimension size of the dimension name ", *it);
2972
2973 // Note: we don't have to use two loops to create the coordinate variables.
2974 // However, there are only 3-4 attributes for this group and so far TES has only
2975 // one additional nLevels.
2976 // So essentially the following loop doesn't hurt the performance.
2977 // KY 2012-2-1
2978 for (auto ira = vc_group->attrs.begin(); ira != vc_group->attrs.end();
2979 ++ira) {
2980 if ((eos5_pre_attr_name == (*ira)->name) && ((*ira)->count == (dimsize_candidate - 1))) {
2981
2982 // Should change the attr_value from char type to float type when reading the data
2983 // Here just adding a coordinate variable by using this name.
2984 auto EOS5cvar = new EOS5CVar();
2985 string reduced_dimname = HDF5CFUtil::obtain_string_after_lastslash(*it);
2986 string orig_dimname = "nLevels";
2987 if ("nLevels" == reduced_dimname)
2988 EOS5cvar->name = eos5_pre_attr_name + "_CV";
2989 else
2990 // the dimname will be ..._CV_1 etc.
2991 EOS5cvar->name = eos5_pre_attr_name + "_CV"
2992 + reduced_dimname.substr(orig_dimname.size());
2993 Create_Added_Var_NewName_FullPath(SWATH, cfswath->name, EOS5cvar->name, EOS5cvar->newname,
2994 EOS5cvar->fullpath);
2995 EOS5cvar->rank = 1;
2996 EOS5cvar->dtype = (*ira)->dtype;
2997 auto eos5cvar_dim = new Dimension(dimsize_candidate);
2998 eos5cvar_dim->name = *it;
2999 if (1 == this->eos5cfswaths.size())
3000 eos5cvar_dim->newname = reduced_dimname;
3001 else
3002 eos5cvar_dim->newname = eos5cvar_dim->name;
3003
3004 EOS5cvar->dims.push_back(eos5cvar_dim);
3005 EOS5cvar->cvartype = CV_SPECIAL;
3006 EOS5cvar->cfdimname = eos5cvar_dim->name;
3007 EOS5cvar->eos_type = SWATH;
3008
3009 // Save this cv to the cv vector
3010 this->cvars.push_back(EOS5cvar);
3011 } // if ((eos5_pre_attr_name == (*ira)->name) && ...
3012 } // for (auto ira = vc_group->attrs.begin();
3013 } // if ((*it).find(dimname_candidate) != string::npos)
3014 } // for (it = tempvardimnamelist.begin(); ...
3015 } // if (true == has_vc_attr) ...
3016 } // if (true == this->isaura && ...
3017}
3018
3019// Handle Zonal average coordinate variables.
3020void EOS5File::Handle_Za_CVar(bool isaugmented)
3021{
3022
3023 BESDEBUG("h5", "Coming to Handle_Za_CVar()"<<endl);
3024 // We are not supporting non-augmented zonal average HDF-EOS5 product now. KY:2012-1-20
3025 if (false == isaugmented) return;
3026
3027 for (auto irv = this->eos5cfzas.begin(); irv != this->eos5cfzas.end(); ++irv)
3028 Handle_Single_Augment_CVar(*irv, ZA);
3029
3030}
3031
3032// Adjust the newname(final names appeared at DDS) for variable and dimensions before flattening.
3034{
3035
3036 BESDEBUG("h5", "Coming to Adjust_Var_Dim_NewName_Before_Flattening()"<<endl);
3037 auto num_grids = (int)(this->eos5cfgrids.size());
3038 auto num_swaths = (int)(this->eos5cfswaths.size());
3039 auto num_zas = (int)(this->eos5cfzas.size());
3040
3041 bool mixed_eos5typefile = false;
3042
3043 // Check if this file mixes grid,swath and zonal average
3044 if (((num_grids > 0) && (num_swaths > 0)) || ((num_grids > 0) && (num_zas > 0))
3045 || ((num_swaths > 0) && (num_zas > 0))) mixed_eos5typefile = true;
3046
3047 // This file doesn't mix swath, grid and zonal average
3048 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv)
3049 Adjust_Per_Var_Dim_NewName_Before_Flattening(*irv, mixed_eos5typefile, num_grids, num_swaths, num_zas);
3050
3051 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv)
3052 Adjust_Per_Var_Dim_NewName_Before_Flattening(*irv, mixed_eos5typefile, num_grids, num_swaths, num_zas);
3053#if 0
3054 for (auto irv = this->cvars.begin();
3055 irv != this->cvars.end(); ++irv) {
3056 cerr<<"eos5svar var new name "<<(*irv)->newname <<endl;
3057 for (auto ird = (*irv)->dims.begin();
3058 ird !=(*irv)->dims.end(); ++ird) {
3059 cerr<<"eos5svar dimension new name "<<(*ird)->newname <<endl;
3060 }
3061 }
3062#endif
3063 // If (lat,lon) is shared for grids, more consideration for the names
3064 Adjust_SharedLatLon_Grid_Var_Dim_Name();
3065
3066}
3067
3068// Adjust the final name of one variable or dim. before flattening the names
3069template<class T>
3070void EOS5File::Adjust_Per_Var_Dim_NewName_Before_Flattening(T* var, bool mixed_eos5type, int num_grids, int num_swaths,
3071 int num_zas)
3072{
3073
3074 BESDEBUG("h5", "Coming to Adjust_Per_Var_Dim_NewName_Before_Flattening()"<<endl);
3075
3076 string eos5typestr;
3077 EOS5Type vartype = Get_Var_EOS5_Type(var);
3078 switch (vartype) {
3079
3080 case GRID: {
3081 eos5typestr = "/GRIDS/";
3082 if (false == mixed_eos5type) {
3083 if (0 == num_grids)
3084 var->newname = ((1 == this->orig_num_grids) ? var->name : var->newname.substr(eos5typestr.size()));
3085 else
3086 var->newname = ((1 == num_grids) ? var->name : var->newname.substr(eos5typestr.size()));
3087 // Dimension newname is unlike Var newname, when num_grids is equal to 1, the
3088 // newname is Dimension name already. So we don't need to do anything with
3089 // the dimension newname when the num_grids is 1. The main reason we handle
3090 // the var newname and the dimension newname differently is that the variable name is
3091 // more critical for users to pick up the meanings of that variable. So we would like
3092 // to work hard to keep the original form. However, the dimension name is mainly used to
3093 // generate the coordinate variables. So the different usage makes us relax the dimension
3094 // name a little bit. This is an example of end-user priority driven implementation.
3095 // KY 2012-1-24
3096 // Just receive a user request: the dimension name is also very important.
3097 // So a bunch of code has been updated. For number of grid/swath/za = 1, I still maintain
3098 // the newname to be the same as the last part of the dim name. Hopefully this
3099 // will handle the current HDF-EOS5 products. Improvement for complicate HDF-EOS5 products
3100 // will be supported as demanded in the future. KY 2012-1-26
3101 if (num_grids > 1) {
3102 for (auto ird = var->dims.begin(); ird != var->dims.end(); ird++) {
3103 if ((*ird)->newname.size() <= eos5typestr.size())
3104 throw5("The size of the dimension new name ", (*ird)->newname, "of variable ", var->newname,
3105 " is too small");
3106 (*ird)->newname = (*ird)->newname.substr(eos5typestr.size());
3107 }
3108 }
3109 } // if(false == mixed_eos5type)
3110 else {
3111 // No need to set the dimension newname for the reason listed above.
3112 var->newname = ((1 == num_grids) ? (eos5typestr + var->name) : var->newname);
3113 }
3114 }
3115 break;
3116
3117 case SWATH: {
3118 eos5typestr = "/SWATHS/";
3119 if (false == mixed_eos5type) {
3120 var->newname = ((1 == num_swaths) ? var->name : var->newname.substr(eos5typestr.size()));
3121 if (num_swaths > 1) {
3122 for (auto ird = var->dims.begin(); ird != var->dims.end(); ird++) {
3123 if ((*ird)->newname.size() <= eos5typestr.size())
3124 throw5("The size of the dimension new name ", (*ird)->newname, "of variable ", var->newname,
3125 " is too small");
3126 (*ird)->newname = (*ird)->newname.substr(eos5typestr.size());
3127 }
3128 }
3129 }
3130 else {
3131 var->newname = ((1 == num_swaths) ? (eos5typestr + var->name) : var->newname);
3132 }
3133 }
3134 break;
3135
3136 case ZA: {
3137 eos5typestr = "/ZAS/";
3138 if (false == mixed_eos5type) {
3139 var->newname = ((1 == num_zas) ? var->name : var->newname.substr(eos5typestr.size()));
3140 if (num_zas > 1) {
3141 for (auto ird = var->dims.begin(); ird != var->dims.end(); ird++) {
3142 if ((*ird)->newname.size() <= eos5typestr.size())
3143 throw5("The size of the dimension new name ", (*ird)->newname, "of variable ", var->newname,
3144 " is too small");
3145 (*ird)->newname = (*ird)->newname.substr(eos5typestr.size());
3146 }
3147 }
3148 }
3149 else {
3150 var->newname = ((1 == num_zas) ? (eos5typestr + var->name) : var->newname);
3151 }
3152 }
3153 break;
3154 case OTHERVARS:
3155 break;
3156 default:
3157 throw1("Non-supported EOS type");
3158 }
3159
3160}
3161
3162// Adjust shared var and dim names for shared lat/lon grid case.
3163void EOS5File::Adjust_SharedLatLon_Grid_Var_Dim_Name()
3164{
3165
3166 BESDEBUG("h5", "Adjust_SharedLatLon_Grid_Var_Dim_Name()"<<endl);
3167 // Remove the EOS5 type string("GRIDS") and the GRID Name from
3168 // the variable newname and the dimension newname
3169 // This case won't happen for the current version, but may occur
3170 // if curviliner grid exists in the file. KY 2012-1-26
3171 if ((this->eos5cfgrids.size() > 1) && (0 == this->eos5cfswaths.size()) && (0 == this->eos5cfzas.size())
3172 && (false == this->grids_multi_latloncvs)) {
3173
3174 // We would like to condense the dimension name and the coordinate variable name for lat/lon.
3175 string lat_dimname;
3176 string lat_dimnewname;
3177 string lon_dimname;
3178 string lon_dimnewname;
3179 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
3180 if ("lat" == (*irv)->name || "Latitude" == (*irv)->name) {
3181 (*irv)->newname = (*irv)->name;
3182 lat_dimnewname = (((*irv)->dims)[0])->newname;
3183 lat_dimnewname = HDF5CFUtil::obtain_string_after_lastslash(lat_dimnewname);
3184 if ("" == lat_dimnewname)
3185 throw2("/ is not included in the dimension new name ", (((*irv)->dims)[0])->newname);
3186 (((*irv)->dims)[0])->newname = lat_dimnewname;
3187 lat_dimname = (*irv)->cfdimname;
3188 }
3189 else if ("lon" == (*irv)->name || "Longitude" == (*irv)->name) {
3190 (*irv)->newname = (*irv)->name;
3191 lon_dimnewname = (((*irv)->dims)[0])->newname;
3192 lon_dimnewname = HDF5CFUtil::obtain_string_after_lastslash(lon_dimnewname);
3193 if ("" == lon_dimnewname)
3194 throw2("/ is not included in the dimension new name ", (((*irv)->dims)[0])->newname);
3195 (((*irv)->dims)[0])->newname = lon_dimnewname;
3196 lon_dimname = (*irv)->cfdimname;
3197 }
3198 } // for (auto irv = this->cvars.begin(); ...
3199
3200 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
3201 for (auto ird = (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
3202 if ((*ird)->name == lat_dimname)
3203 (*ird)->newname = lat_dimnewname;
3204 else if ((*ird)->name == lon_dimname) (*ird)->newname = lon_dimnewname;
3205 }
3206 }
3207 } // if ((this->eos5cfgrids.size() > 1) && ...
3208}
3209
3210// Flatten the object names.
3211void EOS5File::Flatten_Obj_Name(bool include_attr)
3212{
3213
3214 BESDEBUG("h5", "Coming to Flatten_Obj_Name()"<<endl);
3215 File::Flatten_Obj_Name(include_attr);
3216
3217 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
3218 (*irv)->newname = get_CF_string((*irv)->newname);
3219
3220 for (auto ird = (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
3221 (*ird)->newname = get_CF_string((*ird)->newname);
3222 }
3223
3224 if (true == include_attr) {
3225 for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
3226 (*ira)->newname = File::get_CF_string((*ira)->newname);
3227 }
3228 }
3229 } // for (auto irv = this->cvars.begin(); ...
3230}
3231
3232// Handle Object Name clashing
3234{
3235
3236 BESDEBUG("h5", "Coming to Handle_Obj_NameClashing()"<<endl);
3237 // objnameset will be filled with all object names that we are going to check the name clashing.
3238 // For example, we want to see if there are any name clashings for all variable names in this file.
3239 // objnameset will include all variable names. If a name clashing occurs, we can figure out from the set operation immediately.
3240 set<string> objnameset;
3241 Handle_EOS5CVar_NameClashing(objnameset);
3242 File::Handle_GeneralObj_NameClashing(include_attr, objnameset);
3243 if (true == include_attr) {
3244 Handle_EOS5CVar_AttrNameClashing();
3245 }
3246#if 0
3247 //if (this->cvars.size() >0)
3248 // Handle_DimNameClashing();
3249#endif
3250}
3251
3252// Handle EOS5 coordinate variable name clashing
3253void EOS5File::Handle_EOS5CVar_NameClashing(set<string> &objnameset)
3254{
3255
3256 BESDEBUG("h5", "Coming to Handle_EOS5CVar_NameClashing()"<<endl);
3257 EOS5Handle_General_NameClashing(objnameset, this->cvars);
3258}
3259
3260// Handle EOS5 coordinate varaible attribute name clashing
3261void EOS5File::Handle_EOS5CVar_AttrNameClashing()
3262{
3263
3264 BESDEBUG("h5", "Coming to Handle_EOS5CVar_AttrNameClashing()"<<endl);
3265 set<string> objnameset;
3266
3267 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
3268 Handle_General_NameClashing(objnameset, (*irv)->attrs);
3269 objnameset.clear();
3270 }
3271}
3272// The routine to handle general name clashing
3273//class T must have member string newname
3274template<class T> void EOS5File::EOS5Handle_General_NameClashing(set<string>&objnameset, vector<T*>& objvec)
3275
3276{
3277
3278 BESDEBUG("h5", "Coming to EOS5Handle_General_NameClashing()"<<endl);
3279 pair<set<string>::iterator, bool> setret;
3280 set<string>::iterator iss;
3281
3282 vector<string> clashnamelist;
3283
3284 map<int, int> cl_to_ol;
3285 int ol_index = 0;
3286 int cl_index = 0;
3287
3288#if 0
3289 typename vector<T*>::iterator irv;
3290#endif
3291
3292 for (auto irv = objvec.begin(); irv != objvec.end(); ++irv) {
3293
3294 setret = objnameset.insert((*irv)->newname);
3295 if (!setret.second) {
3296 clashnamelist.insert(clashnamelist.end(), (*irv)->newname);
3297 cl_to_ol[cl_index] = ol_index;
3298 cl_index++;
3299 }
3300 ol_index++;
3301 }
3302
3303 // Now change the clashed elements to unique elements;
3304 // Generate the set which has the same size as the original vector.
3305 for (auto ivs = clashnamelist.begin(); ivs != clashnamelist.end(); ++ivs) {
3306 int clash_index = 1;
3307 string temp_clashname = *ivs + '_';
3308 HDF5CFUtil::gen_unique_name(temp_clashname, objnameset, clash_index);
3309 *ivs = temp_clashname;
3310 }
3311
3312 // Now go back to the original vector, make it unique.
3313 for (unsigned int i = 0; i < clashnamelist.size(); i++)
3314 objvec[cl_to_ol[i]]->newname = clashnamelist[i];
3315
3316}
3317
3318// Handle Dimension name clashing
3320{
3321
3322 BESDEBUG("h5", "Coming to Handle_DimNameClashing()"<<endl);
3323 map<string, string> dimname_to_dimnewname;
3324 pair<map<string, string>::iterator, bool> mapret;
3325 set<string> dimnameset;
3326 vector<Dimension*> vdims;
3327 set<string> dimnewnameset;
3328 pair<set<string>::iterator, bool> setret;
3329
3330 // First: Generate the dimset/dimvar based on coordinate variables.
3331 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
3332 for (auto ird = (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
3333#if 0
3334 //setret = dimnameset.insert((*ird)->newname);
3335#endif
3336 setret = dimnameset.insert((*ird)->name);
3337 if (setret.second) vdims.push_back(*ird);
3338 }
3339 }
3340
3341 // For some cases, dimension names are provided but there are no corresponding coordinate
3342 // variables. For now, we will assume no such cases.
3343 // Actually, we find such a case in our fake testsuite. So we need to fix it.
3344
3345 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
3346 for (auto ird = (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
3347#if 0
3348 //setret = dimnameset.insert((*ird)->newname);
3349#endif
3350 setret = dimnameset.insert((*ird)->name);
3351 if (setret.second) vdims.push_back(*ird);
3352 }
3353 }
3354
3355#if 0
3356 for (auto ird=vdims.begin();ird!=vdims.end();++ird)
3357 cerr<<"dimension name "<<(*ird)->name <<endl;
3358#endif
3359
3360 // For some cases, dimension names are provided but there are no corresponding coordinate
3361 // variables. For now, we will assume no such cases.
3362 EOS5Handle_General_NameClashing(dimnewnameset, vdims);
3363
3364 // Third: Make dimname_to_dimnewname map
3365 for (auto ird = vdims.begin(); ird != vdims.end(); ++ird) {
3366 mapret = dimname_to_dimnewname.insert(pair<string, string>((*ird)->name, (*ird)->newname));
3367 if (false == mapret.second)
3368 throw4("The dimension name ", (*ird)->name, " should map to ", (*ird)->newname);
3369 }
3370
3371 // Fourth: Change the original dimension new names to the unique dimension new names
3372 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv)
3373 for (auto ird = (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird)
3374 (*ird)->newname = dimname_to_dimnewname[(*ird)->name];
3375
3376 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv)
3377 for (auto ird = (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird)
3378 (*ird)->newname = dimname_to_dimnewname[(*ird)->name];
3379
3380}
3381
3382// Set COARDS Status, if we can follow COARDS, we should follow COARDS.
3383// http://ferret.wrc.noaa.gov/noaa_coop/coop_cdf_profile.html
3385{
3386
3387 BESDEBUG("h5", "Coming to Set_COARDS_Status()"<<endl);
3388 iscoard = true;
3389 for (auto irg = this->eos5cfgrids.begin(); irg != this->eos5cfgrids.end(); ++irg) {
3390 if (false == (*irg)->has_1dlatlon) {
3391 if (false == (*irg)->has_nolatlon || (HE5_GCTP_GEO != (*irg)->eos5_projcode)) iscoard = false;
3392 break;
3393 }
3394 }
3395
3396 if (true == iscoard) {
3397 for (auto irg = this->eos5cfswaths.begin(); irg != this->eos5cfswaths.end(); ++irg) {
3398 if (false == (*irg)->has_1dlatlon) {
3399 iscoard = false;
3400 break;
3401 }
3402 }
3403 }
3404}
3405
3406// Adjust attribute info., mostly for CF name correction of Aura files.
3408{
3409
3410 BESDEBUG("h5", "Coming to Adjust_Attr_Info()"<<endl);
3411 if (true == this->isaura) {
3412 Adjust_Aura_Attr_Name();
3413 Adjust_Aura_Attr_Value();
3414 }
3415 else {
3416 Handle_EOS5CVar_Unit_Attr();
3417 Add_EOS5_Grid_CF_Attr();
3418 }
3419}
3420
3421// Adjust Attribute Name, mostly for Aura files.
3422void EOS5File::Adjust_Aura_Attr_Name()
3423{
3424
3425 BESDEBUG("h5", "Coming to Adjust_Attr_Name() for Aura"<<endl);
3426 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
3427 for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
3428 if (eos5_to_cf_attr_map.find((*ira)->name) != eos5_to_cf_attr_map.end()) (*ira)->newname =
3429 eos5_to_cf_attr_map[(*ira)->name];
3430
3431 }
3432 }
3433
3434 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
3435 for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
3436 if (eos5_to_cf_attr_map.find((*ira)->name) != eos5_to_cf_attr_map.end()) (*ira)->newname =
3437 eos5_to_cf_attr_map[(*ira)->name];
3438
3439 }
3440 }
3441}
3442
3443void EOS5File::Adjust_Aura_Attr_Value()
3444{
3445
3446 BESDEBUG("h5", "Coming to Adjust_Attr_Value() for Aura"<<endl);
3447 // Handle Units
3448 Handle_EOS5CVar_Unit_Attr();
3449 Handle_Aura_Special_Attr();
3450
3451 // Handle Time. This is just for Aura files.
3452 // This is for speical NASA requests only for Aura.
3453 // We need to pay attention if things get changed later.
3454 string time_cf_units_value = "seconds since 1993-01-01";
3455 for (auto irv = this->vars.begin(); irv != this->vars.end(); irv++) {
3456 if (((*irv)->name == "Time") || ((*irv)->name == "nTimes")) {
3457 for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ira++) {
3458 if ("units" == (*ira)->name) {
3459 Retrieve_H5_Attr_Value(*ira, (*irv)->fullpath);
3460 string units_value((*ira)->value.begin(), (*ira)->value.end());
3461 if (time_cf_units_value != units_value) {
3462
3463 units_value = time_cf_units_value;
3464 (*ira)->value.resize(units_value.size());
3465 if (H5FSTRING == (*ira)->dtype) (*ira)->fstrsize = units_value.size();
3466 // strsize is used by both fixed and variable length strings.
3467 (*ira)->strsize.resize(1);
3468 (*ira)->strsize[0] = units_value.size();
3469
3470 copy(units_value.begin(), units_value.end(), (*ira)->value.begin());
3471 }
3472 break;
3473 } // if ("units" == (*ira)->name)
3474 } // for(vector <Attribute*>::iterator ira = (*irv)->attrs.begin();
3475 } // if(((*irv)->name == "Time") || ((*irv)->name == "nTimes"))
3476 } // for (auto irv = this->vars.begin()...
3477}
3478
3479// Handle EOS5 coordinate variable special attributes.
3480void EOS5File::Handle_Aura_Special_Attr()
3481{
3482
3483 BESDEBUG("h5", "Coming to Handle_Aura_Special_Attr()"<<endl);
3484 // Need to handle MLS aura file specially.
3485 if (true == this->isaura && MLS == this->aura_name) {
3486
3487 const string File_attr_group_path = "/HDFEOS/ADDITIONAL/FILE_ATTRIBUTES";
3488 const string PCF1_attr_name = "PCF1";
3489 bool find_group = false;
3490 bool find_attr = false;
3491 for (auto it_g = this->groups.begin(); it_g != this->groups.end(); ++it_g) {
3492 if (File_attr_group_path == (*it_g)->path) {
3493 find_group = true;
3494 for (auto ira = (*it_g)->attrs.begin(); ira != (*it_g)->attrs.end(); ++ira) {
3495 if (PCF1_attr_name == (*ira)->name) {
3496 Retrieve_H5_Attr_Value(*ira, (*it_g)->path);
3497 string pcf_value((*ira)->value.begin(), (*ira)->value.end());
3498 HDF5CFDAPUtil::replace_double_quote(pcf_value);
3499 (*ira)->value.resize(pcf_value.size());
3500 if (H5FSTRING == (*ira)->dtype) (*ira)->fstrsize = pcf_value.size();
3501 // strsize is used by both fixed and variable length strings.
3502 (*ira)->strsize.resize(1);
3503 (*ira)->strsize[0] = pcf_value.size();
3504
3505 copy(pcf_value.begin(), pcf_value.end(), (*ira)->value.begin());
3506 find_attr = true;
3507 break;
3508 } // if (PCF1_attr_name == (*ira)->name)
3509 } // for (auto ira = (*it_g)->attrs.begin()
3510 } // if (File_attr_group_path == (*it_g)->path)
3511 if (true == find_group && true == find_attr) break;
3512 } // end of for it_g ...
3513 } // if (true == this->isaura && MLS == this->aura_name)
3514}
3515
3516// Handle coordinate variable units attribute
3517void EOS5File::Handle_EOS5CVar_Unit_Attr()
3518{
3519
3520 BESDEBUG("h5", "Coming to Handle_EOS5CVar_Unit_Attr()"<<endl);
3521 string unit_attrname = "units";
3522 string nonll_cf_level_attrvalue = "level";
3523 string lat_cf_unit_attrvalue = "degrees_north";
3524 string lon_cf_unit_attrvalue = "degrees_east";
3525 string tes_cf_pre_attrvalue = "hPa";
3526
3527 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
3528 switch ((*irv)->cvartype) {
3529 case CV_EXIST:
3530 case CV_MODIFY: {
3531 for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
3532 if ((*ira)->newname == unit_attrname) {
3533 Retrieve_H5_Attr_Value(*ira, (*irv)->fullpath);
3534 string units_value((*ira)->value.begin(), (*ira)->value.end());
3535 if ((lat_cf_unit_attrvalue != units_value)
3536 && (((*irv)->name == "Latitude") || ((this->eos5cfzas.empty() == false) && ((*irv)->name == "nLats")))) {
3537 units_value = lat_cf_unit_attrvalue;
3538#if 0
3539//cerr<<"coming to obtain the correct units_value: "<<units_value <<endl;
3540//cerr<<"cvar name is "<<(*irv)->newname <<endl;
3541#endif
3542 (*ira)->value.resize(units_value.size());
3543 if (H5FSTRING == (*ira)->dtype) (*ira)->fstrsize = units_value.size();
3544 // strsize is used by both fixed and variable length strings.
3545 (*ira)->strsize.resize(1);
3546 (*ira)->strsize[0] = units_value.size();
3547 copy(units_value.begin(), units_value.end(), (*ira)->value.begin());
3548 }
3549 else if ((lon_cf_unit_attrvalue != units_value) && (*irv)->name == "Longitude") {
3550 units_value = lon_cf_unit_attrvalue;
3551 (*ira)->value.resize(units_value.size());
3552 if (H5FSTRING == (*ira)->dtype) (*ira)->fstrsize = units_value.size();
3553 // strsize is used by both fixed and variable length strings.
3554 (*ira)->strsize.resize(1);
3555 (*ira)->strsize[0] = units_value.size();
3556
3557 copy(units_value.begin(), units_value.end(), (*ira)->value.begin());
3558 }
3559 break;
3560 } // if ((*ira)->newname ==unit_attrname)
3561 }
3562 }
3563 break;
3564
3565 case CV_LAT_MISS: {
3566 auto attr = new Attribute();
3567 Add_Str_Attr(attr, unit_attrname, lat_cf_unit_attrvalue);
3568 (*irv)->attrs.push_back(attr);
3569 }
3570 break;
3571
3572 case CV_LON_MISS: {
3573 auto attr = new Attribute();
3574 Add_Str_Attr(attr, unit_attrname, lon_cf_unit_attrvalue);
3575 (*irv)->attrs.push_back(attr);
3576 }
3577 break;
3578
3579 case CV_NONLATLON_MISS: {
3580 auto attr = new Attribute();
3581 Add_Str_Attr(attr, unit_attrname, nonll_cf_level_attrvalue);
3582 (*irv)->attrs.push_back(attr);
3583 }
3584 break;
3585 case CV_SPECIAL: {
3586 if (true == this->isaura && TES == this->aura_name) {
3587 auto attr = new Attribute();
3588 Add_Str_Attr(attr, unit_attrname, tes_cf_pre_attrvalue);
3589 (*irv)->attrs.push_back(attr);
3590 }
3591 }
3592 break;
3593 default:
3594 throw1("Non-supported Coordinate Variable Type.");
3595 }
3596 } // for (auto irv = this->cvars.begin() ...
3597}
3598
3599void EOS5File::Add_EOS5_Grid_CF_Attr()
3600{
3601 BESDEBUG("h5", "Coming to Add_EOS5_Grid_CF_Attr()"<<endl);
3602
3603 bool has_eos5_grid_nongeo_proj = false;
3604
3605 // Check if we have EOS5 grids that are not using the geographic projection.
3606 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
3607 if ((*irv)->cvartype == CV_LAT_MISS) {
3608 if((*irv)->eos5_projcode !=HE5_GCTP_GEO) {
3609 has_eos5_grid_nongeo_proj = true;
3610 break;
3611 }
3612 }
3613 }
3614
3615 // We would like to add the CF conventions mark if the mark is not there.
3616 if(true == has_eos5_grid_nongeo_proj) {
3617 string conventions_attrname = "Conventions";
3618 string conventions_attrvalue = "CF-1.7";
3619 bool has_conventions_attr=false;
3620 for(vector<HDF5CF::Attribute *>::const_iterator it_ra=this->root_attrs.begin();
3621 it_ra!=this->root_attrs.end();it_ra++) {
3622 if((*it_ra)->name==conventions_attrname){
3623 has_conventions_attr = true;
3624 break;
3625 }
3626
3627 }
3628 if(false==has_conventions_attr) {
3629 auto attr = new Attribute();
3630 Add_Str_Attr(attr,conventions_attrname,conventions_attrvalue);
3631 this->root_attrs.push_back(attr);
3632 }
3633 }
3634
3635}
3636
3637
3638
3639// Adjust Dimension name
3641{
3642
3643 BESDEBUG("h5", "Coming to Adjust_Dim_Name()"<<endl);
3644 // No need if this is following COARDS.
3645 if (false == this->iscoard)
3646 return;
3647 else {
3648 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); irv++) {
3649 if ((*irv)->dims.size() != 1)
3650 throw3("Coard coordinate variable ", (*irv)->name, "is not 1D");
3651 if ((*irv)->newname != (((*irv)->dims)[0]->newname)) {
3652 ((*irv)->dims)[0]->newname = (*irv)->newname;
3653
3654 // For all variables that have this dimension,the dimension newname should also change.
3655 for (auto irv2 = this->vars.begin(); irv2 != this->vars.end(); irv2++) {
3656 for (auto ird = (*irv2)->dims.begin(); ird != (*irv2)->dims.end(); ird++) {
3657 // This is the key, the dimension name of this dimension
3658 // should be equal to the dimension name of the coordinate variable.
3659 // Then the dimension name matches and the dimension name should be changed to
3660 // the new dimension name.
3661 if ((*ird)->name == ((*irv)->dims)[0]->name) (*ird)->newname = ((*irv)->dims)[0]->newname;
3662 }
3663 }
3664 } // if ((*irv)->newname != (((*irv)->dims)[0]->newname))
3665 } // for (auto irv = this->cvars.begin();
3666 } // else
3667}
3668
3669// Add supplemental attributes such as origname and fullpath.
3671{
3672
3673 BESDEBUG("h5", "Coming to Add_Supplement_Attrs()"<<endl);
3674 if (true == add_path) {
3675
3677
3678 // Adding variable original name(origname) and full path(fullpath)
3679 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
3680 if (((*irv)->cvartype == CV_EXIST) || ((*irv)->cvartype == CV_MODIFY)) {
3681 auto attr = new Attribute();
3682 const string varname = (*irv)->name;
3683 const string attrname = "origname";
3684 Add_Str_Attr(attr, attrname, varname);
3685 (*irv)->attrs.push_back(attr);
3686 }
3687 }
3688
3689 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
3690 if (((*irv)->cvartype == CV_EXIST) || ((*irv)->cvartype == CV_MODIFY)) {
3691 // Turn off the fullnamepath attribute when zero_storage_size is 0.
3692 // Use the BES key since quite a few testing cases will be affected.
3693 // KY 2020-03-23
3694 if((*irv)->zero_storage_size==false
3695 || HDF5RequestHandler::get_no_zero_size_fullnameattr() == false) {
3696 auto attr = new Attribute();
3697 const string varname = (*irv)->fullpath;
3698 const string attrname = "fullnamepath";
3699 Add_Str_Attr(attr, attrname, varname);
3700 (*irv)->attrs.push_back(attr);
3701 }
3702 }
3703 }
3704 } // if(true == add_path)
3705
3706 if (true == this->iscoard) {
3707 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
3708 if (((*irv)->cvartype == CV_EXIST) || ((*irv)->cvartype == CV_MODIFY)) {
3709 auto attr = new Attribute();
3710 const string attrname = "orig_dimname";
3711 string orig_dimname = (((*irv)->dims)[0])->name;
3712 orig_dimname = HDF5CFUtil::obtain_string_after_lastslash(orig_dimname);
3713 if ("" == orig_dimname)
3714 throw2("wrong dimension name ", orig_dimname);
3715 if (orig_dimname.find("FakeDim") != string::npos) orig_dimname = "";
3716 Add_Str_Attr(attr, attrname, orig_dimname);
3717 (*irv)->attrs.push_back(attr);
3718 }
3719 } // for (auto irv = this->cvars.begin()
3720
3721 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
3722
3723 if ((*irv)->dims.empty() == false) {
3724 auto attr = new Attribute();
3725 if (1 == (*irv)->dims.size()) {
3726 const string attrname = "orig_dimname";
3727 string orig_dimname = (((*irv)->dims)[0])->name;
3728 if ("" == orig_dimname)
3729 orig_dimname = "NoDimName";
3730 else
3731 orig_dimname = HDF5CFUtil::obtain_string_after_lastslash(orig_dimname);
3732 if (orig_dimname.find("FakeDim") != string::npos) orig_dimname = "NoDimName";
3733 Add_Str_Attr(attr, attrname, orig_dimname);
3734 }
3735 else {
3736 const string attrname = "orig_dimname_list";
3737 string orig_dimname_list;
3738 for (auto ird = (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
3739 string orig_dimname = (*ird)->name;
3740 if ("" == orig_dimname)
3741 orig_dimname = "NoDimName";
3742 else
3743 orig_dimname = HDF5CFUtil::obtain_string_after_lastslash((*ird)->name);
3744 if (orig_dimname.find("FakeDim") != string::npos) orig_dimname = "NoDimName";
3745 if ("" == orig_dimname_list)
3746 orig_dimname_list = orig_dimname;
3747 else
3748 orig_dimname_list = orig_dimname_list + " " + orig_dimname;
3749#if 0
3750// orig_dimname_list = orig_dimname_list + " ";
3751#endif
3752 }
3753 Add_Str_Attr(attr, attrname, orig_dimname_list);
3754 }
3755 (*irv)->attrs.push_back(attr);
3756 }
3757 } // for (auto irv = this->vars.begin();
3758 } // if(true == this->iscoard )
3759
3760}
3761
3762// Handle coordinate attributes.
3764{
3765
3766 BESDEBUG("h5", "Coming to Handle_Coor_Attr()"<<endl);
3767 string co_attrname = "coordinates";
3768 string co_attrvalue = "";
3769
3770 if (iscoard) return;
3771
3772 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
3773
3774 for (auto ird = (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
3775 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ++ircv) {
3776 if ((*ird)->name == (*ircv)->cfdimname)
3777 co_attrvalue = (co_attrvalue.empty()) ? (*ircv)->newname : co_attrvalue + " " + (*ircv)->newname;
3778 }
3779 }
3780 if (false == co_attrvalue.empty()) {
3781 auto attr = new Attribute();
3782 Add_Str_Attr(attr, co_attrname, co_attrvalue);
3783 (*irv)->attrs.push_back(attr);
3784 }
3785 co_attrvalue.clear();
3786 } // for (auto irv = this->vars.begin(); ...
3787
3788 // We will check if 2dlatlon coordinate variables exist
3789 bool has_2dlatlon_cv = false;
3790 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ++ircv) {
3791 if (true == (*ircv)->is_2dlatlon) {
3792 has_2dlatlon_cv = true;
3793 break;
3794 }
3795 }
3796
3797 if (true == has_2dlatlon_cv) {
3798
3799 string dimname1;
3800 string dimname2;
3801 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ++ircv) {
3802 if (true == (*ircv)->is_2dlatlon) {
3803 dimname1 = (((*ircv)->dims)[0])->name;
3804 dimname2 = (((*ircv)->dims)[1])->name;
3805 break;
3806 }
3807 }
3808
3809 int num_latlondims = 0;
3810
3811 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
3812 for (auto ird = (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
3813 if (dimname1 == (*ird)->name) num_latlondims++;
3814 if (dimname2 == (*ird)->name) num_latlondims++;
3815 }
3816 if ((num_latlondims != 0) && (num_latlondims != 2)) {
3817 // need to remove the coordinates attribute.
3818 for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
3819 if (co_attrname == (*ira)->name) {
3820 delete (*ira);
3821 (*irv)->attrs.erase(ira);
3822 break;
3823 }
3824 }
3825 }
3826 num_latlondims = 0;
3827 } // for (auto irv = this->vars.begin();
3828 } // if (true == has_2dlatlon_cv)
3829}
3830
3831// This function is from the original requirement of NASA, then
3832// NASA changes the requirment. Still leave it here for future usage.
3833#if 0
3834void EOS5File::Adjust_Special_EOS5CVar_Name() {
3835
3836 int num_grids =this->eos5cfgrids.size();
3837 int num_swaths = this->eos5cfswaths.size();
3838 int num_zas = this->eos5cfzas.size();
3839
3840 bool mixed_eos5typefile = false;
3841
3842 // Check if this file mixes grid,swath and zonal average
3843 if (((num_grids > 0) && (num_swaths > 0)) ||
3844 ((num_grids > 0) && (num_zas > 0)) ||
3845 ((num_swaths >0) && (num_zas > 0)))
3846 mixed_eos5typefile = true;
3847
3848 if (false == mixed_eos5typefile) {
3849
3850 // Grid is very special since all grids may share the same lat/lon.
3851 // so we also consider this case.
3852
3853 if ((1 == num_swaths) || ( 1 == num_zas) ||
3854 (1 == num_grids) || ((num_grids >1) && (this->grids_multi_latloncvs))) {
3855
3856 string unit_attrname = "units";
3857 string nonll_cf_level_attralue ="level";
3858 string lat_cf_unit_attrvalue ="degrees_north";
3859 string lon_cf_unit_attrvalue ="degrees_east";
3860
3861 for (auto irv = this->cvars.begin();
3862 irv != this->cvars.end(); irv++) {
3863 switch((*irv)->eos_type) {
3864 case CV_EXIST:
3865 case CV_MODIFY:
3866 case CV_LAT_MISS:
3867 case CV_LON_MISS:
3868 {
3869 for(vector <Attribute*>::iterator ira = (*irv)->attrs.begin();
3870 ira != (*irv)->attrs.end(); ira++) {
3871 if ((*ira)->name ==unit_attrname) {
3872 if ((*ira)->value.size() > 0) {
3873 string units_value((*ira)->value.begin(),(*ira)->value.end());
3874 if (lat_cf_unit_attrvalue ==units_value) (*irv)->newname = "lat";
3875 if (lon_cf_unit_attrvalue ==units_value) (*irv)->newname = "lon";
3876 }
3877 }
3878 }
3879 }
3880 break;
3881 case CV_NONLATLON_MISS:
3882 {
3883 for(vector <Attribute*>::iterator ira = (*irv)->attrs.begin();
3884 ira != (*irv)->attrs.end(); ira++) {
3885 if ((*ira)->name ==unit_attrname) {
3886 if ((*ira)->value.size() > 0) {
3887 string units_value((*ira)->value.begin(),(*ira)->value.end());
3888 if (nonll_cf_level_attralue ==units_value) {
3889 (*irv)->newname = "lev";
3890 break;
3891 }
3892 }
3893 }
3894 }
3895 }
3896 break;
3897 default:
3898 throw1("Non-supported coordinate variable type");
3899 }
3900 }
3901 }
3902 }
3903}
3904#endif
3905
3906// Create missing coordinate variables. Some NASA files don't provide coordinate
3907// variables for some dimensions. To make the visualization tools plot the data,
3908// we provide index number as coordinate variable values for these missing coordinate variables.
3909// These missing coordinate variables are all 1-D.
3910template<class T>
3911void EOS5File::Create_Missing_CV(T* eos5data, EOS5CVar *EOS5cvar, const string& dimname, EOS5Type eos5type,
3912 int num_eos5data)
3913{
3914
3915 BESDEBUG("h5", "Coming to Create_Missing_CV()"<<endl);
3916 string reduced_dimname = HDF5CFUtil::obtain_string_after_lastslash(dimname);
3917 if ("" == reduced_dimname) throw2("wrong dimension name ", dimname);
3918 EOS5cvar->name = reduced_dimname;
3919 Create_Added_Var_NewName_FullPath(eos5type, eos5data->name, EOS5cvar->name, EOS5cvar->newname, EOS5cvar->fullpath);
3920 EOS5cvar->rank = 1;
3921 EOS5cvar->dtype = H5INT32;
3922 hsize_t eos5cvar_dimsize = (eos5data->dimnames_to_dimsizes)[dimname];
3923 auto eos5cvar_dim = new Dimension(eos5cvar_dimsize);
3924 eos5cvar_dim->name = dimname;
3925 eos5cvar_dim->unlimited_dim = (eos5data->dimnames_to_unlimited)[dimname];
3926 if (1 == num_eos5data)
3927 eos5cvar_dim->newname = reduced_dimname;
3928 else
3929 eos5cvar_dim->newname = dimname;
3930
3931 EOS5cvar->dims.push_back(eos5cvar_dim);
3932 EOS5cvar->cfdimname = dimname;
3933 EOS5cvar->cvartype = CV_NONLATLON_MISS;
3934 EOS5cvar->eos_type = eos5type;
3935}
3936
3937// Helper function for Create_Missing_CV
3938void EOS5File::Create_Added_Var_NewName_FullPath(EOS5Type eos5type, const string& eos5_groupname, const string& varname,
3939 string &var_newname, string &var_fullpath)
3940{
3941
3942 BESDEBUG("h5", "Coming to Create_Added_Var_NewName_FullPath()"<<endl);
3943 string fslash_str = "/";
3944 string eos5typestr = "";
3945 string top_eos5_groupname = "/HDFEOS";
3946
3947 switch (eos5type) {
3948 case GRID: {
3949 eos5typestr = "/GRIDS/";
3950 var_newname = eos5typestr + eos5_groupname + fslash_str + varname;
3951 var_fullpath = top_eos5_groupname + eos5typestr + eos5_groupname + fslash_str + varname;
3952 }
3953 break;
3954
3955 case SWATH: {
3956 eos5typestr = "/SWATHS/";
3957 var_newname = eos5typestr + eos5_groupname + fslash_str + varname;
3958 var_fullpath = top_eos5_groupname + eos5typestr + eos5_groupname + fslash_str + varname;
3959
3960 }
3961 break;
3962
3963 case ZA: {
3964 eos5typestr = "/ZAS/";
3965 var_newname = eos5typestr + eos5_groupname + fslash_str + varname;
3966 var_fullpath = top_eos5_groupname + eos5typestr + eos5_groupname + fslash_str + varname;
3967
3968 }
3969 break;
3970 case OTHERVARS:
3971 default:
3972 throw1("Non-supported EOS type");
3973 }
3974}
3975
3976// Handle special variables, various speical cases are handled here.
3978{
3979
3980 BESDEBUG("h5", "Coming to Handle_SpVar()"<<endl);
3981 if (true == this->isaura && TES == this->aura_name) {
3982 const string ProHist_full_path = "/HDFEOS/ADDITIONAL/FILE_ATTRIBUTES/ProductionHistory";
3983 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
3984 if (ProHist_full_path == (*irv)->fullpath) {
3985 delete (*irv);
3986 this->vars.erase(irv);
3987 break;
3988 }
3989 }
3990 }
3991
3992 // First, if the duplicate dimension exists,
3993 if (dimname_to_dupdimnamelist.empty() == false) {
3994 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ircv++) {
3995 if ((*ircv)->cvartype == CV_EXIST) {
3996 pair<multimap<string, string>::iterator, multimap<string, string>::iterator> mm_er_ret;
3997 multimap<string, string>::iterator itmm;
3998 for (itmm = dimname_to_dupdimnamelist.begin(); itmm != dimname_to_dupdimnamelist.end(); ++itmm) {
3999
4000 // Find the original dimension(the coordinate variable)
4001 if ((*ircv)->cfdimname == (*itmm).first) {
4002
4003 // Loop through the cv again,this time just check CV_NONLATLON_MISS
4004 for (auto irv2 = this->cvars.begin(); irv2 != this->cvars.end();
4005 irv2++) {
4006 if ((*irv2)->cvartype == CV_NONLATLON_MISS) {
4007#if 0
4008 //cerr<<"the duplicate cf dimension name "<<(*irv2)->cfdimname <<endl;
4009 //if((*irv2)->cfdimname == (*ircv)->cfdimname) {
4010#endif
4011 // Obtain the fake CV that has the duplicate dimension.
4012 if ((*irv2)->cfdimname == (*itmm).second) {
4013
4014 //find the duplicate dimension name
4015 string dup_var_name = (*irv2)->newname;
4016 Replace_Var_Info_EOS((*ircv), (*irv2));
4017 // The following two lines are key to make sure duplicate CV
4018 // using a different name but keep all other info.
4019 (*irv2)->newname = dup_var_name;
4020 (*irv2)->getDimensions()[0]->newname = dup_var_name;
4021 }
4022 }
4023 }
4024 }
4025 }
4026 }
4027 }
4028 }
4029
4030 // No need to loop through the variables. We just need to loop through the coordinate variables and check cfdimname.
4031#if 0
4032 // For the EOS case, Loop through every variable that has a >=2 rank,
4033 for (auto irv = this->vars.begin();
4034 irv != this->vars.end(); ++irv) {
4035
4036 // Check if having the duplicate dimensions.
4037 if((*irv)->rank >=2) {
4038 // Loop through the dimensions
4039 for (auto ird = (*irv)->dims.begin();
4040 ird != (*irv)->dims.end(); ++ ird) {
4041 pair<multimap<string,string>::iterator,multimap<string,string>::iterator> mm_er_ret;
4042 multimap<string,string>::iterator itmm;
4043 for (itmm = dimname_to_dupdimnamelist.begin(); itmm!=dimname_to_dupdimnamelist.end();++itmm) {
4044//cerr<<"the original dim. name is "<<(*itmm).first <<endl;
4045//cerr<<"the duplicate dim. name is "<<(*itmm).second <<endl;
4046//if((*irv)->name == "RetrievalAveragingKernelMatrixDay")
4047 cerr<<"duplicate dimension name of a variable is "<<(*ird)->name <<endl;
4048 // Find the duplicated dim name in the dimname_to_dupdimnamelist,
4049 // Now retrieve the dim. name and loop through all CV_EXIST variable to see if
4050 // one CV_EXIST variable has a dimension of which name is the dim. name.
4051 // If yes, loop through all CV_NONLLMISS variables and find the CV variable that has the
4052 // duplicate dim. name. If found, replace this variable's information(except name and newname) with the
4053 // fullpath of the CV_EXIST variable. In this way, the duplicate CV variable will read
4054 // correctly the existing CV values and other information. This is the most complicate process.
4055
4056// if((*itmm).second == HDF5CFUtil::obtain_string_after_lastslash((*ird)->name)) {
4057 if((*itmm).second == (*ird)->name) {
4058 cerr<<"coming to find the duplicate dim. name "<<endl;
4059 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ircv++) {
4060 if((*ircv)->cvartype == CV_EXIST) {
4061 cerr<<"cf dim. name is "<<(*ircv)->cfdimname <<endl;
4062 // Find the original dimension(the coordinate variable)
4063 if((*ircv)->cfdimname == (*itmm).first) {
4064 // Loop through the cv again,this time just check CV_NONLATLON_MISS
4065 for (auto irv2 = this->cvars.begin();
4066 irv2 != this->cvars.end(); irv2++) {
4067 if((*irv2)->cvartype == CV_NONLATLON_MISS) {
4068 // Obtain the fake CV that has the duplicate dimension.
4069 if((*irv2)->cfdimname == (*itmm).second) {
4070 string dup_var_name = (*irv2)->newname;
4071 Replace_Var_Info_EOS((*ircv),(*irv2));
4072 (*irv2)->newname = dup_var_name;
4073 (*irv2)->getDimensions()[0]->newname = dup_var_name;
4074 }
4075
4076 }
4077 }
4078
4079 }
4080
4081 }
4082
4083 }
4084
4085 }
4086
4087 }
4088
4089 }
4090
4091 }
4092 }
4093
4094}
4095#endif
4096}
4097
4098// Handle special variable attributes
4100{
4101
4102 BESDEBUG("h5", "Coming to Handle_SpVar_Attr()"<<endl);
4103
4104 // First, if the duplicate dimension exists,
4105 if (dimname_to_dupdimnamelist.empty() == false) {
4106
4107 pair<multimap<string, string>::iterator, multimap<string, string>::iterator> mm_er_ret;
4108 multimap<string, string>::iterator itmm;
4109 for (itmm = dimname_to_dupdimnamelist.begin(); itmm != dimname_to_dupdimnamelist.end(); ++itmm) {
4110 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ircv++) {
4111 // The duplicated CV must share with an existing coordinate variable
4112 if ((*ircv)->cvartype == CV_EXIST) {
4113
4114 // Find the original dimension(the coordinate variable)
4115 if ((*ircv)->cfdimname == (*itmm).first) {
4116
4117 // Loop through the cv again,this time just check CV_NONLATLON_MISS
4118 // The duplciated CV must be CV_NONLATLON_MISS.
4119 for (auto irv2 = this->cvars.begin(); irv2 != this->cvars.end();
4120 irv2++) {
4121 if ((*irv2)->cvartype == CV_NONLATLON_MISS) {
4122
4123 // Obtain the fake CV that has the duplicate dimension.
4124#if 0
4125 //if((*irv2)->cfdimname == (*ircv)->cfdimname)
4126#endif
4127 if ((*irv2)->cfdimname == (*itmm).second) Replace_Var_Attrs_EOS((*ircv), (*irv2));
4128
4129 }
4130 }
4131 } // if((*ircv)->cfdimname == (*itmm).first)
4132 } // if((*ircv)->cvartype == CV_EXIST)
4133 } // for (auto ircv = this->cvars.begin()
4134 } // for (itmm = dimname_to_dupdimnamelist.begin();
4135 } // if(dimname_to_dupdimnamelist.size() > 0)
4136}
4137
4138// Handle special variables, various speical cases are handled here.
4140{
4141
4142 BESDEBUG("h5", "Coming to Handle_SpVar_DMR()"<<endl);
4143 if (true == this->isaura && TES == this->aura_name) {
4144 const string ProHist_full_path = "/HDFEOS/ADDITIONAL/FILE_ATTRIBUTES/ProductionHistory";
4145 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
4146 if (ProHist_full_path == (*irv)->fullpath) {
4147 delete (*irv);
4148 this->vars.erase(irv);
4149 break;
4150 }
4151 }
4152 }
4153
4154 // First, if the duplicate dimension exists,
4155 if (dimname_to_dupdimnamelist.size() > 0) {
4156 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ircv++) {
4157 if ((*ircv)->cvartype == CV_EXIST) {
4158 pair<multimap<string, string>::iterator, multimap<string, string>::iterator> mm_er_ret;
4159 multimap<string, string>::iterator itmm;
4160 for (itmm = dimname_to_dupdimnamelist.begin(); itmm != dimname_to_dupdimnamelist.end(); ++itmm) {
4161
4162 // Find the original dimension(the coordinate variable)
4163 if ((*ircv)->cfdimname == (*itmm).first) {
4164
4165 // Loop through the cv again,this time just check CV_NONLATLON_MISS
4166 for (auto irv2 = this->cvars.begin(); irv2 != this->cvars.end();
4167 irv2++) {
4168 if ((*irv2)->cvartype == CV_NONLATLON_MISS) {
4169#if 0
4170 //cerr<<"the duplicate cf dimension name "<<(*irv2)->cfdimname <<endl;
4171 //if((*irv2)->cfdimname == (*ircv)->cfdimname) {
4172#endif
4173 // Obtain the fake CV that has the duplicate dimension.
4174 if ((*irv2)->cfdimname == (*itmm).second) {
4175
4176 Replace_Var_Attrs_EOS((*ircv), (*irv2));
4177 //find the duplicate dimension name
4178 string dup_var_name = (*irv2)->newname;
4179 Replace_Var_Info_EOS((*ircv), (*irv2));
4180
4181 // The following two lines are key to make sure duplicate CV
4182 // using a different name but keep all other info.
4183 (*irv2)->newname = dup_var_name;
4184 (*irv2)->getDimensions()[0]->newname = dup_var_name;
4185 }
4186 }
4187 }
4188 }
4189 }
4190 }
4191 }
4192 }
4193
4194}
4195
4196
4198{
4199 //Intentionally unimplemented, may have use cases for the future.
4200}
4201
4208
4209
4210// Sometimes need to replace informaton of a variable with the information of another variable.
4211void EOS5File::Replace_Var_Info_EOS(EOS5CVar *src, EOS5CVar*target)
4212{
4213
4214 BESDEBUG("h5", "Coming to Replace_Var_Info_EOS()"<<endl);
4215 File::Replace_Var_Info(src, target);
4216 target->cfdimname = src->cfdimname;
4217 target->cvartype = src->cvartype;
4218 target->eos_type = src->eos_type;
4219 target->total_elems = src->total_elems;
4220
4221}
4222
4223//Sometimes the attributes of a variable need to replace with the attribute of another variable.
4224void EOS5File::Replace_Var_Attrs_EOS(EOS5CVar *src, EOS5CVar*target)
4225{
4226
4227 BESDEBUG("h5", "Coming to Replace_Var_Attrs_EOS()"<<endl);
4228 File::Replace_Var_Attrs(src, target);
4229
4230}
4231
4232#if 0
4233void
4234EOS5File:: add_ignored_info_attrs(bool is_grp,bool is_first) {
4235
4236}
4237void
4238EOS5File:: add_ignored_info_objs(bool is_dim_related, bool is_first) {
4239
4240}
4241#endif
4242
This class specifies the core engineering of mapping HDF5 to DAP by following CF.
include the entry functions to execute the handlers
This class represents one attribute.
Definition HDF5CF.h:189
This class repersents one dimension of an HDF5 dataset(variable).
Definition HDF5CF.h:145
This class simulates an HDF-EOS5 Grid. Currently only geographic projection is supported.
Definition HDF5CF.h:1064
This class simulates an HDF-EOS5 Swath.
Definition HDF5CF.h:1112
This class simulates an HDF-EOS5 Zonal average object.
Definition HDF5CF.h:1140
This class is a derived class of CVar. It represents a coordinate variable for HDF-EOS5 files.
Definition HDF5CF.h:431
void Adjust_Obj_Name() override
This method is a no-op operation. Leave here since the method in the base class is pure virtual.
void Add_EOS5File_Info(HE5Parser *, bool)
Add HDF-EOS5 dimension and coordinate variable related info. to EOS5Grid,EOS5Swath etc.
Definition HDFEOS5CF.cc:838
void Handle_Grid_Mapping_Vars() override
Handle Grid Mapping Vars.
void Retrieve_H5_Info(const char *path, hid_t file_id, bool include_attr) override
Retrieve DDS information from the HDF5 file; a real implementation for HDF-EOS5 products.
Definition HDFEOS5CF.cc:161
void Add_Supplement_Attrs(bool) override
Add the supplemental attributes for HDF-EOS5 products.
void Set_COARDS_Status()
Set COARDS flag.
void Adjust_Var_Dim_NewName_Before_Flattening()
Adjust variable dimension names before the flattening for HDF-EOS5 files.
void Handle_CVar() override
Handle coordinate variable for HDF-EOS5 files.
void Adjust_Attr_Info()
Adjust the attribute info for HDF-EOS5 products.
void Handle_SpVar_Attr() override
Handle special variables for HDF-EOS5 files.
void Handle_Obj_NameClashing(bool)
Handle the object name clashing for HDF-EOS5 products.
void Handle_Unsupported_Dtype(bool) override
Handle unsupported HDF5 datatypes for HDF-EOS5 products.
Definition HDFEOS5CF.cc:207
bool Have_Grid_Mapping_Attrs() override
Check if having Grid Mapping Attrs.
void Handle_DimNameClashing() override
void Handle_Unsupported_Others(bool) override
Handle other unmapped objects/attributes for HDF-EOS5 products.
Definition HDFEOS5CF.cc:360
void Retrieve_H5_CVar_Supported_Attr_Values() override
Retrieve coordinate variable attributes.
Definition HDFEOS5CF.cc:168
void Handle_SpVar() override
Handle special variables for HDF-EOS5 files.
void Adjust_Var_NewName_After_Parsing()
Adjust variable names for HDF-EOS5 files.
void Retrieve_H5_Supported_Attr_Values() override
Retrieve attribute values for the supported HDF5 datatypes for HDF-EOS5 products.
Definition HDFEOS5CF.cc:184
void Add_Dim_Name(HE5Parser *)
Add the dimension name for HDF-EOS5 files.
void Adjust_Dim_Name() override
Adjust the dimension name for HDF-EOS5 products.
void Handle_SpVar_DMR() override
Handle special variables and attributes for HDF-EOS5 files(for DMR)
void Handle_Unsupported_Dspace(bool) override
Handle unsupported HDF5 dataspaces for HDF-EOS5 products.
Definition HDFEOS5CF.cc:301
void Flatten_Obj_Name(bool include_attr) override
Flatten the object name for HDF-EOS5 files.
void Check_Aura_Product_Status()
Check if the HDF-EOS5 file is an Aura file. Special CF operations need to be used.
void Adjust_EOS5Dim_Info(HE5Parser *strmeta_info)
Adjust HDF-EOS5 dimension information.
Definition HDFEOS5CF.cc:537
void Handle_Coor_Attr() override
Handle the coordinates attribute for HDF-EOS5 products.
std::vector< Group * > groups
Non-root group vectors.
Definition HDF5CF.h:795
virtual void Handle_Unsupported_Dspace(bool)
Handle unsupported HDF5 dataspaces for datasets.
Definition HDF5CF.cc:1273
virtual void Handle_Grid_Mapping_Vars()
Handle Grid Mapping Vars.
Definition HDF5CF.cc:2208
virtual void Retrieve_H5_Supported_Attr_Values()
Retrieve attribute values for the supported HDF5 datatypes.
Definition HDF5CF.cc:727
std::vector< Var * > vars
Var vectors.
Definition HDF5CF.h:789
virtual void Add_Supplement_Attrs(bool)
Add supplemental attributes such as fullpath and original name.
Definition HDF5CF.cc:2027
virtual void Retrieve_H5_Info(const char *path, hid_t file_id, bool)
Definition HDF5CF.cc:168
std::vector< Attribute * > root_attrs
Root attribute vectors.
Definition HDF5CF.h:792
virtual void Handle_Unsupported_Dtype(bool)
Handle unsupported HDF5 datatypes.
Definition HDF5CF.cc:916
virtual void Flatten_Obj_Name(bool)
Flatten the object name.
Definition HDF5CF.cc:1370
virtual bool Have_Grid_Mapping_Attrs()
Check if having Grid Mapping Attrs.
Definition HDF5CF.cc:2188
This class represents an HDF5 group. The group will be flattened according to the CF conventions.
Definition HDF5CF.h:525
This class represents one HDF5 dataset(CF variable)
Definition HDF5CF.h:256
Helper functions for generating DAS attributes and a function to check BES Key.
Definition HE5Dim.h:7
double point_right
The rightmost coordinate value of a Grid.
Definition HE5Grid.h:25
double point_upper
The top coordinate value of a Grid.
Definition HE5Grid.h:21
double point_left
The leftmost coordinate value of a Grid.
Definition HE5Grid.h:23
double point_lower
The bottom coordinate value of a Grid.
Definition HE5Grid.h:19
Definition HE5Var.h:8
Definition HE5Za.h:6