bes Updated for version 3.20.10
ArrayAggregateOnOuterDimension.cc
1
2// This file is part of the "NcML Module" project, a BES module designed
3// to allow NcML files to be used to be used as a wrapper to add
4// AIS to existing datasets of any format.
5//
6// Copyright (c) 2010 OPeNDAP, Inc.
7// Author: Michael Johnson <m.johnson@opendap.org>
8//
9// For more information, please also see the main website: http://opendap.org/
10//
11// This library is free software; you can redistribute it and/or
12// modify it under the terms of the GNU Lesser General Public
13// License as published by the Free Software Foundation; either
14// version 2.1 of the License, or (at your option) any later version.
15//
16// This library is distributed in the hope that it will be useful,
17// but WITHOUT ANY WARRANTY; without even the implied warranty of
18// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19// Lesser General Public License for more details.
20//
21// You should have received a copy of the GNU Lesser General Public
22// License along with this library; if not, write to the Free Software
23// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24//
25// Please see the files COPYING and COPYRIGHT for more information on the GLPL.
26//
27// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
29
30#include "config.h"
31
32#include "ArrayAggregateOnOuterDimension.h"
33#include "AggregationException.h"
34
35#include <libdap/DataDDS.h> // libdap::DataDDS
36#include <libdap/Marshaller.h>
37
38// only NCML backlinks we want in this agg_util class.
39#include "NCMLDebug.h" // BESDEBUG and throw macros
40#include "NCMLUtil.h" // SAFE_DELETE, NCMLUtil::getVariableNoRecurse
41#include "BESDebug.h"
42#include "BESStopWatch.h"
43
44// BES debug channel we output to
45static const string DEBUG_CHANNEL("agg_util");
46
47// Local flag for whether to print constraints, to help debugging
48static const bool PRINT_CONSTRAINTS = false;
49
50extern BESStopWatch *bes_timing::elapsedTimeToReadStart;
51extern BESStopWatch *bes_timing::elapsedTimeToTransmitStart;
52
53// Timeouts are now handled in/by the BES framework in BESInterface.
54// jhrg 12/29/15
55#undef USE_LOCAL_TIMEOUT_SCHEME
56
57namespace agg_util {
58
60 const AMDList& memberDatasets, std::auto_ptr<ArrayGetterInterface>& arrayGetter, const Dimension& newDim) :
61 ArrayAggregationBase(proto, memberDatasets, arrayGetter) // no new dim yet in super chain
62 , _newDim(newDim)
63{
64 BESDEBUG(DEBUG_CHANNEL, "ArrayAggregateOnOuterDimension: ctor called!" << endl);
65
66 // Up the rank of the array using the new dimension as outer (prepend)
67 BESDEBUG(DEBUG_CHANNEL, "ArrayAggregateOnOuterDimension: adding new outer dimension: " << _newDim.name << endl);
68 prepend_dim(_newDim.size, _newDim.name);
69}
70
72 ArrayAggregationBase(proto), _newDim()
73{
74 BESDEBUG(DEBUG_CHANNEL, "ArrayAggregateOnOuterDimension() copy ctor called!" << endl);
75 duplicate(proto);
76}
77
79{
80 BESDEBUG(DEBUG_CHANNEL, "~ArrayAggregateOnOuterDimension() dtor called!" << endl);
81 cleanup();
82}
83
86{
87 return new ArrayAggregateOnOuterDimension(*this);
88}
89
92{
93 if (this != &rhs) {
94 cleanup();
95 ArrayAggregationBase::operator=(rhs);
96 duplicate(rhs);
97 }
98 return *this;
99}
100
101// Set this to 0 to get the old behavior where the entire response
102// (for this variable) is built in memory and then sent to the client.
103#define PIPELINING 1
104
125bool ArrayAggregateOnOuterDimension::serialize(libdap::ConstraintEvaluator &eval, libdap::DDS &dds,
126 libdap::Marshaller &m, bool ce_eval)
127{
128
129 BESStopWatch sw;
130 if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start("ArrayAggregateOnOuterDimension::serialize", "");
131
132 // Only continue if we are supposed to serialize this object at all.
133 if (!(send_p() || is_in_selection())) {
134 BESDEBUG_FUNC(DEBUG_CHANNEL, "Object not in output, skipping... name=" << name() << endl);
135 return true;
136 }
137
138 bool status = false;
139
140 delete bes_timing::elapsedTimeToReadStart;
141 bes_timing::elapsedTimeToReadStart = 0;
142
143 if (!read_p()) {
144
145 if (PRINT_CONSTRAINTS) {
146 BESDEBUG_FUNC(DEBUG_CHANNEL, "Constraints on this Array are:" << endl);
147 printConstraints(*this);
148 }
149
150 // call subclass impl
152
153 if (PRINT_CONSTRAINTS) {
154 BESDEBUG_FUNC(DEBUG_CHANNEL, "After transfer, constraints on the member template Array are: " << endl);
156 }
157
158 // outer one is the first in iteration
159 const Array::dimension& outerDim = *(dim_begin());
160 BESDEBUG(DEBUG_CHANNEL,
161 "Aggregating datasets array with outer dimension constraints: " << " start=" << outerDim.start << " stride=" << outerDim.stride << " stop=" << outerDim.stop << endl);
162
163 // Be extra sure we have enough datasets for the given request
164 if (static_cast<unsigned int>(outerDim.size) != getDatasetList().size()) {
165 // Not sure whose fault it was, but tell the author
166 THROW_NCML_PARSE_ERROR(-1, "The new outer dimension of the joinNew aggregation doesn't "
167 " have the same size as the number of datasets in the aggregation!");
168 }
169
170#if PIPELINING
171 // Prepare our output buffer for our constrained length
172 m.put_vector_start(length());
173#else
174 reserve_value_capacity();
175#endif
176 // this index pointing into the value buffer for where to write.
177 // The buffer has a stride equal to the _pSubArrayProto->length().
178
179 // Keep this to do some error checking
180 int nextElementIndex = 0;
181
182 // Traverse the dataset array respecting hyperslab
183 for (int i = outerDim.start; i <= outerDim.stop && i < outerDim.size; i += outerDim.stride) {
184 AggMemberDataset& dataset = *((getDatasetList())[i]);
185
186 try {
187#if USE_LOCAL_TIMEOUT_SCHEME
188 dds.timeout_on();
189#endif
191 name(), dataset, getArrayGetterInterface(), DEBUG_CHANNEL);
192#if USE_LOCAL_TIMEOUT_SCHEME
193 dds.timeout_off();
194#endif
195#if PIPELINING
196 delete bes_timing::elapsedTimeToTransmitStart;
197 bes_timing::elapsedTimeToTransmitStart = 0;
198 m.put_vector_part(pDatasetArray->get_buf(), getGranuleTemplateArray().length(), var()->width(),
199 var()->type());
200#else
201 this->set_value_slice_from_row_major_vector(*pDatasetArray, nextElementIndex);
202#endif
203
204 pDatasetArray->clear_local_data();
205 }
207 std::ostringstream oss;
208 oss << "Got AggregationException while streaming dataset index=" << i << " data for location=\""
209 << dataset.getLocation() << "\" The error msg was: " << std::string(ex.what());
210 THROW_NCML_PARSE_ERROR(-1, oss.str());
211 }
212
213 // Jump forward by the amount we added.
214 nextElementIndex += getGranuleTemplateArray().length();
215 }
216
217 // If we succeeded, we are at the end of the array!
218 NCML_ASSERT_MSG(nextElementIndex == length(), "Logic error:\n"
219 "ArrayAggregateOnOuterDimension::read(): "
220 "At end of aggregating, expected the nextElementIndex to be the length of the "
221 "aggregated array, but it wasn't!");
222
223#if PIPELINING
224 m.put_vector_end();
225 status = true;
226#else
227 // Set the cache bit to avoid recomputing
228 set_read_p(true);
229
230 delete bes_timing::elapsedTimeToTransmitStart;
231 bes_timing::elapsedTimeToTransmitStart = 0;
232 status = libdap::Array::serialize(eval, dds, m, ce_eval);
233#endif
234 }
235 else {
236 status = libdap::Array::serialize(eval, dds, m, ce_eval);
237 }
238
239 return status;
240}
241
243// helpers
244
245void ArrayAggregateOnOuterDimension::duplicate(const ArrayAggregateOnOuterDimension& rhs)
246{
247 _newDim = rhs._newDim;
248}
249
250void ArrayAggregateOnOuterDimension::cleanup() throw ()
251{
252}
253
254/* virtual */
256{
257 // transfer the constraints from this object into the subArray template
258 // skipping our first dim which is the new one and not in the subArray.
260 *this, // from this
261 true, // skip first dim in the copy since we handle it special
262 false, // also skip it in the toArray for the same reason.
263 true, // print debug
264 DEBUG_CHANNEL); // on this channel
265}
266
267/* virtual */
268// In this version of the code, I broke apart the call to
269// agg_util::AggregationUtil::addDatasetArrayDataToAggregationOutputArray()
270// into two calls: AggregationUtil::readDatasetArrayDataForAggregation()
271// and this->set_value_slice_from_row_major_vector(). This
273{
274 BESStopWatch sw;
275 if (BESDebug::IsSet(TIMING_LOG_KEY))
276 sw.start("ArrayAggregateOnOuterDimension::readConstrainedGranuleArraysAndAggregateDataHook", "");
277
278 // outer one is the first in iteration
279 const Array::dimension& outerDim = *(dim_begin());
280 BESDEBUG(DEBUG_CHANNEL,
281 "Aggregating datasets array with outer dimension constraints: " << " start=" << outerDim.start << " stride=" << outerDim.stride << " stop=" << outerDim.stop << endl);
282
283 // Be extra sure we have enough datasets for the given request
284 if (static_cast<unsigned int>(outerDim.size) != getDatasetList().size()) {
285 // Not sure whose fault it was, but tell the author
286 THROW_NCML_PARSE_ERROR(-1, "The new outer dimension of the joinNew aggregation doesn't "
287 " have the same size as the number of datasets in the aggregation!");
288 }
289
290 // Prepare our output buffer for our constrained length
291 reserve_value_capacity();
292
293 // this index pointing into the value buffer for where to write.
294 // The buffer has a stride equal to the _pSubArrayProto->length().
295 int nextElementIndex = 0;
296
297 // Traverse the dataset array respecting hyperslab
298 for (int i = outerDim.start; i <= outerDim.stop && i < outerDim.size; i += outerDim.stride) {
299 AggMemberDataset& dataset = *((getDatasetList())[i]);
300
301 try {
302 agg_util::AggregationUtil::addDatasetArrayDataToAggregationOutputArray(*this, // into the output buffer of this object
303 nextElementIndex, // into the next open slice
304 getGranuleTemplateArray(), // constraints template
305 name(), // aggvar name
306 dataset, // Dataset who's DDS should be searched
307 getArrayGetterInterface(), DEBUG_CHANNEL);
308#if 0
309 // The code above is conceptually similar to this, but
310 // makes more efficient use of memory. jhrg8/18/15
312 name(), dataset, getArrayGetterInterface(), DEBUG_CHANNEL);
313
314 this->set_value_slice_from_row_major_vector(*pDatasetArray, nextElementIndex);
315#endif
316 }
318 std::ostringstream oss;
319 oss << "Got AggregationException while streaming dataset index=" << i << " data for location=\""
320 << dataset.getLocation() << "\" The error msg was: " << std::string(ex.what());
321 THROW_NCML_PARSE_ERROR(-1, oss.str());
322 }
323
324 // Jump forward by the amount we added.
325 nextElementIndex += getGranuleTemplateArray().length();
326 }
327
328 // If we succeeded, we are at the end of the array!
329 NCML_ASSERT_MSG(nextElementIndex == length(), "Logic error:\n"
330 "ArrayAggregateOnOuterDimension::read(): "
331 "At end of aggregating, expected the nextElementIndex to be the length of the "
332 "aggregated array, but it wasn't!");
333}
334
335}
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
Definition: BESDebug.h:168
virtual bool start(std::string name)
Definition: BESStopWatch.cc:67
const std::string & getLocation() const
static void addDatasetArrayDataToAggregationOutputArray(libdap::Array &oOutputArray, unsigned int atIndex, const libdap::Array &constrainedTemplateArray, const string &varName, AggMemberDataset &dataset, const ArrayGetterInterface &arrayGetter, const string &debugChannel)
static void transferArrayConstraints(libdap::Array *pToArray, const libdap::Array &fromArray, bool skipFirstFromDim, bool skipFirstToDim, bool printDebug=false, const std::string &debugChannel="agg_util")
static libdap::Array * readDatasetArrayDataForAggregation(const libdap::Array &constrainedTemplateArray, const std::string &varName, AggMemberDataset &dataset, const ArrayGetterInterface &arrayGetter, const std::string &debugChannel)
ArrayAggregateOnOuterDimension & operator=(const ArrayAggregateOnOuterDimension &rhs)
ArrayAggregateOnOuterDimension(const libdap::Array &proto, const AMDList &memberDatasets, std::auto_ptr< ArrayGetterInterface > &arrayGetter, const Dimension &newDim)
virtual bool serialize(libdap::ConstraintEvaluator &eval, libdap::DDS &dds, libdap::Marshaller &m, bool ce_eval)
virtual ArrayAggregateOnOuterDimension * ptr_duplicate()
const AMDList & getDatasetList() const
void printConstraints(const Array &fromArray)
const ArrayGetterInterface & getArrayGetterInterface() const
Helper class for temporarily hijacking an existing dhi to load a DDX response for one particular file...