bes Updated for version 3.20.10
NgapContainer.cc
1// NgapContainer.cc
2
3// -*- mode: c++; c-basic-offset:4 -*-
4
5// This file is part of ngap_module, A C++ module that can be loaded in to
6// the OPeNDAP Back-End Server (BES) and is able to handle remote requests.
7
8// Copyright (c) 2020 OPeNDAP, Inc.
9// Author: Nathan Potter <ndp@opendap.org>
10//
11// This library is free software; you can redistribute it and/or
12// modify it under the terms of the GNU Lesser General Public
13// License as published by the Free Software Foundation; either
14// version 2.1 of the License, or (at your option) any later version.
15//
16// This library is distributed in the hope that it will be useful,
17// but WITHOUT ANY WARRANTY; without even the implied warranty of
18// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19// Lesser General Public License for more details.
20//
21// You should have received a copy of the GNU Lesser General Public
22// License along with this library; if not, write to the Free Software
23// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24//
25// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
26// Authors:
27// ndp Nathan Potter <ndp@opendap.org>
28
29#include "config.h"
30
31#include <cstdio>
32#include <map>
33#include <sstream>
34#include <string>
35#include <fstream>
36#include <streambuf>
37#include <time.h>
38
39#include "BESStopWatch.h"
40#include "BESLog.h"
41#include "BESSyntaxUserError.h"
42#include "BESNotFoundError.h"
43#include "BESInternalError.h"
44#include "BESDebug.h"
45#include "BESUtil.h"
46#include "TheBESKeys.h"
47#include "AllowedHosts.h"
48#include "BESContextManager.h"
49#include "CurlUtils.h"
50#include "HttpUtils.h"
51#include "RemoteResource.h"
52#include "url_impl.h"
53
54#include "NgapContainer.h"
55#include "NgapApi.h"
56#include "NgapNames.h"
57
58#define prolog std::string("NgapContainer::").append(__func__).append("() - ")
59
60using namespace std;
61using namespace bes;
62
63namespace ngap {
64
75NgapContainer::NgapContainer(const string &sym_name,
76 const string &real_name,
77 const string &type) :
78 BESContainer(sym_name, real_name, type),
79 d_dmrpp_rresource(0) {
80 initialize();
81}
82
83
84void NgapContainer::initialize()
85{
86 BESDEBUG(MODULE, prolog << "BEGIN (obj_addr: "<< (void *) this << ")" << endl);
87 BESDEBUG(MODULE, prolog << "sym_name: "<< get_symbolic_name() << endl);
88 BESDEBUG(MODULE, prolog << "real_name: "<< get_real_name() << endl);
89 BESDEBUG(MODULE, prolog << "type: "<< get_container_type() << endl);
90
91 bool found;
92
93 NgapApi ngap_api;
94 if (get_container_type().empty())
95 set_container_type("ngap");
96
97 string uid = BESContextManager::TheManager()->get_context(EDL_UID_KEY, found);
98 BESDEBUG(MODULE, prolog << "EDL_UID_KEY(" << EDL_UID_KEY << "): " << uid << endl);
99
100 string data_access_url = ngap_api.convert_ngap_resty_path_to_data_access_url(get_real_name(), uid);
101
102 set_real_name(data_access_url);
103// Because we know the name is really a URL, then we know the "relative_name" is meaningless
104// So we set it to be the same as "name"
105 set_relative_name(data_access_url);
106 BESDEBUG(MODULE, prolog << "END (obj_addr: "<< (void *) this << ")" << endl);
107
108}
109
113NgapContainer::NgapContainer(const NgapContainer &copy_from) :
114 BESContainer(copy_from),
115 d_dmrpp_rresource(copy_from.d_dmrpp_rresource) {
116 BESDEBUG(MODULE, prolog << "BEGIN object address: "<< (void *) this << " Copying from: " << (void *) &copy_from << endl);
117 // we can not make a copy of this container once the request has
118 // been made
119 if (d_dmrpp_rresource) {
120 string err = (string) "The Container has already been accessed, "
121 + "can not create a copy of this container.";
122 throw BESInternalError(err, __FILE__, __LINE__);
123 }
124 BESDEBUG(MODULE, prolog << "object address: "<< (void *) this << endl);
125}
126
127void NgapContainer::_duplicate(NgapContainer &copy_to) {
128 if (copy_to.d_dmrpp_rresource) {
129 string err = (string) "The Container has already been accessed, "
130 + "can not duplicate this resource.";
131 throw BESInternalError(err, __FILE__, __LINE__);
132 }
133 BESDEBUG(MODULE, prolog << "BEGIN object address: "<< (void *) this << " Copying to: " << (void *) &copy_to << endl);
134 copy_to.d_dmrpp_rresource = d_dmrpp_rresource;
136}
137
140 NgapContainer *container = new NgapContainer;
141 _duplicate(*container);
142 BESDEBUG(MODULE, prolog << "object address: "<< (void *) this << " to: " << (void *)container << endl);
143 return container;
144}
145
146NgapContainer::~NgapContainer() {
147 BESDEBUG(MODULE, prolog << "BEGIN object address: "<< (void *) this << endl);
148 if (d_dmrpp_rresource) {
149 release();
150 }
151 BESDEBUG(MODULE, prolog << "END object address: "<< (void *) this << endl);
152}
153
154
155
162 BESDEBUG(MODULE, prolog << "BEGIN (obj_addr: "<< (void *) this << ")" << endl);
163
164 // Since this the ngap we know that the real_name is a URL.
165 string data_access_url_str = get_real_name();
166
167 // And we know that the dmr++ file should "right next to it" (side-car)
168 string dmrpp_url_str = data_access_url_str + ".dmrpp";
169
170 // And if there's a missing data file (side-car) it should be "right there" too.
171 string missing_data_url_str = data_access_url_str + ".missing";
172
173 BESDEBUG(MODULE, prolog << " data_access_url: " << data_access_url_str << endl);
174 BESDEBUG(MODULE, prolog << " dmrpp_url: " << dmrpp_url_str << endl);
175 BESDEBUG(MODULE, prolog << "missing_data_url: " << missing_data_url_str << endl);
176
177 // TODO 10/8/21 Is this a syntax error? Should the \" be after 'true'. jhrg
178 string trusted_url_hack="\" dmrpp:trust=\"true";
179 string data_access_url_with_trusted_attr_str = data_access_url_str + trusted_url_hack;
180 string dmrpp_url_with_trusted_attr_str = dmrpp_url_str + trusted_url_hack;
181 string missing_data_url_with_trusted_attr_str = missing_data_url_str + trusted_url_hack;
182
183 BESDEBUG(MODULE, prolog << " data_access_url_with_trusted_attr_str: " << data_access_url_with_trusted_attr_str << endl);
184 BESDEBUG(MODULE, prolog << " dmrpp_url_with_trusted_attr_str: " << dmrpp_url_with_trusted_attr_str << endl);
185 BESDEBUG(MODULE, prolog << "missing_data_url_with_trusted_attr_str: " << missing_data_url_with_trusted_attr_str << endl);
186
187 string type = get_container_type();
188 if (type == "ngap")
189 type = "";
190
191 if (!d_dmrpp_rresource) {
192 BESDEBUG(MODULE, prolog << "Building new RemoteResource (dmr++)." << endl);
193 map<string,string> content_filters;
194 if (inject_data_url()) {
195 content_filters.insert(pair<string,string>(DATA_ACCESS_URL_KEY,data_access_url_with_trusted_attr_str));
196 content_filters.insert(pair<string,string>(MISSING_DATA_ACCESS_URL_KEY,missing_data_url_with_trusted_attr_str));
197 }
198 shared_ptr<http::url> dmrpp_url(new http::url(dmrpp_url_str, true));
199 {
200 d_dmrpp_rresource = new http::RemoteResource(dmrpp_url);
201 BESStopWatch besTimer;
202 if (BESISDEBUG(MODULE) || BESDebug::IsSet(TIMING_LOG_KEY) || BESLog::TheLog()->is_verbose()){
203 besTimer.start("DMR++ retrieval: "+ dmrpp_url->str());
204 }
205 d_dmrpp_rresource->retrieveResource(content_filters);
206 }
207 BESDEBUG(MODULE, prolog << "Retrieved remote resource: " << dmrpp_url->str() << endl);
208 }
209
210 // TODO This file should be read locked before leaving this method.
211 // 10/8/21 I think the RemoteResource should do that. jhrg
212 string cachedResource = d_dmrpp_rresource->getCacheFileName();
213 BESDEBUG(MODULE, prolog << "Using local cache file: " << cachedResource << endl);
214
215 type = d_dmrpp_rresource->getType();
216 set_container_type(type);
217 BESDEBUG(MODULE, prolog << "Type: " << type << endl);
218 BESDEBUG(MODULE, prolog << "Done retrieving: " << dmrpp_url_str << " returning cached file " << cachedResource << endl);
219 BESDEBUG(MODULE, prolog << "END (obj_addr: "<< (void *) this << ")" << endl);
220
221 return cachedResource; // this should return the dmr++ file name from the NgapCache
222}
223
224
232 // TODO The cache file (that will be) read locked in the access() method must be unlocked here.
233 // If we make that part of the RemoteResource dtor, the unlock will happen here. jhrg
234 if (d_dmrpp_rresource) {
235 BESDEBUG(MODULE, prolog << "Releasing RemoteResource" << endl);
236 delete d_dmrpp_rresource;
237 d_dmrpp_rresource = 0;
238 }
239
240 BESDEBUG(MODULE, prolog << "Done releasing Ngap response" << endl);
241 return true;
242}
243
251void NgapContainer::dump(ostream &strm) const {
252 strm << BESIndent::LMarg << "NgapContainer::dump - (" << (void *) this
253 << ")" << endl;
254 BESIndent::Indent();
255 BESContainer::dump(strm);
256 if (d_dmrpp_rresource) {
257 strm << BESIndent::LMarg << "RemoteResource.getCacheFileName(): " << d_dmrpp_rresource->getCacheFileName()
258 << endl;
259 strm << BESIndent::LMarg << "response headers: ";
260 vector<string> *hdrs = d_dmrpp_rresource->getResponseHeaders();
261 if (hdrs) {
262 strm << endl;
263 BESIndent::Indent();
264 vector<string>::const_iterator i = hdrs->begin();
265 vector<string>::const_iterator e = hdrs->end();
266 for (; i != e; i++) {
267 string hdr_line = (*i);
268 strm << BESIndent::LMarg << hdr_line << endl;
269 }
270 BESIndent::UnIndent();
271 } else {
272 strm << "none" << endl;
273 }
274 } else {
275 strm << BESIndent::LMarg << "response not yet obtained" << endl;
276 }
277 BESIndent::UnIndent();
278}
279
280bool NgapContainer::inject_data_url(){
281 bool result = false;
282 bool found;
283 string key_value;
284 TheBESKeys::TheKeys()->get_value(NGAP_INJECT_DATA_URL_KEY, key_value, found);
285 if (found && key_value == "true") {
286 result = true;
287 }
288 BESDEBUG(MODULE, prolog << "NGAP_INJECT_DATA_URL_KEY(" << NGAP_INJECT_DATA_URL_KEY << "): " << result << endl);
289 return result;
290}
291}
A container is something that holds data. E.G., a netcdf file or a database entry.
Definition: BESContainer.h:65
void set_container_type(const std::string &type)
set the type of data that this container represents, such as cedar or netcdf.
Definition: BESContainer.h:161
std::string get_symbolic_name() const
retrieve the symbolic name for this container
Definition: BESContainer.h:221
virtual void dump(std::ostream &strm) const
dumps information about this object
Definition: BESContainer.cc:73
void set_real_name(const std::string &real_name)
set the real name for this container, such as a file name if reading a data file.
Definition: BESContainer.h:146
std::string get_container_type() const
retrieve the type of data this container holds, such as cedar or netcdf.
Definition: BESContainer.h:232
void set_relative_name(const std::string &relative)
Set the relative name of the object in this container.
Definition: BESContainer.h:152
void _duplicate(BESContainer &copy_to)
duplicate this instance into the passed container
Definition: BESContainer.cc:54
std::string get_real_name() const
retrieve the real name for this container, such as a file name.
Definition: BESContainer.h:180
virtual std::string get_context(const std::string &name, bool &found)
retrieve the value of the specified context from the BES
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
Definition: BESDebug.h:168
exception thrown if internal error encountered
virtual bool start(std::string name)
Definition: BESStopWatch.cc:67
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:340
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:71
std::string getCacheFileName()
std::vector< std::string > * getResponseHeaders()
std::string getType()
std::string convert_ngap_resty_path_to_data_access_url(const std::string &restified_path, const std::string &uid="")
Converts an NGAP restified granule path into a CMR metadata query for the granule.
Definition: NgapApi.cc:423
virtual void dump(std::ostream &strm) const
dumps information about this object
virtual std::string access()
access the remote target response by making the remote request
virtual BESContainer * ptr_duplicate()
pure abstract method to duplicate this instances of BESContainer
virtual bool release()
release the resources