40#include <libdap/util.h>
44#include "BESInternalError.h"
45#include "BESForbiddenError.h"
46#include "AllowedHosts.h"
48#include "DmrppCommon.h"
49#include "DmrppNames.h"
51#include "CurlHandlePool.h"
53#include "CredentialsManager.h"
54#include "AccessCredentials.h"
59#define prolog std::string("CurlHandlePool::").append(__func__).append("() - ")
74string pthread_error(
unsigned int err){
78 error_msg =
"The mutex was either created with the "
79 "protocol attribute having the value "
80 "PTHREAD_PRIO_PROTECT and the calling "
81 "thread's priority is higher than the "
82 "mutex's current priority ceiling."
83 "OR The value specified by mutex does not "
84 "refer to an initialized mutex object.";
88 error_msg =
"The mutex could not be acquired "
89 "because it was already locked.";
93 error_msg =
"The mutex could not be acquired because "
94 "the maximum number of recursive locks "
95 "for mutex has been exceeded.";
99 error_msg =
"The current thread already owns the mutex";
103 error_msg =
"The current thread does not own the mutex.";
107 error_msg =
"Unknown pthread error type.";
114Lock::Lock(pthread_mutex_t &lock) : m_mutex(lock) {
115 int status = pthread_mutex_lock(&m_mutex);
117 throw BESInternalError(prolog +
"Failed to acquire mutex lock. msg: " + pthread_error(status), __FILE__, __LINE__);
121 int status = pthread_mutex_unlock(&m_mutex);
123 ERROR_LOG(prolog +
"Failed to release mutex lock. msg: " + pthread_error(status));
133string dump(
const char *text,
unsigned char *ptr,
size_t size)
137 unsigned int width=0x10;
140 oss << text <<
", " << std::setw(10) << (long)size << std::setbase(16) << (long)size << endl;
142 for(i=0; i<size; i+= width) {
143 oss << std::setw(4) << (long)i;
147 for(c = 0; c < width; c++) {
149 oss << std::setw(2) << ptr[i+c];
159 for(c = 0; (c < width) && (i+c < size); c++) {
160 char x = (ptr[i+c] >= 0x20 && ptr[i+c] < 0x80) ? ptr[i+c] :
'.';
162 oss << std::setw(1) << x;
180int curl_trace(CURL *, curl_infotype type,
char *data,
size_t ,
void *)
186 case CURLINFO_HEADER_OUT:
187 case CURLINFO_HEADER_IN: {
190 while ((pos = text.find(
'\n')) != string::npos)
191 text = text.substr(0, pos);
196 case CURLINFO_DATA_OUT:
197 case CURLINFO_SSL_DATA_OUT:
198 case CURLINFO_DATA_IN:
199 case CURLINFO_SSL_DATA_IN:
207 LOG(
"libcurl == Info: " << text << endl);
210 case CURLINFO_HEADER_OUT:
211 LOG(
"libcurl == Send header: " << text << endl);
213 case CURLINFO_HEADER_IN:
214 LOG(
"libcurl == Recv header: " << text << endl);
218 case CURLINFO_DATA_OUT:
219 case CURLINFO_SSL_DATA_OUT:
220 case CURLINFO_DATA_IN:
221 case CURLINFO_SSL_DATA_IN:
235 d_handle = curl_easy_init();
236 if (!d_handle)
throw BESInternalError(
"Could not allocate CURL handle", __FILE__, __LINE__);
238 curl::set_error_buffer(d_handle, d_errbuf);
240 res = curl_easy_setopt(d_handle, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2);
241 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_SSLVERSION", d_errbuf, __FILE__, __LINE__);
245 res = curl_easy_setopt(d_handle, CURLOPT_DEBUGFUNCTION, curl_trace);
246 curl::check_setopt_result(res, prolog,
"CURLOPT_DEBUGFUNCTION", d_errbuf, __FILE__, __LINE__);
249 res = curl_easy_setopt(d_handle, CURLOPT_VERBOSE, 1L);
250 curl::check_setopt_result(res, prolog,
"CURLOPT_VERBOSE", d_errbuf, __FILE__, __LINE__);
253 res = curl_easy_setopt(d_handle, CURLOPT_HEADERFUNCTION, chunk_header_callback);
254 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_HEADERFUNCTION", d_errbuf, __FILE__, __LINE__);
257 res = curl_easy_setopt(d_handle, CURLOPT_WRITEFUNCTION, chunk_write_data);
258 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_WRITEFUNCTION", d_errbuf, __FILE__, __LINE__);
260#ifdef CURLOPT_TCP_KEEPALIVE
262 res = curl_easy_setopt(d_handle, CURLOPT_TCP_KEEPALIVE, 1L);
263 curl::check_setopt_result(res, prolog,
"CURLOPT_TCP_KEEPALIVE", d_errbuf, __FILE__, __LINE__);
266#ifdef CURLOPT_TCP_KEEPIDLE
268 res = curl_easy_setopt(d_handle, CURLOPT_TCP_KEEPIDLE, 120L);
269 curl::check_setopt_result(res, prolog,
"CURLOPT_TCP_KEEPIDLE", d_errbuf, __FILE__, __LINE__);
272#ifdef CURLOPT_TCP_KEEPINTVL
274 res = curl_easy_setopt(d_handle, CURLOPT_TCP_KEEPINTVL, 120L)
275 curl::check_setopt_result(res, prolog,
"CURLOPT_TCP_KEEPINTVL", d_errbuf, __FILE__, __LINE__);
282dmrpp_easy_handle::~dmrpp_easy_handle() {
283 if (d_handle) curl_easy_cleanup(d_handle);
284 if (d_request_headers) curl_slist_free_all(d_request_headers);
301 if (d_url->protocol() == HTTPS_PROTOCOL || d_url->protocol() == HTTP_PROTOCOL) {
302 curl::super_easy_perform(d_handle);
305 CURLcode curl_code = curl_easy_perform(d_handle);
306 if (CURLE_OK != curl_code) {
307 string msg = prolog +
"ERROR - Data transfer error: ";
308 throw BESInternalError(msg.append(curl::error_message(curl_code, d_errbuf)), __FILE__, __LINE__);
312 d_chunk->set_is_read(
true);
326CurlHandlePool::CurlHandlePool() {
327 d_max_easy_handles = DmrppRequestHandler::d_max_transfer_threads;
329 for (
unsigned int i = 0; i < d_max_easy_handles; ++i) {
332 unsigned int status = pthread_mutex_init(&d_get_easy_handle_mutex, 0);
334 throw BESInternalError(
"Could not initialize mutex in CurlHandlePool. msg: " + pthread_error(status), __FILE__, __LINE__);
338CurlHandlePool::CurlHandlePool() {
346CurlHandlePool::CurlHandlePool(
unsigned int max_handles) : d_max_easy_handles(max_handles) {
347 for (
unsigned int i = 0; i < d_max_easy_handles; ++i) {
351 unsigned int status = pthread_mutex_init(&d_get_easy_handle_mutex, 0);
353 throw BESInternalError(
"Could not initialize mutex in CurlHandlePool. msg: " + pthread_error(status), __FILE__, __LINE__);
368static struct curl_slist *append_http_header(curl_slist *slist,
const string &header,
const string &value) {
369 string full_header = header;
370 full_header.append(
" ").append(value);
372 struct curl_slist *temp = curl_slist_append(slist, full_header.c_str());
396 string reason =
"The requested resource does not match any of the AllowedHost rules.";
399 ss <<
"ERROR! The chunk url "<< chunk->
get_data_url()->str() <<
" was rejected because: " << reason;
403 Lock lock(d_get_easy_handle_mutex);
406 for (
auto i = d_easy_handles.begin(), e = d_easy_handles.end(); i != e; ++i) {
407 if (!(*i)->d_in_use) {
415 handle->d_in_use =
true;
418 handle->d_chunk = chunk;
420 CURLcode res = curl_easy_setopt(handle->d_handle, CURLOPT_URL, chunk->
get_data_url()->str().c_str());
421 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_URL", handle->d_errbuf, __FILE__, __LINE__);
425 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_RANGE", handle->d_errbuf, __FILE__, __LINE__);
428 res = curl_easy_setopt(handle->d_handle, CURLOPT_HEADERDATA,
reinterpret_cast<void *
>(chunk));
429 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_HEADERDATA", handle->d_errbuf, __FILE__, __LINE__);
432 res = curl_easy_setopt(handle->d_handle, CURLOPT_WRITEDATA,
reinterpret_cast<void *
>(chunk));
433 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_WRITEDATA", handle->d_errbuf, __FILE__, __LINE__);
436 res = curl_easy_setopt(handle->d_handle, CURLOPT_PRIVATE,
reinterpret_cast<void *
>(handle));
437 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_PRIVATE", handle->d_errbuf, __FILE__, __LINE__);
440 res = curl_easy_setopt(handle->d_handle, CURLOPT_COOKIEFILE, curl::get_cookie_filename().c_str());
441 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_COOKIEFILE", handle->d_errbuf, __FILE__, __LINE__);
443 res = curl_easy_setopt(handle->d_handle, CURLOPT_COOKIEJAR, curl::get_cookie_filename().c_str());
444 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_COOKIEJAR", handle->d_errbuf, __FILE__, __LINE__);
447 res = curl_easy_setopt(handle->d_handle, CURLOPT_FOLLOWLOCATION, 1);
448 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_FOLLOWLOCATION", handle->d_errbuf, __FILE__, __LINE__);
450 res = curl_easy_setopt(handle->d_handle, CURLOPT_MAXREDIRS, curl::max_redirects());
451 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_MAXREDIRS", handle->d_errbuf, __FILE__, __LINE__);
454 res = curl_easy_setopt(handle->d_handle, CURLOPT_USERAGENT, curl::hyrax_user_agent().c_str());
455 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_USERAGENT", handle->d_errbuf, __FILE__, __LINE__);
460 res = curl_easy_setopt(handle->d_handle, CURLOPT_HTTPAUTH, (
long) CURLAUTH_ANY);
461 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_HTTPAUTH", handle->d_errbuf, __FILE__, __LINE__);
464 res = curl_easy_setopt(handle->d_handle, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
465 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_NETRC", handle->d_errbuf, __FILE__, __LINE__);
469 string netrc_file = curl::get_netrc_filename();
470 if (!netrc_file.empty()) {
471 res = curl_easy_setopt(handle->d_handle, CURLOPT_NETRC_FILE, netrc_file.c_str());
472 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_NETRC_FILE", handle->d_errbuf, __FILE__, __LINE__);
478 if (credentials && credentials->
is_s3_cred()) {
480 prolog <<
"Got AccessCredentials instance: " << endl << credentials->to_json() << endl);
483 const std::time_t request_time = std::time(0);
485 const std::string auth_header =
486 AWSV4::compute_awsv4_signature(
489 credentials->
get(AccessCredentials::ID_KEY),
490 credentials->
get(AccessCredentials::KEY_KEY),
491 credentials->
get(AccessCredentials::REGION_KEY),
495 handle->d_request_headers = curl::append_http_header((curl_slist *)0,
"Authorization", auth_header);
496 handle->d_request_headers = curl::append_http_header(handle->d_request_headers,
"x-amz-content-sha256",
497 "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
498 handle->d_request_headers = curl::append_http_header(handle->d_request_headers,
"x-amz-date", AWSV4::ISO8601_date(request_time));
504 handle->d_request_headers = append_http_header(0,
"Authorization:", auth_header);
505 if (!handle->d_request_headers)
507 string(
"CURL Error setting Authorization header: ").append(
508 curl::error_message(res, handle->d_errbuf)), __FILE__, __LINE__);
511 curl_slist *temp = append_http_header(handle->d_request_headers,
"x-amz-content-sha256:",
512 "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
515 string(
"CURL Error setting x-amz-content-sha256: ").append(
516 curl::error_message(res, handle->d_errbuf)),
518 handle->d_request_headers = temp;
520 temp = append_http_header(handle->d_request_headers,
"x-amz-date:", AWSV4::ISO8601_date(request_time));
523 string(
"CURL Error setting x-amz-date header: ").append(
524 curl::error_message(res, handle->d_errbuf)),
526 handle->d_request_headers = temp;
531 res = curl_easy_setopt(handle->d_handle, CURLOPT_HTTPHEADER, handle->d_request_headers);
532 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_HTTPHEADER", handle->d_errbuf, __FILE__, __LINE__);
553 Lock lock(d_get_easy_handle_mutex);
558 handle->d_url =
nullptr;
560 handle->d_in_use =
false;
564 for (std::vector<dmrpp_easy_handle *>::iterator i = d_easy_handles.begin(), e = d_easy_handles.end(); i != e; ++i) {
566 BESDEBUG(
"dmrpp:5",
"Found a handle match for the " << i - d_easy_handles.begin() <<
"th easy handle." << endl);
581 for (std::vector<dmrpp_easy_handle *>::iterator i = d_easy_handles.begin(), e = d_easy_handles.end(); i != e; ++i) {
582 if ((*i)->d_chunk == chunk) {
597 for (std::vector<dmrpp_easy_handle *>::iterator i = d_easy_handles.begin(), e = d_easy_handles.end(); i != e; ++i) {
virtual std::string get(const std::string &key)
virtual bool is_s3_cred()
Do the URL, ID, Key amd Region items make up an S3 Credential?
error thrown if the BES is not allowed to access the resource requested
exception thrown if internal error encountered
AccessCredentials * get(std::shared_ptr< http::url > &url)
static CredentialsManager * theCM()
Returns the singleton instance of the CrednetialsManager.
virtual std::string get_curl_range_arg_string()
Returns a curl range argument. The libcurl requires a string argument for range-ge activitys,...
virtual std::shared_ptr< http::url > get_data_url() const
Get the data url for this Chunk's data block.
void release_handle(dmrpp_easy_handle *h)
void release_all_handles()
dmrpp_easy_handle * get_easy_handle(Chunk *chunk)
Add the given header & value to the curl slist.
Bundle a libcurl easy handle with other information.
void read_data()
This is the read_data() method for all transfers.
dmrpp_easy_handle()
Build a string with hex info about stuff libcurl gets.
static AllowedHosts * theHosts()
Static accessor for the singleton.