33 #include <curl/curl.h>
35 #include "CurlUtils.h"
43 #include "BESInternalError.h"
44 #include "BESForbiddenError.h"
45 #include "AllowedHosts.h"
47 #include "DmrppCommon.h"
48 #include "DmrppNames.h"
50 #include "CurlHandlePool.h"
52 #include "CredentialsManager.h"
53 #include "AccessCredentials.h"
56 #define CURL_VERBOSE 0
58 #define prolog std::string("CurlHandlePool::").append(__func__).append("() - ")
61 static const int MAX_WAIT_MSECS = 30 * 1000;
63 static const unsigned int retry_limit = 10;
64 static const useconds_t uone_second = 1000 * 1000;
67 const bool have_curl_multi_api =
false;
70 using namespace dmrpp;
74 string pthread_error(
unsigned int err){
78 error_msg =
"The mutex was either created with the "
79 "protocol attribute having the value "
80 "PTHREAD_PRIO_PROTECT and the calling "
81 "thread's priority is higher than the "
82 "mutex's current priority ceiling."
83 "OR The value specified by mutex does not "
84 "refer to an initialized mutex object.";
88 error_msg =
"The mutex could not be acquired "
89 "because it was already locked.";
93 error_msg =
"The mutex could not be acquired because "
94 "the maximum number of recursive locks "
95 "for mutex has been exceeded.";
99 error_msg =
"The current thread already owns the mutex";
103 error_msg =
"The current thread does not own the mutex.";
107 error_msg =
"Unknown pthread error type.";
114 Lock::Lock(pthread_mutex_t &lock) : m_mutex(lock) {
115 int status = pthread_mutex_lock(&m_mutex);
117 throw BESInternalError(prolog +
"Failed to acquire mutex lock. msg: " + pthread_error(status), __FILE__, __LINE__);
121 int status = pthread_mutex_unlock(&m_mutex);
123 ERROR_LOG(prolog +
"Failed to release mutex lock. msg: " + pthread_error(status));
133 string dump(
const char *text,
unsigned char *ptr,
size_t size)
137 unsigned int width=0x10;
140 oss << text <<
", " << std::setw(10) << (long)size << std::setbase(16) << (long)size << endl;
142 for(i=0; i<size; i+= width) {
143 oss << std::setw(4) << (long)i;
147 for(c = 0; c < width; c++) {
149 oss << std::setw(2) << ptr[i+c];
159 for(c = 0; (c < width) && (i+c < size); c++) {
160 char x = (ptr[i+c] >= 0x20 && ptr[i+c] < 0x80) ? ptr[i+c] :
'.';
162 oss << std::setw(1) << x;
180 int curl_trace(CURL *, curl_infotype type,
char *data,
size_t ,
void *)
186 case CURLINFO_HEADER_OUT:
187 case CURLINFO_HEADER_IN: {
190 while ((pos = text.find(
'\n')) != string::npos)
191 text = text.substr(0, pos);
196 case CURLINFO_DATA_OUT:
197 case CURLINFO_SSL_DATA_OUT:
198 case CURLINFO_DATA_IN:
199 case CURLINFO_SSL_DATA_IN:
207 LOG(
"libcurl == Info: " << text << endl);
210 case CURLINFO_HEADER_OUT:
211 LOG(
"libcurl == Send header: " << text << endl);
213 case CURLINFO_HEADER_IN:
214 LOG(
"libcurl == Recv header: " << text << endl);
218 case CURLINFO_DATA_OUT:
219 case CURLINFO_SSL_DATA_OUT:
220 case CURLINFO_DATA_IN:
221 case CURLINFO_SSL_DATA_IN:
234 d_handle = curl_easy_init();
235 if (!d_handle)
throw BESInternalError(
"Could not allocate CURL handle", __FILE__, __LINE__);
237 curl::set_error_buffer(d_handle, d_errbuf);
239 res = curl_easy_setopt(d_handle, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2);
240 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_SSLVERSION", d_errbuf, __FILE__, __LINE__);
244 res = curl_easy_setopt(d_handle, CURLOPT_DEBUGFUNCTION, curl_trace);
245 curl::check_setopt_result(res, prolog,
"CURLOPT_DEBUGFUNCTION", d_errbuf, __FILE__, __LINE__);
248 res = curl_easy_setopt(d_handle, CURLOPT_VERBOSE, 1L);
249 curl::check_setopt_result(res, prolog,
"CURLOPT_VERBOSE", d_errbuf, __FILE__, __LINE__);
252 res = curl_easy_setopt(d_handle, CURLOPT_HEADERFUNCTION, chunk_header_callback);
253 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_HEADERFUNCTION", d_errbuf, __FILE__, __LINE__);
256 res = curl_easy_setopt(d_handle, CURLOPT_WRITEFUNCTION, chunk_write_data);
257 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_WRITEFUNCTION", d_errbuf, __FILE__, __LINE__);
259 #ifdef CURLOPT_TCP_KEEPALIVE
261 res = curl_easy_setopt(d_handle, CURLOPT_TCP_KEEPALIVE, 1L);
262 curl::check_setopt_result(res, prolog,
"CURLOPT_TCP_KEEPALIVE", d_errbuf, __FILE__, __LINE__);
265 #ifdef CURLOPT_TCP_KEEPIDLE
267 res = curl_easy_setopt(d_handle, CURLOPT_TCP_KEEPIDLE, 120L);
268 curl::check_setopt_result(res, prolog,
"CURLOPT_TCP_KEEPIDLE", d_errbuf, __FILE__, __LINE__);
271 #ifdef CURLOPT_TCP_KEEPINTVL
273 res = curl_easy_setopt(d_handle, CURLOPT_TCP_KEEPINTVL, 120L)
274 curl::check_setopt_result(res, prolog,
"CURLOPT_TCP_KEEPINTVL", d_errbuf, __FILE__, __LINE__);
282 dmrpp_easy_handle::~dmrpp_easy_handle() {
283 if (d_handle) curl_easy_cleanup(d_handle);
284 if (d_request_headers) curl_slist_free_all(d_request_headers);
301 if (d_url.find(
"https://") == 0 || d_url.find(
"http://") == 0) {
302 curl::super_easy_perform(d_handle);
305 CURLcode curl_code = curl_easy_perform(d_handle);
306 if (CURLE_OK != curl_code) {
307 string msg = prolog +
"ERROR - Data transfer error: ";
308 throw BESInternalError(msg.append(curl::error_message(curl_code, d_errbuf)), __FILE__, __LINE__);
312 d_chunk->set_is_read(
true);
326 CurlHandlePool::CurlHandlePool() {
327 d_max_easy_handles = DmrppRequestHandler::d_max_parallel_transfers;
329 for (
unsigned int i = 0; i < d_max_easy_handles; ++i) {
332 unsigned int status = pthread_mutex_init(&d_get_easy_handle_mutex, 0);
334 throw BESInternalError(
"Could not initialize mutex in CurlHandlePool. msg: " + pthread_error(status), __FILE__, __LINE__);
338 CurlHandlePool::CurlHandlePool() {
346 CurlHandlePool::CurlHandlePool(
unsigned int max_handles) : d_max_easy_handles(max_handles) {
347 for (
unsigned int i = 0; i < d_max_easy_handles; ++i) {
351 unsigned int status = pthread_mutex_init(&d_get_easy_handle_mutex, 0);
353 throw BESInternalError(
"Could not initialize mutex in CurlHandlePool. msg: " + pthread_error(status), __FILE__, __LINE__);
368 static struct curl_slist *append_http_header(curl_slist *slist,
const string &header,
const string &value) {
369 string full_header = header;
370 full_header.append(
" ").append(value);
372 struct curl_slist *temp = curl_slist_append(slist, full_header.c_str());
396 if (!AllowedHosts::theHosts()->is_allowed(chunk->
get_data_url())) {
397 string msg =
"ERROR!! The chunk url " + chunk->
get_data_url() +
" does not match any of the AllowedHost rules. ";
401 Lock lock(d_get_easy_handle_mutex);
404 for (vector<dmrpp_easy_handle *>::iterator i = d_easy_handles.begin(), e = d_easy_handles.end(); i != e; ++i) {
405 if (!(*i)->d_in_use) {
413 handle->d_in_use =
true;
416 handle->d_chunk = chunk;
418 CURLcode res = curl_easy_setopt(handle->d_handle, CURLOPT_URL, chunk->
get_data_url().c_str());
419 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_URL", handle->d_errbuf, __FILE__, __LINE__);
423 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_RANGE", handle->d_errbuf, __FILE__, __LINE__);
426 res = curl_easy_setopt(handle->d_handle, CURLOPT_HEADERDATA,
reinterpret_cast<void *
>(chunk));
427 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_HEADERDATA", handle->d_errbuf, __FILE__, __LINE__);
430 res = curl_easy_setopt(handle->d_handle, CURLOPT_WRITEDATA,
reinterpret_cast<void *
>(chunk));
431 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_WRITEDATA", handle->d_errbuf, __FILE__, __LINE__);
434 res = curl_easy_setopt(handle->d_handle, CURLOPT_PRIVATE,
reinterpret_cast<void *
>(handle));
435 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_PRIVATE", handle->d_errbuf, __FILE__, __LINE__);
438 res = curl_easy_setopt(handle->d_handle, CURLOPT_COOKIEFILE, curl::get_cookie_filename().c_str());
439 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_COOKIEFILE", handle->d_errbuf, __FILE__, __LINE__);
441 res = curl_easy_setopt(handle->d_handle, CURLOPT_COOKIEJAR, curl::get_cookie_filename().c_str());
442 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_COOKIEJAR", handle->d_errbuf, __FILE__, __LINE__);
445 res = curl_easy_setopt(handle->d_handle, CURLOPT_FOLLOWLOCATION, 1);
446 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_FOLLOWLOCATION", handle->d_errbuf, __FILE__, __LINE__);
448 res = curl_easy_setopt(handle->d_handle, CURLOPT_MAXREDIRS, curl::max_redirects());
449 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_MAXREDIRS", handle->d_errbuf, __FILE__, __LINE__);
452 res = curl_easy_setopt(handle->d_handle, CURLOPT_USERAGENT, curl::hyrax_user_agent().c_str());
453 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_USERAGENT", handle->d_errbuf, __FILE__, __LINE__);
458 res = curl_easy_setopt(handle->d_handle, CURLOPT_HTTPAUTH, (
long) CURLAUTH_ANY);
459 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_HTTPAUTH", handle->d_errbuf, __FILE__, __LINE__);
462 res = curl_easy_setopt(handle->d_handle, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
463 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_NETRC", handle->d_errbuf, __FILE__, __LINE__);
467 string netrc_file = curl::get_netrc_filename();
468 if (!netrc_file.empty()) {
469 res = curl_easy_setopt(handle->d_handle, CURLOPT_NETRC_FILE, netrc_file.c_str());
470 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_NETRC_FILE", handle->d_errbuf, __FILE__, __LINE__);
476 if (credentials && credentials->
is_s3_cred()) {
478 prolog <<
"Got AccessCredentials instance: " << endl << credentials->to_json() << endl);
481 const std::time_t request_time = std::time(0);
483 const std::string auth_header =
484 AWSV4::compute_awsv4_signature(
487 credentials->
get(AccessCredentials::ID_KEY),
488 credentials->
get(AccessCredentials::KEY_KEY),
489 credentials->
get(AccessCredentials::REGION_KEY),
493 handle->d_request_headers = curl::append_http_header((curl_slist *)0,
"Authorization", auth_header);
494 handle->d_request_headers = curl::append_http_header(handle->d_request_headers,
"x-amz-content-sha256",
495 "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
496 handle->d_request_headers = curl::append_http_header(handle->d_request_headers,
"x-amz-date", AWSV4::ISO8601_date(request_time));
502 handle->d_request_headers = append_http_header(0,
"Authorization:", auth_header);
503 if (!handle->d_request_headers)
505 string(
"CURL Error setting Authorization header: ").append(
506 curl::error_message(res, handle->d_errbuf)), __FILE__, __LINE__);
509 curl_slist *temp = append_http_header(handle->d_request_headers,
"x-amz-content-sha256:",
510 "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
513 string(
"CURL Error setting x-amz-content-sha256: ").append(
514 curl::error_message(res, handle->d_errbuf)),
516 handle->d_request_headers = temp;
518 temp = append_http_header(handle->d_request_headers,
"x-amz-date:", AWSV4::ISO8601_date(request_time));
521 string(
"CURL Error setting x-amz-date header: ").append(
522 curl::error_message(res, handle->d_errbuf)),
524 handle->d_request_headers = temp;
529 res = curl_easy_setopt(handle->d_handle, CURLOPT_HTTPHEADER, handle->d_request_headers);
530 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_HTTPHEADER", handle->d_errbuf, __FILE__, __LINE__);
551 Lock lock(d_get_easy_handle_mutex);
558 handle->d_in_use =
false;
562 for (std::vector<dmrpp_easy_handle *>::iterator i = d_easy_handles.begin(), e = d_easy_handles.end(); i != e; ++i) {
564 BESDEBUG(
"dmrpp:5",
"Found a handle match for the " << i - d_easy_handles.begin() <<
"th easy handle." << endl);
579 for (std::vector<dmrpp_easy_handle *>::iterator i = d_easy_handles.begin(), e = d_easy_handles.end(); i != e; ++i) {
580 if ((*i)->d_chunk == chunk) {
595 for (std::vector<dmrpp_easy_handle *>::iterator i = d_easy_handles.begin(), e = d_easy_handles.end(); i != e; ++i) {
virtual std::string get(const std::string &key)
virtual bool is_s3_cred()
Do the URL, ID, Key amd Region items make up an S3 Credential?
error thrown if the BES is not allowed to access the resource requested
exception thrown if internal error encountered
AccessCredentials * get(const std::string &url)
virtual std::string get_curl_range_arg_string()
Returns a curl range argument. The libcurl requires a string argument for range-ge activitys,...
virtual std::string get_data_url() const
Get the data url string for this Chunk's data block.
void release_handle(dmrpp_easy_handle *h)
void release_all_handles()
dmrpp_easy_handle * get_easy_handle(Chunk *chunk)
Add the given header & value to the curl slist.
Bundle a libcurl easy handle with other information.
void read_data()
This is the read_data() method for all transfers.
dmrpp_easy_handle()
Build a string with hex info about stuff libcurl gets.