XRootD
Loading...
Searching...
No Matches
XrdClHttp::CurlOperation Class Referenceabstract

#include <XrdClHttpOps.hh>

Inheritance diagram for XrdClHttp::CurlOperation:
Collaboration diagram for XrdClHttp::CurlOperation:

Public Types

using HeaderList = std::vector<std::pair<std::string, std::string>>
enum class  HttpVerb {
  COPY ,
  DELETE ,
  HEAD ,
  GET ,
  MKCOL ,
  OPTIONS ,
  PROPFIND ,
  PUT ,
  Count
}
enum  OpError {
  ErrNone ,
  ErrHeaderTimeout ,
  ErrCallback ,
  ErrOperationTimeout ,
  ErrTransferClientStall ,
  ErrTransferStall ,
  ErrTransferSlow
}
enum class  RedirectAction {
  Fail ,
  Reinvoke ,
  ReinvokeAfterAllow
}

Public Member Functions

 CurlOperation (const CurlOperation &)=delete
 CurlOperation (XrdCl::ResponseHandler *handler, const std::string &url, std::chrono::steady_clock::time_point expiry, XrdCl::Log *log, CreateConnCalloutType, HeaderCallout *header_callout)
 CurlOperation (XrdCl::ResponseHandler *handler, const std::string &url, struct timespec timeout, XrdCl::Log *log, CreateConnCalloutType, HeaderCallout *header_callout)
virtual ~CurlOperation ()
virtual bool ContinueHandle ()
virtual void Fail (uint16_t errCode, uint32_t errNum, const std::string &)
bool FinishSetup (CURL *curl)
std::pair< XErrorCode, std::string > GetCallbackError () const
CreateConnCalloutType GetConnCalloutFunc () const
std::string GetCurlErrorMessage () const
CURLGetCurlHandle () const
OpError GetError () const
std::chrono::steady_clock::time_point GetHeaderExpiry () const
std::chrono::steady_clock::time_point GetOperationExpiry ()
std::unique_ptr< ResponseInfoGetResponseInfo ()
int GetStatusCode () const
std::string GetStatusMessage () const
bool GetTriedBoker () const
const std::string & GetUrl () const
virtual HttpVerb GetVerb () const =0
bool HasFailed () const
bool HeaderTimeoutExpired (const std::chrono::steady_clock::time_point &now)
bool IsDone () const
bool IsPaused () const
bool IsRedirect () const
std::unique_ptr< ResponseInfoMoveResponseInfo ()
bool OperationTimeoutExpired (const std::chrono::steady_clock::time_point &now)
virtual void OptionsDone ()
virtual RedirectAction Redirect (std::string &target)
virtual void ReleaseHandle ()
virtual bool RequiresOptions () const
virtual void SetContinueQueue (std::shared_ptr< XrdClHttp::HandlerQueue > queue)
void SetTriedBoker ()
virtual bool Setup (CURL *curl, CurlWorker &)
bool StartConnectionCallout (std::string &err)
std::tuple< uint64_t, std::chrono::steady_clock::duration, std::chrono::steady_clock::duration, std::chrono::steady_clock::duration > StatisticsReset ()
virtual void Success ()=0
bool TransferStalled (uint64_t xfer_bytes, const std::chrono::steady_clock::time_point &now)
bool UseConnectionCallout ()
virtual int WaitSocket ()
virtual int WaitSocketCallback (std::string &err)

Static Public Member Functions

static void CleanupDnsCache ()
static int GetDefaultSlowRateBytesSec ()
static int GetDefaultStallTimeout ()
static const std::string GetVerbString (HttpVerb)
static void SetSlowRateBytesSec (int rate)
static void SetStallTimeout (const std::chrono::steady_clock::duration &stall_interval)
static void SetStallTimeout (int stall_interval)

Protected Member Functions

int FailCallback (XErrorCode ecode, const std::string &emsg)
void SetDone (bool has_failed)
void SetPaused (bool paused)
void UpdateBytes (uint64_t bytes)

Protected Attributes

std::unique_ptr< CURL, void(*)(CURL *)> m_curl
XrdCl::ResponseHandlerm_handler {nullptr}
HeaderCalloutm_header_callout
std::chrono::steady_clock::time_point m_header_expiry
HeaderParser m_headers
std::vector< std::pair< std::string, std::string > > m_headers_list
XrdCl::Logm_logger
int m_minimum_rate {m_minimum_transfer_rate}
std::chrono::steady_clock::time_point m_operation_expiry
const std::string m_url

Static Protected Attributes

static constexpr int m_default_minimum_rate {1024 * 256}
static int m_minimum_transfer_rate {CurlOperation::m_default_minimum_rate}

Detailed Description

Definition at line 56 of file XrdClHttpOps.hh.

Member Typedef Documentation

◆ HeaderList

using XrdClHttp::CurlOperation::HeaderList = std::vector<std::pair<std::string, std::string>>

Definition at line 58 of file XrdClHttpOps.hh.

Member Enumeration Documentation

◆ HttpVerb

Enumerator
COPY 
DELETE 
HEAD 
GET 
MKCOL 
OPTIONS 
PROPFIND 
PUT 
Count 

Definition at line 60 of file XrdClHttpOps.hh.

60 {
61 COPY,
62 DELETE,
63 HEAD,
64 GET,
65 MKCOL,
66 OPTIONS,
67 PROPFIND,
68 PUT,
69 Count
70 };

◆ OpError

Enumerator
ErrNone 
ErrHeaderTimeout 
ErrCallback 
ErrOperationTimeout 
ErrTransferClientStall 
ErrTransferStall 
ErrTransferSlow 

Definition at line 206 of file XrdClHttpOps.hh.

206 {
207 ErrNone, // No error
208 ErrHeaderTimeout, // Header was not sent back in time
209 ErrCallback, // Error in the read/write callback (e.g., response too large for propfind)
210 ErrOperationTimeout, // Entire curl request operation has timed out
211 ErrTransferClientStall, // Transfer stalled while client had paused it (no data was available)
212 ErrTransferStall, // Transfer has stalled, not receiving any data within 60 seconds
213 ErrTransferSlow, // Average transfer rate is below the minimum
214 };

◆ RedirectAction

Enumerator
Fail 
Reinvoke 
ReinvokeAfterAllow 

Definition at line 135 of file XrdClHttpOps.hh.

135 {
136 Fail, // The redirect parsing failed and Fail() was called
137 Reinvoke, // Reinvoke the curl handle, following redirect
138 ReinvokeAfterAllow, // Reinvoke the Redirect function once the allowed verbs are known.
139 };

Constructor & Destructor Documentation

◆ CurlOperation() [1/3]

CurlOperation::CurlOperation ( XrdCl::ResponseHandler * handler,
const std::string & url,
struct timespec timeout,
XrdCl::Log * log,
CreateConnCalloutType callout,
HeaderCallout * header_callout )

Definition at line 166 of file XrdClHttpOps.cc.

168 :
169 CurlOperation::CurlOperation(handler, url, CalculateExpiry(timeout), logger, callout, header_callout)
170 {}
std::chrono::steady_clock::time_point CalculateExpiry(struct timespec timeout)
CurlOperation(XrdCl::ResponseHandler *handler, const std::string &url, struct timespec timeout, XrdCl::Log *log, CreateConnCalloutType, HeaderCallout *header_callout)

References CurlOperation(), and CalculateExpiry().

Referenced by XrdClHttp::CurlCopyOp::CurlCopyOp(), XrdClHttp::CurlDeleteOp::CurlDeleteOp(), XrdClHttp::CurlListdirOp::CurlListdirOp(), XrdClHttp::CurlMkcolOp::CurlMkcolOp(), CurlOperation(), CurlOperation(), XrdClHttp::CurlOptionsOp::CurlOptionsOp(), XrdClHttp::CurlPutOp::CurlPutOp(), XrdClHttp::CurlPutOp::CurlPutOp(), XrdClHttp::CurlReadOp::CurlReadOp(), XrdClHttp::CurlStatOp::CurlStatOp(), and XrdClHttp::CurlVectorReadOp::CurlVectorReadOp().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ CurlOperation() [2/3]

CurlOperation::CurlOperation ( XrdCl::ResponseHandler * handler,
const std::string & url,
std::chrono::steady_clock::time_point expiry,
XrdCl::Log * log,
CreateConnCalloutType callout,
HeaderCallout * header_callout )

Definition at line 172 of file XrdClHttpOps.cc.

174 :
175 m_header_expiry(expiry),
176 m_header_callout(header_callout),
177 m_last_reset(std::chrono::steady_clock::now()),
178 m_last_header_reset(m_last_reset),
179 m_start_op(m_last_reset),
180 m_header_start(m_last_reset),
181 m_conn_callout(callout),
182 m_url(DavToHttp(url)),
183 m_handler(handler),
184 m_curl(nullptr, &curl_easy_cleanup),
185 m_logger(logger)
186 {}
const std::string m_url
std::chrono::steady_clock::time_point m_header_expiry
std::unique_ptr< CURL, void(*)(CURL *)> m_curl
HeaderCallout * m_header_callout
XrdCl::ResponseHandler * m_handler

References m_curl, m_handler, m_header_callout, m_header_expiry, m_logger, and m_url.

◆ ~CurlOperation()

CurlOperation::~CurlOperation ( )
virtual

Definition at line 188 of file XrdClHttpOps.cc.

188{}

◆ CurlOperation() [3/3]

XrdClHttp::CurlOperation::CurlOperation ( const CurlOperation & )
delete

References CurlOperation().

Here is the call graph for this function:

Member Function Documentation

◆ CleanupDnsCache()

void CurlOperation::CleanupDnsCache ( )
static

Definition at line 694 of file XrdClHttpOps.cc.

695{
696 auto now = std::chrono::steady_clock::now();
697 for (auto it = fake_dns_refcount.begin(); it != fake_dns_refcount.end(); ) {
698 if (it->second->count <= 0 && it->second->IsExpired(now)) {
699 auto rev_iter = reverse_fake_dns_map.find(*it->first);
700 if (rev_iter != reverse_fake_dns_map.end()) {
701 fake_dns_map.erase(rev_iter->second.first);
702 reverse_fake_dns_map.erase(rev_iter);
703 }
704 it = fake_dns_refcount.erase(it);
705 } else {
706 ++it;
707 }
708 }
709}

Referenced by XrdClHttp::CurlWorker::Run().

Here is the caller graph for this function:

◆ ContinueHandle()

virtual bool XrdClHttp::CurlOperation::ContinueHandle ( )
inlinevirtual

Reimplemented in XrdClHttp::CurlPutOp, and XrdClHttp::CurlReadOp.

Definition at line 129 of file XrdClHttpOps.hh.

129{return true;}

Referenced by XrdClHttp::CurlWorker::Run().

Here is the caller graph for this function:

◆ Fail()

void CurlOperation::Fail ( uint16_t errCode,
uint32_t errNum,
const std::string & msg )
virtual

Reimplemented in XrdClHttp::CurlMkcolOp, XrdClHttp::CurlOpenOp, XrdClHttp::CurlOptionsOp, XrdClHttp::CurlPutOp, XrdClHttp::CurlReadOp, and XrdClHttp::CurlVectorReadOp.

Definition at line 191 of file XrdClHttpOps.cc.

192{
193 SetDone(true);
194 if (m_handler == nullptr) {return;}
195 if (!msg.empty()) {
196 m_logger->Debug(kLogXrdClHttp, "curl operation failed with message: %s", msg.c_str());
197 } else {
198 m_logger->Debug(kLogXrdClHttp, "curl operation failed with status code %d", errNum);
199 }
200 auto status = new XrdCl::XRootDStatus(XrdCl::stError, errCode, errNum, msg);
201 auto handle = m_handler;
202 m_handler = nullptr;
203 handle->HandleResponse(status, nullptr);
204}
void SetDone(bool has_failed)
const uint64_t kLogXrdClHttp
const uint16_t stError
An error occurred that could potentially be retried.

References XrdClHttp::kLogXrdClHttp, m_handler, m_logger, SetDone(), and XrdCl::stError.

Referenced by Redirect(), XrdClHttp::CurlWorker::Run(), StartConnectionCallout(), XrdClHttp::CurlListdirOp::Success(), XrdClHttp::CurlQueryOp::Success(), and XrdClHttp::CurlStatOp::SuccessImpl().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ FailCallback()

int CurlOperation::FailCallback ( XErrorCode ecode,
const std::string & emsg )
protected

Definition at line 207 of file XrdClHttpOps.cc.

207 {
208 m_callback_error_code = ecode;
209 m_callback_error_str = emsg;
210 m_error = OpError::ErrCallback;
211 m_logger->Debug(kLogXrdClHttp, "%s", emsg.c_str());
212 return 0;
213}
int emsg(int rc, char *msg)

References emsg(), ErrCallback, XrdClHttp::kLogXrdClHttp, and m_logger.

Referenced by XrdClHttp::CurlVectorReadOp::Write().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ FinishSetup()

bool CurlOperation::FinishSetup ( CURL * curl)

Definition at line 216 of file XrdClHttpOps.cc.

217{
218 if (!m_header_callout) {
219 m_header_slist.reset();
220 for (const auto &header : m_headers_list) {
221 m_header_slist.reset(curl_slist_append(m_header_slist.release(),
222 (header.first + ": " + header.second).c_str()));
223 }
224 return curl_easy_setopt(curl, CURLOPT_HTTPHEADER, m_header_slist.get()) == CURLE_OK;
225 }
226 const auto &verb = GetVerbString(GetVerb());
227
228 auto extra_headers = m_header_callout->GetHeaders(verb, m_url, m_headers_list);
229 if (!extra_headers) {
230 m_logger->Error(kLogXrdClHttp, "Failed to get headers from header callout for %s", m_url.c_str());
231 return false;
232 }
233 m_header_slist.reset();
234 for (const auto &header : *extra_headers) {
235 if (!strcasecmp(header.first.c_str(), "Content-Length")) {
236 auto upload_size = std::stoull(header.second);
237 curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, upload_size);
238 continue;
239 }
240 m_header_slist.reset(curl_slist_append(m_header_slist.release(),
241 (header.first + ": " + header.second).c_str()));
242 }
243 return curl_easy_setopt(curl, CURLOPT_HTTPHEADER, m_header_slist.get()) == CURLE_OK;
244}
static const std::string GetVerbString(HttpVerb)
virtual HttpVerb GetVerb() const =0
std::vector< std::pair< std::string, std::string > > m_headers_list

References GetVerb(), GetVerbString(), XrdClHttp::kLogXrdClHttp, m_header_callout, m_headers_list, m_logger, and m_url.

Referenced by XrdClHttp::CurlWorker::Run().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ GetCallbackError()

std::pair< XErrorCode, std::string > XrdClHttp::CurlOperation::GetCallbackError ( ) const
inline

Definition at line 226 of file XrdClHttpOps.hh.

226{return std::make_pair(m_callback_error_code, m_callback_error_str);}

Referenced by XrdClHttp::CurlWorker::Run().

Here is the caller graph for this function:

◆ GetConnCalloutFunc()

CreateConnCalloutType XrdClHttp::CurlOperation::GetConnCalloutFunc ( ) const
inline

Definition at line 99 of file XrdClHttpOps.hh.

99{return m_conn_callout;}

Referenced by XrdClHttp::CurlWorker::Run().

Here is the caller graph for this function:

◆ GetCurlErrorMessage()

std::string XrdClHttp::CurlOperation::GetCurlErrorMessage ( ) const
inline

Definition at line 252 of file XrdClHttpOps.hh.

252 {
253 if (m_curl_error_buffer[0] != '\0')
254 return m_curl_error_buffer;
255 return "";
256 }

Referenced by XrdClHttp::CurlWorker::Run().

Here is the caller graph for this function:

◆ GetCurlHandle()

CURL * XrdClHttp::CurlOperation::GetCurlHandle ( ) const
inline

Definition at line 217 of file XrdClHttpOps.hh.

217{return m_curl.get();}

References m_curl.

Referenced by XrdClHttp::CurlWorker::Run().

Here is the caller graph for this function:

◆ GetDefaultSlowRateBytesSec()

int XrdClHttp::CurlOperation::GetDefaultSlowRateBytesSec ( )
inlinestatic

Definition at line 278 of file XrdClHttpOps.hh.

279 {
281 }
static constexpr int m_default_minimum_rate

References m_default_minimum_rate.

◆ GetDefaultStallTimeout()

int XrdClHttp::CurlOperation::GetDefaultStallTimeout ( )
inlinestatic

Definition at line 272 of file XrdClHttpOps.hh.

273 {
274 return std::chrono::duration_cast<std::chrono::seconds>(m_default_stall_interval).count();
275 }

◆ GetError()

OpError XrdClHttp::CurlOperation::GetError ( ) const
inline

Definition at line 220 of file XrdClHttpOps.hh.

220{return m_error;}

Referenced by XrdClHttp::CurlWorker::Run().

Here is the caller graph for this function:

◆ GetHeaderExpiry()

std::chrono::steady_clock::time_point XrdClHttp::CurlOperation::GetHeaderExpiry ( ) const
inline

Definition at line 110 of file XrdClHttpOps.hh.

110{return m_header_expiry;}

References m_header_expiry.

Referenced by XrdClHttp::CurlOptionsOp::CurlOptionsOp(), and GetOperationExpiry().

Here is the caller graph for this function:

◆ GetOperationExpiry()

std::chrono::steady_clock::time_point XrdClHttp::CurlOperation::GetOperationExpiry ( )
inline

Definition at line 113 of file XrdClHttpOps.hh.

113 {
114 if (m_last_xfer == std::chrono::steady_clock::time_point()) {
115 return GetHeaderExpiry();
116 }
117 return m_last_xfer + m_stall_interval;
118 }
std::chrono::steady_clock::time_point GetHeaderExpiry() const

References GetHeaderExpiry().

Here is the call graph for this function:

◆ GetResponseInfo()

std::unique_ptr< ResponseInfo > XrdClHttp::CurlOperation::GetResponseInfo ( )

◆ GetStatusCode()

int XrdClHttp::CurlOperation::GetStatusCode ( ) const
inline

Definition at line 229 of file XrdClHttpOps.hh.

229{return m_headers.GetStatusCode();}

References m_headers.

Referenced by XrdClHttp::CurlMkcolOp::Fail(), XrdClHttp::CurlWorker::Run(), and XrdClHttp::CurlVectorReadOp::Write().

Here is the caller graph for this function:

◆ GetStatusMessage()

std::string XrdClHttp::CurlOperation::GetStatusMessage ( ) const
inline

Definition at line 232 of file XrdClHttpOps.hh.

232{return m_headers.GetStatusMessage();}

References m_headers.

Referenced by XrdClHttp::CurlWorker::Run().

Here is the caller graph for this function:

◆ GetTriedBoker()

bool XrdClHttp::CurlOperation::GetTriedBoker ( ) const
inline

Definition at line 167 of file XrdClHttpOps.hh.

167{return m_tried_broker;} // Returns true if the connection broker has been tried.

Referenced by XrdClHttp::CurlWorker::Run().

Here is the caller graph for this function:

◆ GetUrl()

const std::string & XrdClHttp::CurlOperation::GetUrl ( ) const
inline

Definition at line 177 of file XrdClHttpOps.hh.

177{return m_url;}

References m_url.

Referenced by XrdClHttp::CurlWorker::Run().

Here is the caller graph for this function:

◆ GetVerb()

virtual HttpVerb XrdClHttp::CurlOperation::GetVerb ( ) const
pure virtual

Implemented in XrdClHttp::CurlChecksumOp, XrdClHttp::CurlCopyOp, XrdClHttp::CurlDeleteOp, XrdClHttp::CurlListdirOp, XrdClHttp::CurlMkcolOp, XrdClHttp::CurlOptionsOp, XrdClHttp::CurlPgReadOp, XrdClHttp::CurlPutOp, XrdClHttp::CurlReadOp, XrdClHttp::CurlStatOp, and XrdClHttp::CurlVectorReadOp.

Referenced by FinishSetup().

Here is the caller graph for this function:

◆ GetVerbString()

const std::string CurlOperation::GetVerbString ( CurlOperation::HttpVerb verb)
static

Definition at line 247 of file XrdClHttpOps.cc.

248{
249 switch (verb) {
250 case HttpVerb::COPY:
251 return "COPY";
252 case HttpVerb::DELETE:
253 return "DELETE";
254 case HttpVerb::GET:
255 return "GET";
256 case HttpVerb::HEAD:
257 return "HEAD";
258 case HttpVerb::MKCOL:
259 return "MKCOL";
261 return "OPTIONS";
263 return "PROPFIND";
264 case HttpVerb::PUT:
265 return "PUT";
266 case HttpVerb::Count:
267 return "UNKNOWN";
268 }
269 return "UNKNOWN";
270}

References COPY, Count, DELETE, GET, HEAD, MKCOL, OPTIONS, PROPFIND, and PUT.

Referenced by FinishSetup(), and XrdClHttp::CurlWorker::GetMonitoringJson().

Here is the caller graph for this function:

◆ HasFailed()

bool XrdClHttp::CurlOperation::HasFailed ( ) const
inline

Definition at line 241 of file XrdClHttpOps.hh.

241{return m_has_failed.load(std::memory_order_acquire);}

◆ HeaderTimeoutExpired()

bool CurlOperation::HeaderTimeoutExpired ( const std::chrono::steady_clock::time_point & now)

Definition at line 450 of file XrdClHttpOps.cc.

450 {
451 if (m_received_header) return false;
452
453 if (now > m_header_expiry) {
454 if (m_error == OpError::ErrNone) m_error = OpError::ErrHeaderTimeout;
455 return true;
456 }
457 return false;
458}

References ErrHeaderTimeout, ErrNone, and m_header_expiry.

◆ IsDone()

bool XrdClHttp::CurlOperation::IsDone ( ) const
inline

Definition at line 235 of file XrdClHttpOps.hh.

235{return m_done;}

Referenced by XrdClHttp::CurlReadOp::ContinueHandle(), and XrdClHttp::CurlWorker::Run().

Here is the caller graph for this function:

◆ IsPaused()

bool XrdClHttp::CurlOperation::IsPaused ( ) const
inline

Definition at line 238 of file XrdClHttpOps.hh.

238{return m_is_paused;}

Referenced by XrdClHttp::CurlWorker::Run(), StatisticsReset(), and TransferStalled().

Here is the caller graph for this function:

◆ IsRedirect()

bool XrdClHttp::CurlOperation::IsRedirect ( ) const
inline

Definition at line 151 of file XrdClHttpOps.hh.

151{return m_headers.GetStatusCode() >= 300 && m_headers.GetStatusCode() < 400;}

References m_headers.

Referenced by XrdClHttp::CurlWorker::Run().

Here is the caller graph for this function:

◆ MoveResponseInfo()

std::unique_ptr< ResponseInfo > XrdClHttp::CurlOperation::MoveResponseInfo ( )
inline

Definition at line 223 of file XrdClHttpOps.hh.

223{return std::move(m_response_info);}

Referenced by XrdClHttp::CurlChecksumOp::Success(), XrdClHttp::CurlDeleteOp::Success(), XrdClHttp::CurlListdirOp::Success(), XrdClHttp::CurlMkcolOp::Success(), and XrdClHttp::CurlStatOp::SuccessImpl().

Here is the caller graph for this function:

◆ OperationTimeoutExpired()

bool CurlOperation::OperationTimeoutExpired ( const std::chrono::steady_clock::time_point & now)

Definition at line 461 of file XrdClHttpOps.cc.

461 {
462 if (m_operation_expiry == std::chrono::steady_clock::time_point{} ||
463 !m_received_header) {
464 return false;
465 }
466
467 if (now > m_operation_expiry) {
468 if (m_error == OpError::ErrNone) m_error = OpError::ErrOperationTimeout;
469 return true;
470 }
471 return false;
472}
std::chrono::steady_clock::time_point m_operation_expiry

References ErrNone, ErrOperationTimeout, and m_operation_expiry.

◆ OptionsDone()

virtual void XrdClHttp::CurlOperation::OptionsDone ( )
inlinevirtual

Reimplemented in XrdClHttp::CurlChecksumOp, and XrdClHttp::CurlStatOp.

Definition at line 174 of file XrdClHttpOps.hh.

174{}

Referenced by XrdClHttp::CurlWorker::Run().

Here is the caller graph for this function:

◆ Redirect()

CurlOperation::RedirectAction CurlOperation::Redirect ( std::string & target)
virtual

Reimplemented in XrdClHttp::CurlChecksumOp, and XrdClHttp::CurlStatOp.

Definition at line 305 of file XrdClHttpOps.cc.

306{
307 m_callout.reset();
308 m_conn_callout_result = -1;
309 m_conn_callout_listener = -1;
310 m_tried_broker = false;
311
312 auto location = m_headers.GetLocation();
313 if (location.empty()) {
314 m_logger->Warning(kLogXrdClHttp, "After request to %s, server returned a redirect with no new location", m_url.c_str());
315 Fail(XrdCl::errErrorResponse, kXR_ServerError, "Server returned redirect without updated location");
317 }
318 if (location.size() && location[0] == '/') { // hostname not included in the location - redirect to self.
319 std::string_view orig_url(m_url);
320 auto scheme_loc = orig_url.find("://");
321 if (scheme_loc == std::string_view::npos) {
322 Fail(XrdCl::errErrorResponse, kXR_ServerError, "Server returned a location with unknown hostname");
324 }
325 auto path_loc = orig_url.find('/', scheme_loc + 3);
326 if (path_loc == std::string_view::npos) {
327 location = m_url + location;
328 } else {
329 location = std::string(orig_url.substr(0, path_loc)) + location;
330 }
331 }
332 m_logger->Debug(kLogXrdClHttp, "Request for %s redirected to %s", m_url.c_str(), location.c_str());
333 target = location;
334 curl_easy_setopt(m_curl.get(), CURLOPT_URL, location.c_str());
335 int disable_x509;
336 auto env = XrdCl::DefaultEnv::GetEnv();
337 if (env->GetInt("HttpDisableX509", disable_x509) && !disable_x509) {
338 std::string cert, key;
339 env->GetString("HttpClientCertFile", cert);
340 env->GetString("HttpClientKeyFile", key);
341 if (!cert.empty())
342 curl_easy_setopt(m_curl.get(), CURLOPT_SSLCERT, cert.c_str());
343 if (!key.empty())
344 curl_easy_setopt(m_curl.get(), CURLOPT_SSLKEY, key.c_str());
345 }
346 m_headers = HeaderParser();
347
348 if (m_conn_callout) {
349 auto conn_callout = m_conn_callout(location, *m_response_info);
350 if (conn_callout != nullptr) {
351
352 auto [host, port] = ParseHostPort(location);
353 if (host.empty() || port == -1) {
354 Fail(XrdCl::errInternal, 0, "Failed to parse host and port from URL " + location);
356 }
357 auto fake_addr = GetFakeEndpointForHost(host, port);
358 if (!fake_addr || fake_addr->empty()) {
359 Fail(XrdCl::errInternal, 0, "Failed to generate a fake address for host " + host);
361 }
362 m_resolve_slist.reset(curl_slist_append(m_resolve_slist.release(),
363 (host + ":" + std::to_string(port) + ":" + *fake_addr).c_str()));
364 m_logger->Debug(kLogXrdClHttp, "For connection callout in redirect, mapping %s:%d -> %s", host.c_str(), port, fake_addr->c_str());
365
366 m_callout.reset(conn_callout);
367 std::string err;
369 if ((m_conn_callout_listener = m_callout->BeginCallout(err, m_header_expiry)) == -1) {
370 auto errMsg = "Failed to start a connection callout request: " + err;
371 Fail(XrdCl::errInternal, 0, errMsg.c_str());
373 }
374 curl_easy_setopt(m_curl.get(), CURLOPT_OPENSOCKETFUNCTION, CurlOperation::OpenSocketCallback);
375 curl_easy_setopt(m_curl.get(), CURLOPT_CLOSESOCKETFUNCTION, CurlOperation::CloseSocketCallback);
376 curl_easy_setopt(m_curl.get(), CURLOPT_OPENSOCKETDATA, this);
377 curl_easy_setopt(m_curl.get(), CURLOPT_CLOSESOCKETDATA, fake_addr);
378 curl_easy_setopt(m_curl.get(), CURLOPT_SOCKOPTFUNCTION, CurlOperation::SockOptCallback);
379 curl_easy_setopt(m_curl.get(), CURLOPT_SOCKOPTDATA, this);
380 curl_easy_setopt(m_curl.get(), CURLOPT_CONNECT_TO, m_resolve_slist.get());
381 }
382 }
383 m_received_header = false;
384
385 m_last_header_reset = m_last_reset = m_header_start = m_start_op = m_header_lastop = std::chrono::steady_clock::now();
387}
@ kXR_ServerError
virtual void Fail(uint16_t errCode, uint32_t errNum, const std::string &)
static Env * GetEnv()
Get default client environment.
const uint16_t errErrorResponse
const uint16_t errInternal
Internal error.

References XrdCl::errErrorResponse, XrdCl::errInternal, Fail(), Fail, XrdCl::DefaultEnv::GetEnv(), XrdClHttp::kLogXrdClHttp, kXR_ServerError, m_curl, m_header_expiry, m_headers, m_logger, m_url, Reinvoke, and SetTriedBoker().

Referenced by XrdClHttp::CurlChecksumOp::Redirect(), XrdClHttp::CurlStatOp::Redirect(), and XrdClHttp::CurlWorker::Run().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ ReleaseHandle()

void CurlOperation::ReleaseHandle ( )
virtual

Reimplemented in XrdClHttp::CurlChecksumOp, XrdClHttp::CurlCopyOp, XrdClHttp::CurlDeleteOp, XrdClHttp::CurlListdirOp, XrdClHttp::CurlMkcolOp, XrdClHttp::CurlOpenOp, XrdClHttp::CurlOptionsOp, XrdClHttp::CurlPutOp, XrdClHttp::CurlReadOp, XrdClHttp::CurlStatOp, and XrdClHttp::CurlVectorReadOp.

Definition at line 603 of file XrdClHttpOps.cc.

604{
605 m_conn_callout_listener = -1;
606 m_conn_callout_result = -1;
607 m_tried_broker = false;
608 m_callout.reset();
609
610 if (m_curl == nullptr) return;
611 curl_easy_setopt(m_curl.get(), CURLOPT_OPENSOCKETFUNCTION, nullptr);
612 curl_easy_setopt(m_curl.get(), CURLOPT_CLOSESOCKETFUNCTION, nullptr);
613 curl_easy_setopt(m_curl.get(), CURLOPT_OPENSOCKETDATA, nullptr);
614 curl_easy_setopt(m_curl.get(), CURLOPT_CLOSESOCKETDATA, nullptr);
615 curl_easy_setopt(m_curl.get(), CURLOPT_SOCKOPTFUNCTION, nullptr);
616 curl_easy_setopt(m_curl.get(), CURLOPT_SOCKOPTDATA, nullptr);
617 curl_easy_setopt(m_curl.get(), CURLOPT_SSLCERT, nullptr);
618 curl_easy_setopt(m_curl.get(), CURLOPT_SSLKEY, nullptr);
619 curl_easy_setopt(m_curl.get(), CURLOPT_HTTPHEADER, nullptr);
620 curl_easy_setopt(m_curl.get(), CURLOPT_CONNECT_TO, nullptr);
621 m_header_slist.reset();
622 m_curl.release();
623}

References m_curl.

Referenced by XrdClHttp::CurlCopyOp::ReleaseHandle(), XrdClHttp::CurlDeleteOp::ReleaseHandle(), XrdClHttp::CurlListdirOp::ReleaseHandle(), XrdClHttp::CurlMkcolOp::ReleaseHandle(), XrdClHttp::CurlOptionsOp::ReleaseHandle(), XrdClHttp::CurlPutOp::ReleaseHandle(), XrdClHttp::CurlReadOp::ReleaseHandle(), XrdClHttp::CurlStatOp::ReleaseHandle(), XrdClHttp::CurlVectorReadOp::ReleaseHandle(), and XrdClHttp::CurlWorker::Run().

Here is the caller graph for this function:

◆ RequiresOptions()

virtual bool XrdClHttp::CurlOperation::RequiresOptions ( ) const
inlinevirtual

Reimplemented in XrdClHttp::CurlStatOp.

Definition at line 171 of file XrdClHttpOps.hh.

171{return false;}

Referenced by XrdClHttp::CurlWorker::Run().

Here is the caller graph for this function:

◆ SetContinueQueue()

virtual void XrdClHttp::CurlOperation::SetContinueQueue ( std::shared_ptr< XrdClHttp::HandlerQueue > queue)
inlinevirtual

Reimplemented in XrdClHttp::CurlPutOp, and XrdClHttp::CurlReadOp.

Definition at line 133 of file XrdClHttpOps.hh.

133{}

Referenced by XrdClHttp::CurlWorker::Run().

Here is the caller graph for this function:

◆ SetDone()

void XrdClHttp::CurlOperation::SetDone ( bool has_failed)
inlineprotected

◆ SetPaused()

void CurlOperation::SetPaused ( bool paused)
protected

Definition at line 400 of file XrdClHttpOps.cc.

400 {
401 m_is_paused = paused;
402 if (m_is_paused) {
403 m_pause_start = std::chrono::steady_clock::now();
404 } else if (m_pause_start != std::chrono::steady_clock::time_point{}) {
405 m_pause_duration += std::chrono::steady_clock::now() - m_pause_start;
406 m_pause_start = std::chrono::steady_clock::time_point{};
407 }
408}

Referenced by XrdClHttp::CurlPutOp::ContinueHandle(), XrdClHttp::CurlReadOp::ContinueHandle(), XrdClHttp::CurlPutOp::Pause(), and XrdClHttp::CurlReadOp::Pause().

Here is the caller graph for this function:

◆ SetSlowRateBytesSec()

void XrdClHttp::CurlOperation::SetSlowRateBytesSec ( int rate)
inlinestatic

Definition at line 284 of file XrdClHttpOps.hh.

285 {
287 }

References m_minimum_transfer_rate.

◆ SetStallTimeout() [1/2]

void XrdClHttp::CurlOperation::SetStallTimeout ( const std::chrono::steady_clock::duration & stall_interval)
inlinestatic

Definition at line 266 of file XrdClHttpOps.hh.

267 {
268 m_stall_interval = stall_interval;
269 }

◆ SetStallTimeout() [2/2]

void XrdClHttp::CurlOperation::SetStallTimeout ( int stall_interval)
inlinestatic

Definition at line 259 of file XrdClHttpOps.hh.

260 {
261 std::chrono::seconds seconds{stall_interval};
262 m_stall_interval = std::chrono::duration_cast<std::chrono::steady_clock::duration>(seconds);
263 }

Referenced by XrdClHttp::File::SetProperty().

Here is the caller graph for this function:

◆ SetTriedBoker()

void XrdClHttp::CurlOperation::SetTriedBoker ( )
inline

Definition at line 168 of file XrdClHttpOps.hh.

168{m_tried_broker = true;} // Note that the connection broker has been attempted.

Referenced by Redirect(), and XrdClHttp::CurlWorker::Run().

Here is the caller graph for this function:

◆ Setup()

bool CurlOperation::Setup ( CURL * curl,
CurlWorker & worker )
virtual

Reimplemented in XrdClHttp::CurlChecksumOp, XrdClHttp::CurlCopyOp, XrdClHttp::CurlDeleteOp, XrdClHttp::CurlListdirOp, XrdClHttp::CurlMkcolOp, XrdClHttp::CurlOptionsOp, XrdClHttp::CurlPutOp, XrdClHttp::CurlReadOp, XrdClHttp::CurlStatOp, and XrdClHttp::CurlVectorReadOp.

Definition at line 523 of file XrdClHttpOps.cc.

524{
525 if (curl == nullptr) {
526 throw std::runtime_error("Unable to setup curl operation with no handle");
527 }
528 struct timespec now;
529 if (clock_gettime(CLOCK_MONOTONIC, &now) == -1) {
530 throw std::runtime_error("Unable to get current time");
531 }
532
533 m_pause_start = {};
534 m_last_header_reset = m_last_reset = m_start_op = m_header_start = m_header_lastop = std::chrono::steady_clock::now();
535
536 m_curl.reset(curl);
537 m_curl_error_buffer[0] = '\0';
538 curl_easy_setopt(m_curl.get(), CURLOPT_URL, m_url.c_str());
539 curl_easy_setopt(m_curl.get(), CURLOPT_ERRORBUFFER, m_curl_error_buffer);
540 curl_easy_setopt(m_curl.get(), CURLOPT_HEADERFUNCTION, CurlStatOp::HeaderCallback);
541 curl_easy_setopt(m_curl.get(), CURLOPT_HEADERDATA, this);
542 curl_easy_setopt(m_curl.get(), CURLOPT_WRITEFUNCTION, NullCallback);
543 curl_easy_setopt(m_curl.get(), CURLOPT_WRITEDATA, nullptr);
544 curl_easy_setopt(m_curl.get(), CURLOPT_XFERINFOFUNCTION, CurlOperation::XferInfoCallback);
545 curl_easy_setopt(m_curl.get(), CURLOPT_XFERINFODATA, this);
546 curl_easy_setopt(m_curl.get(), CURLOPT_NOPROGRESS, 0L);
547 // Note: libcurl is not threadsafe unless this option is set.
548 // Before we set it, we saw deadlocks (and partial deadlocks) in practice.
549 curl_easy_setopt(m_curl.get(), CURLOPT_NOSIGNAL, 1L);
550
551 m_parsed_url.reset(new XrdCl::URL(m_url));
552 auto env = XrdCl::DefaultEnv::GetEnv();
553 int disable_x509;
554 if ((env->GetInt("HttpDisableX509", disable_x509) && !disable_x509)) {
555 auto [cert, key] = worker.ClientX509CertKeyFile();
556 if (!cert.empty()) {
557 m_logger->Debug(kLogXrdClHttp, "Using client X.509 credential found at %s", cert.c_str());
558 curl_easy_setopt(m_curl.get(), CURLOPT_SSLCERT, cert.c_str());
559 if (key.empty()) {
560 m_logger->Error(kLogXrdClHttp, "X.509 client credential specified but not the client key");
561 } else {
562 curl_easy_setopt(m_curl.get(), CURLOPT_SSLKEY, key.c_str());
563 }
564 }
565 }
566
567 if (m_conn_callout) {
568 ResponseInfo info;
569 auto callout = m_conn_callout(m_url, info);
570 if (callout) {
571 m_callout.reset(callout);
572 m_conn_callout_listener = -1;
573 m_conn_callout_result = -1;
574 m_tried_broker = false;
575
576 auto [host, port] = ParseHostPort(m_url);
577 if (host.empty() || port == -1) {
578 throw std::runtime_error ("Failed to parse host and port from URL " + m_url);
579 }
580 auto fake_addr = GetFakeEndpointForHost(host, port);
581 if (!fake_addr || fake_addr->empty()) {
582 throw std::runtime_error("Failed to generate a fake address for host " + host);
583 }
584 m_resolve_slist.reset(curl_slist_append(m_resolve_slist.release(),
585 (host + ":" + std::to_string(port) + ":" + *fake_addr).c_str()));
586 m_logger->Debug(kLogXrdClHttp, "For connection callout in operation setup, mapping %s:%d -> %s", host.c_str(), port, fake_addr->c_str());
587
588 curl_easy_setopt(m_curl.get(), CURLOPT_CONNECT_TO, m_resolve_slist.get());
589
590 curl_easy_setopt(m_curl.get(), CURLOPT_OPENSOCKETFUNCTION, CurlOperation::OpenSocketCallback);
591 curl_easy_setopt(m_curl.get(), CURLOPT_CLOSESOCKETFUNCTION, CurlOperation::CloseSocketCallback);
592 curl_easy_setopt(m_curl.get(), CURLOPT_OPENSOCKETDATA, this);
593 curl_easy_setopt(m_curl.get(), CURLOPT_CLOSESOCKETDATA, fake_addr);
594 curl_easy_setopt(m_curl.get(), CURLOPT_SOCKOPTFUNCTION, CurlOperation::SockOptCallback);
595 curl_easy_setopt(m_curl.get(), CURLOPT_SOCKOPTDATA, this);
596 }
597 }
598
599 return true;
600}
std::tuple< std::string, std::string > ClientX509CertKeyFile() const

References XrdClHttp::CurlWorker::ClientX509CertKeyFile(), XrdCl::DefaultEnv::GetEnv(), XrdClHttp::kLogXrdClHttp, m_curl, m_logger, and m_url.

Referenced by XrdClHttp::CurlWorker::Run(), XrdClHttp::CurlCopyOp::Setup(), XrdClHttp::CurlDeleteOp::Setup(), XrdClHttp::CurlListdirOp::Setup(), XrdClHttp::CurlMkcolOp::Setup(), XrdClHttp::CurlOptionsOp::Setup(), XrdClHttp::CurlPutOp::Setup(), XrdClHttp::CurlReadOp::Setup(), XrdClHttp::CurlStatOp::Setup(), and XrdClHttp::CurlVectorReadOp::Setup().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ StartConnectionCallout()

bool CurlOperation::StartConnectionCallout ( std::string & err)

Definition at line 411 of file XrdClHttpOps.cc.

412{
413 if ((m_conn_callout_listener = m_callout->BeginCallout(err, m_header_expiry)) == -1) {
414 err = "Failed to start a callout for a socket connection: " + err;
415 Fail(XrdCl::errInternal, 1, err.c_str());
416 return false;
417 }
418 return true;
419}

References XrdCl::errInternal, Fail(), and m_header_expiry.

Referenced by XrdClHttp::CurlWorker::Run().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ StatisticsReset()

std::tuple< uint64_t, std::chrono::steady_clock::duration, std::chrono::steady_clock::duration, std::chrono::steady_clock::duration > CurlOperation::StatisticsReset ( )

Definition at line 422 of file XrdClHttpOps.cc.

422 {
423 auto now = std::chrono::steady_clock::now();
424 std::chrono::steady_clock::duration pre_header{}, post_header{}, pause_duration{};
425 if (m_received_header) {
426 if (m_last_header_reset < m_header_start) {
427 pre_header = m_header_start - m_last_header_reset;
428 m_last_header_reset = m_header_start;
429 }
430 post_header = now - ((m_last_reset < m_header_start) ? m_header_start : m_last_reset);
431 m_last_reset = now;
432 } else {
433 pre_header = now - m_last_header_reset;
434 m_last_header_reset = now;
435 }
436 if (IsPaused()) {
437 m_pause_duration += now - m_pause_start;
438 m_pause_start = now;
439 }
440 if (m_pause_duration != std::chrono::steady_clock::duration::zero()) {
441 pause_duration = m_pause_duration;
442 m_pause_duration = std::chrono::steady_clock::duration::zero();
443 }
444 auto bytes = m_bytes;
445 m_bytes = 0;
446 return {bytes, pre_header, post_header, pause_duration};
447}

References IsPaused().

Here is the call graph for this function:

◆ Success()

virtual void XrdClHttp::CurlOperation::Success ( )
pure virtual

◆ TransferStalled()

bool CurlOperation::TransferStalled ( uint64_t xfer_bytes,
const std::chrono::steady_clock::time_point & now )

Definition at line 475 of file XrdClHttpOps.cc.

476{
477 // First, check to see how long it's been since any data was sent.
478 if (m_last_xfer == std::chrono::steady_clock::time_point()) {
479 m_last_xfer = m_header_lastop;
480 }
481 auto elapsed = now - m_last_xfer;
482 uint64_t xfer_diff = 0;
483 if (xfer > m_last_xfer_count) {
484 xfer_diff = xfer - m_last_xfer_count;
485 m_last_xfer_count = xfer;
486 m_last_xfer = now;
487 }
488
489 // If progress is made in this callback do not classify as stalled
490 if (elapsed > m_stall_interval && xfer_diff == 0) {
492 return true;
493 }
494
495 // Curl updated us with new timing but the byte count hasn't changed; no need to update the EMA.
496 if (xfer_diff == 0) {
497 return false;
498 }
499
500 // If the transfer is not stalled, then we check to see if the exponentially-weighted
501 // moving average of the transfer rate is below the minimum.
502
503 // If the stall interval since the last header hasn't passed, then we don't check for slow transfers.
504 auto elapsed_since_last_headerop = now - m_header_lastop;
505 if (elapsed_since_last_headerop < m_stall_interval) {
506 return false;
507 } else if (m_ema_rate < 0) {
508 m_ema_rate = xfer / std::chrono::duration<double>(elapsed_since_last_headerop).count();
509 }
510 // Calculate the exponential moving average of the transfer rate.
511 double elapsed_seconds = std::chrono::duration<double>(elapsed).count();
512 auto recent_rate = static_cast<double>(xfer_diff) / elapsed_seconds;
513 auto alpha = 1.0 - exp(-elapsed_seconds / std::chrono::duration<double>(m_stall_interval).count());
514 m_ema_rate = (1.0 - alpha) * m_ema_rate + alpha * recent_rate;
515 if (m_ema_rate < static_cast<double>(m_minimum_rate)) {
516 if (m_error == OpError::ErrNone) m_error = OpError::ErrTransferSlow;
517 return true;
518 }
519 return false;
520}

References ErrNone, ErrTransferClientStall, ErrTransferSlow, ErrTransferStall, IsPaused(), and m_minimum_rate.

Here is the call graph for this function:

◆ UpdateBytes()

void XrdClHttp::CurlOperation::UpdateBytes ( uint64_t bytes)
inlineprotected

Definition at line 292 of file XrdClHttpOps.hh.

292{m_bytes += bytes;}

Referenced by XrdClHttp::CurlVectorReadOp::Write().

Here is the caller graph for this function:

◆ UseConnectionCallout()

bool XrdClHttp::CurlOperation::UseConnectionCallout ( )
inline

Definition at line 166 of file XrdClHttpOps.hh.

166{return m_callout.get();} // Returns true if the callout should be tried.

Referenced by XrdClHttp::CurlWorker::Run().

Here is the caller graph for this function:

◆ WaitSocket()

virtual int XrdClHttp::CurlOperation::WaitSocket ( )
inlinevirtual

Definition at line 154 of file XrdClHttpOps.hh.

154{return m_conn_callout_listener;}

Referenced by XrdClHttp::CurlWorker::Run().

Here is the caller graph for this function:

◆ WaitSocketCallback()

int CurlOperation::WaitSocketCallback ( std::string & err)
virtual

Definition at line 727 of file XrdClHttpOps.cc.

728{
729 m_conn_callout_result = m_callout ? m_callout->FinishCallout(err) : -1;
730 if (m_callout && m_conn_callout_result == -1) {
731 m_logger->Error(kLogXrdClHttp, "Error when getting socket callout: %s", err.c_str());
732 } else if (m_callout) {
733 m_logger->Debug(kLogXrdClHttp, "Got callback socket %d", m_conn_callout_result);
734 }
735 return m_conn_callout_result;
736}

References XrdClHttp::kLogXrdClHttp, and m_logger.

Member Data Documentation

◆ m_curl

◆ m_default_minimum_rate

int XrdClHttp::CurlOperation::m_default_minimum_rate {1024 * 256}
staticconstexprprotected

Definition at line 304 of file XrdClHttpOps.hh.

304{1024 * 256}; // 256 KB/sec

Referenced by GetDefaultSlowRateBytesSec().

◆ m_handler

◆ m_header_callout

HeaderCallout* XrdClHttp::CurlOperation::m_header_callout
protected

Definition at line 321 of file XrdClHttpOps.hh.

Referenced by CurlOperation(), and FinishSetup().

◆ m_header_expiry

std::chrono::steady_clock::time_point XrdClHttp::CurlOperation::m_header_expiry
protected

◆ m_headers

◆ m_headers_list

◆ m_logger

◆ m_minimum_rate

int XrdClHttp::CurlOperation::m_minimum_rate {m_minimum_transfer_rate}
protected

◆ m_minimum_transfer_rate

int CurlOperation::m_minimum_transfer_rate {CurlOperation::m_default_minimum_rate}
staticprotected

Definition at line 309 of file XrdClHttpOps.hh.

Referenced by SetSlowRateBytesSec().

◆ m_operation_expiry

std::chrono::steady_clock::time_point XrdClHttp::CurlOperation::m_operation_expiry
protected

Definition at line 315 of file XrdClHttpOps.hh.

Referenced by XrdClHttp::CurlStatOp::CurlStatOp(), and OperationTimeoutExpired().

◆ m_url


The documentation for this class was generated from the following files: