XRootD
XrdClHttpFilePlugIn.cc
Go to the documentation of this file.
1 
6 
7 #include <unistd.h>
8 
9 #include <cassert>
10 
13 #include "XrdCl/XrdClDefaultEnv.hh"
14 #include "XrdCl/XrdClLog.hh"
15 #include "XrdCl/XrdClStatus.hh"
16 
17 #include "XrdOuc/XrdOucCRC.hh"
18 
19 namespace {
20 
21 int MakePosixOpenFlags(XrdCl::OpenFlags::Flags flags) {
22  int posix_flags = 0;
23  if (flags & XrdCl::OpenFlags::New) {
24  posix_flags |= O_CREAT | O_EXCL;
25  }
26  if (flags & XrdCl::OpenFlags::Delete) {
27  posix_flags |= O_CREAT | O_TRUNC;
28  }
29  if (flags & XrdCl::OpenFlags::Read) {
30  posix_flags |= O_RDONLY;
31  }
32  if (flags & XrdCl::OpenFlags::Write) {
33  posix_flags |= O_WRONLY;
34  }
35  if (flags & XrdCl::OpenFlags::Update) {
36  posix_flags |= O_RDWR;
37  }
38  return posix_flags;
39 }
40 
41 } // namespace
42 
43 namespace XrdCl {
44 
45 Davix::Context *root_davix_context_ = NULL;
46 Davix::DavPosix *root_davix_client_file_ = NULL;
47 
49  : davix_fd_(nullptr),
50  curr_offset(0),
51  is_open_(false),
52  filesize(0),
53  url_(),
54  properties_(),
55  logger_(DefaultEnv::GetLog()) {
56  SetUpLogging(logger_);
57  logger_->Debug(kLogXrdClHttp, "HttpFilePlugin constructed.");
58 
59  std::string origin = getenv("XRDXROOTD_PROXY")? getenv("XRDXROOTD_PROXY") : "";
60  if ( origin.empty() || origin.find("=") == 0) {
61  davix_context_ = new Davix::Context();
62  davix_client_ = new Davix::DavPosix(davix_context_);
63  }
64  else {
65  if (root_davix_context_ == NULL) {
66  root_davix_context_ = new Davix::Context();
67  root_davix_client_file_ = new Davix::DavPosix(root_davix_context_);
68  }
69  davix_context_ = root_davix_context_;
70  davix_client_ = root_davix_client_file_;
71  }
72 
73 }
74 
76  if (root_davix_context_ == NULL) {
77  delete davix_client_;
78  delete davix_context_;
79  }
80 }
81 
82 XRootDStatus HttpFilePlugIn::Open(const std::string &url,
83  OpenFlags::Flags flags, Access::Mode /*mode*/,
84  ResponseHandler *handler, uint16_t timeout) {
85  if (is_open_) {
86  logger_->Error(kLogXrdClHttp, "URL %s already open", url.c_str());
88  }
89 
90  if (XrdCl::URL(url).GetProtocol().find("https") == 0)
91  isChannelEncrypted = true;
92  else
93  isChannelEncrypted = false;
94 
95  avoid_pread_ = false;
96  if (getenv(HTTP_FILE_PLUG_IN_AVOIDRANGE_ENV) != NULL)
97  avoid_pread_ = true;
98  else {
100  auto search = CGIs.find(HTTP_FILE_PLUG_IN_AVOIDRANGE_CGI);
101  if (search != CGIs.end())
102  avoid_pread_ = true;
103  }
104 
105  Davix::RequestParams params;
106  if (timeout != 0) {
107  struct timespec ts = {timeout, 0};
108  params.setOperationTimeout(&ts);
109  }
110 
112  auto full_path = XrdCl::URL(url).GetLocation();
113  auto pos = full_path.find_last_of('/');
114  auto base_dir =
115  pos != std::string::npos ? full_path.substr(0, pos) : full_path;
116  auto mkdir_status =
117  Posix::MkDir(*davix_client_, base_dir, XrdCl::MkDirFlags::MakePath,
118  XrdCl::Access::None, timeout);
119  if (mkdir_status.IsError()) {
120  logger_->Error(kLogXrdClHttp,
121  "Could not create parent directories when opening: %s",
122  url.c_str());
123  return mkdir_status;
124  }
125  }
126 
127  if (((flags & OpenFlags::Write) || (flags & OpenFlags::Update)) &&
128  (flags & OpenFlags::Delete)) {
129  auto stat_info = new StatInfo();
130  auto status = Posix::Stat(*davix_client_, url, timeout, stat_info);
131  if (status.IsOK()) {
132  auto unlink_status = Posix::Unlink(*davix_client_, url, timeout);
133  if (unlink_status.IsError()) {
134  logger_->Error(
136  "Could not delete existing destination file: %s. Error: %s",
137  url.c_str(), unlink_status.GetErrorMessage().c_str());
138  return unlink_status;
139  }
140  }
141  delete stat_info;
142  }
143  else if (flags & OpenFlags::Read) {
144  auto stat_info = new StatInfo();
145  auto status = Posix::Stat(*davix_client_, url, timeout, stat_info);
146  if (status.IsOK()) {
147  filesize = stat_info->GetSize();
148  }
149  delete stat_info;
150  }
151 
152  auto posix_open_flags = MakePosixOpenFlags(flags);
153 
154  logger_->Debug(kLogXrdClHttp,
155  "Open: URL: %s, XRootD flags: %d, POSIX flags: %d",
156  url.c_str(), flags, posix_open_flags);
157 
158  // res == std::pair<fd, XRootDStatus>
159  auto res = Posix::Open(*davix_client_, url, posix_open_flags, timeout);
160  if (!res.first) {
161  logger_->Error(kLogXrdClHttp, "Could not open: %s, error: %s", url.c_str(),
162  res.second.ToStr().c_str());
163  return res.second;
164  }
165 
166  davix_fd_ = res.first;
167 
168  logger_->Debug(kLogXrdClHttp, "Opened: %s", url.c_str());
169 
170  is_open_ = true;
171  url_ = url;
172 
173  auto status = new XRootDStatus();
174  handler->HandleResponse(status, nullptr);
175 
176  return XRootDStatus();
177 }
178 
180  uint16_t /*timeout*/) {
181  if (!is_open_) {
182  logger_->Error(kLogXrdClHttp,
183  "Cannot close. URL hasn't been previously opened");
185  }
186 
187  logger_->Debug(kLogXrdClHttp, "Closing davix fd: %ld", davix_fd_);
188 
189  auto status = Posix::Close(*davix_client_, davix_fd_);
190  if (status.IsError()) {
191  logger_->Error(kLogXrdClHttp, "Could not close davix fd: %ld, error: %s",
192  davix_fd_, status.ToStr().c_str());
193  return status;
194  }
195 
196  is_open_ = false;
197  url_.clear();
198 
199  handler->HandleResponse(new XRootDStatus(), nullptr);
200 
201  return XRootDStatus();
202 }
203 
205  uint16_t timeout) {
206  if (!is_open_) {
207  logger_->Error(kLogXrdClHttp,
208  "Cannot stat. URL hasn't been previously opened");
210  }
211 
212  auto stat_info = new StatInfo();
213  auto status = Posix::Stat(*davix_client_, url_, timeout, stat_info);
214  // A file that is_open_ = true should not retune 400/3011. the only time this
215  // happen is a newly created file. Davix doesn't issue a http PUT so this file
216  // won't show up for Stat(). Here we fake a response.
217  if (status.IsError() && status.code == 400 && status.errNo == 3011) {
218  std::ostringstream data;
219  data << 140737018595560 << " " << filesize << " " << 33261 << " " << time(NULL);
220  stat_info->ParseServerResponse(data.str().c_str());
221  }
222  else if (status.IsError()) {
223  logger_->Error(kLogXrdClHttp, "Stat failed: %s", status.ToStr().c_str());
224  return status;
225  }
226 
227  logger_->Debug(kLogXrdClHttp, "Stat-ed URL: %s", url_.c_str());
228 
229  auto obj = new AnyObject();
230  obj->Set(stat_info);
231 
232  handler->HandleResponse(new XRootDStatus(), obj);
233 
234  return XRootDStatus();
235 }
236 
237 XRootDStatus HttpFilePlugIn::Read(uint64_t offset, uint32_t size, void *buffer,
238  ResponseHandler *handler,
239  uint16_t /*timeout*/) {
240  if (!is_open_) {
241  logger_->Error(kLogXrdClHttp,
242  "Cannot read. URL hasn't previously been opened");
244  }
245 
246  // DavPosix::pread will return -1 if the pread goes beyond the file size
247  size = (offset + size > filesize)? filesize - offset : size;
248  std::pair<int, XRootDStatus> res;
249  if (! avoid_pread_) {
250  res = Posix::PRead(*davix_client_, davix_fd_, buffer, size, offset);
251  }
252  else {
253  offset_locker.lock();
254  if (offset == curr_offset) {
255  res = Posix::Read(*davix_client_, davix_fd_, buffer, size);
256  }
257  else {
258  res = Posix::PRead(*davix_client_, davix_fd_, buffer, size, offset);
259  }
260  }
261 
262  if (res.second.IsError()) {
263  logger_->Error(kLogXrdClHttp, "Could not read URL: %s, error: %s",
264  url_.c_str(), res.second.ToStr().c_str());
265  if (avoid_pread_) offset_locker.unlock();
266  return res.second;
267  }
268 
269  int num_bytes_read = res.first;
270  curr_offset = offset + num_bytes_read;
271  if (avoid_pread_) offset_locker.unlock();
272 
273  logger_->Debug(kLogXrdClHttp, "Read %d bytes, at offset %d, from URL: %s",
274  num_bytes_read, offset, url_.c_str());
275 
276  auto status = new XRootDStatus();
277  auto chunk_info = new ChunkInfo(offset, num_bytes_read, buffer);
278  auto obj = new AnyObject();
279  obj->Set(chunk_info);
280  handler->HandleResponse(status, obj);
281 
282  return XRootDStatus();
283 }
284 
286  private:
287  XrdCl::ResponseHandler *realHandler;
288  bool isChannelEncrypted;
289  public:
290  // constructor
292  bool isHttps) : realHandler(a), isChannelEncrypted(isHttps) {}
293 
294  // Response Handler
296  XrdCl::AnyObject *rdresp) {
297 
298  if( !status->IsOK() )
299  {
300  realHandler->HandleResponse( status, rdresp );
301  delete this;
302  return;
303  }
304 
305  //using namespace XrdCl;
306 
307  ChunkInfo *chunk = 0;
308  rdresp->Get(chunk);
309 
310  std::vector<uint32_t> cksums;
311  if( isChannelEncrypted )
312  {
313  size_t nbpages = chunk->length / XrdSys::PageSize;
314  if( chunk->length % XrdSys::PageSize )
315  ++nbpages;
316  cksums.reserve( nbpages );
317 
318  size_t size = chunk->length;
319  char *buffer = reinterpret_cast<char*>( chunk->buffer );
320 
321  for( size_t pg = 0; pg < nbpages; ++pg )
322  {
323  size_t pgsize = XrdSys::PageSize;
324  if( pgsize > size ) pgsize = size;
325  uint32_t crcval = XrdOucCRC::Calc32C( buffer, pgsize );
326  cksums.push_back( crcval );
327  buffer += pgsize;
328  size -= pgsize;
329  }
330  }
331 
332  PageInfo *pages = new PageInfo(chunk->offset, chunk->length, chunk->buffer, std::move(cksums));
333  delete rdresp;
334  AnyObject *response = new AnyObject();
335  response->Set( pages );
336  realHandler->HandleResponse( status, response );
337 
338  delete this;
339  }
340 };
341 
342 XRootDStatus HttpFilePlugIn::PgRead(uint64_t offset, uint32_t size, void *buffer,
343  ResponseHandler *handler,
344  uint16_t timeout) {
345  ResponseHandler *substitHandler = new PgReadSubstitutionHandler( handler, isChannelEncrypted );
346  XRootDStatus st = Read(offset, size, buffer, substitHandler, timeout);
347  return st;
348 }
349 
350 XRootDStatus HttpFilePlugIn::Write(uint64_t offset, uint32_t size,
351  const void *buffer, ResponseHandler *handler,
352  uint16_t timeout) {
353  if (!is_open_) {
354  logger_->Error(kLogXrdClHttp,
355  "Cannot write. URL hasn't previously been opened");
357  }
358 
359  // res == std::pair<int, XRootDStatus>
360  auto res =
361  Posix::PWrite(*davix_client_, davix_fd_, offset, size, buffer, timeout);
362  if (res.second.IsError()) {
363  logger_->Error(kLogXrdClHttp, "Could not write URL: %s, error: %s",
364  url_.c_str(), res.second.ToStr().c_str());
365  return res.second;
366  }
367  else
368  filesize += res.first;
369 
370  logger_->Debug(kLogXrdClHttp, "Wrote %d bytes, at offset %d, to URL: %s",
371  res.first, offset, url_.c_str());
372 
373  handler->HandleResponse(new XRootDStatus(), nullptr);
374 
375  return XRootDStatus();
376 }
377 
378 //------------------------------------------------------------------------
380 //------------------------------------------------------------------------
382  uint32_t size,
383  const void *buffer,
384  std::vector<uint32_t> &cksums,
385  ResponseHandler *handler,
386  uint16_t timeout )
387 { (void)cksums;
388  return Write(offset, size, buffer, handler, timeout);
389 }
390 
391 XRootDStatus HttpFilePlugIn::Sync(ResponseHandler *handler, uint16_t timeout) {
392  (void)handler;
393  (void)timeout;
394 
395  logger_->Debug(kLogXrdClHttp, "Sync is a no-op for HTTP.");
396 
397  return XRootDStatus();
398 }
399 
400 
402  ResponseHandler *handler,
403  uint16_t /*timeout*/) {
404  if (!is_open_) {
405  logger_->Error(kLogXrdClHttp,
406  "Cannot read. URL hasn't previously been opened");
408  }
409 
410  const auto num_chunks = chunks.size();
411  std::vector<Davix::DavIOVecInput> input_vector(num_chunks);
412  std::vector<Davix::DavIOVecOuput> output_vector(num_chunks);
413 
414  for (size_t i = 0; i < num_chunks; ++i) {
415  input_vector[i].diov_offset = chunks[i].offset;
416  input_vector[i].diov_size = chunks[i].length;
417  input_vector[i].diov_buffer = chunks[i].buffer;
418  }
419 
420  // res == std::pair<int, XRootDStatus>
421  auto res = Posix::PReadVec(*davix_client_, davix_fd_, chunks, buffer);
422  if (res.second.IsError()) {
423  logger_->Error(kLogXrdClHttp, "Could not vectorRead URL: %s, error: %s",
424  url_.c_str(), res.second.ToStr().c_str());
425  return res.second;
426  }
427 
428  int num_bytes_read = res.first;
429 
430  logger_->Debug(kLogXrdClHttp, "VecRead %d bytes, from URL: %s",
431  num_bytes_read, url_.c_str());
432 
433  char *output = static_cast<char *>(buffer);
434  for (size_t i = 0; i < num_chunks; ++i) {
435  std::memcpy(output + input_vector[i].diov_offset,
436  output_vector[i].diov_buffer, output_vector[i].diov_size);
437  }
438 
439  auto status = new XRootDStatus();
440  auto read_info = new VectorReadInfo();
441  read_info->SetSize(num_bytes_read);
442  read_info->GetChunks() = chunks;
443  auto obj = new AnyObject();
444  obj->Set(read_info);
445  handler->HandleResponse(status, obj);
446 
447  return XRootDStatus();
448 }
449 
450 bool HttpFilePlugIn::IsOpen() const { return is_open_; }
451 
452 bool HttpFilePlugIn::SetProperty(const std::string &name,
453  const std::string &value) {
454  properties_[name] = value;
455  return true;
456 }
457 
458 bool HttpFilePlugIn::GetProperty(const std::string &name,
459  std::string &value) const {
460  const auto p = properties_.find(name);
461  if (p == std::end(properties_)) {
462  return false;
463  }
464 
465  value = p->second;
466  return true;
467 }
468 
469 } // namespace XrdCl
#define HTTP_FILE_PLUG_IN_AVOIDRANGE_ENV
#define HTTP_FILE_PLUG_IN_AVOIDRANGE_CGI
void Set(Type object, bool own=true)
void Get(Type &object)
Retrieve the object being held.
virtual XRootDStatus PgRead(uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout) override
virtual XRootDStatus PgWrite(uint64_t offset, uint32_t size, const void *buffer, std::vector< uint32_t > &cksums, ResponseHandler *handler, uint16_t timeout) override
virtual XRootDStatus Sync(ResponseHandler *handler, uint16_t timeout) override
virtual XRootDStatus Stat(bool force, ResponseHandler *handler, uint16_t timeout) override
virtual XRootDStatus Write(uint64_t offset, uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout) override
virtual XRootDStatus Read(uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout) override
virtual XRootDStatus VectorRead(const ChunkList &chunks, void *buffer, XrdCl::ResponseHandler *handler, uint16_t timeout) override
virtual ~HttpFilePlugIn() noexcept
virtual XRootDStatus Open(const std::string &url, OpenFlags::Flags flags, Access::Mode mode, ResponseHandler *handler, uint16_t timeout) override
virtual bool GetProperty(const std::string &name, std::string &value) const override
virtual XRootDStatus Close(ResponseHandler *handler, uint16_t timeout) override
virtual bool SetProperty(const std::string &name, const std::string &value) override
virtual bool IsOpen() const override
void Error(uint64_t topic, const char *format,...)
Report an error.
Definition: XrdClLog.cc:231
void Debug(uint64_t topic, const char *format,...)
Print a debug message.
Definition: XrdClLog.cc:282
void HandleResponse(XrdCl::XRootDStatus *status, XrdCl::AnyObject *rdresp)
PgReadSubstitutionHandler(XrdCl::ResponseHandler *a, bool isHttps)
Handle an async response.
virtual void HandleResponse(XRootDStatus *status, AnyObject *response)
Object stat info.
URL representation.
Definition: XrdClURL.hh:31
std::map< std::string, std::string > ParamsMap
Definition: XrdClURL.hh:33
std::string GetLocation() const
Get location (protocol://host:port/path)
Definition: XrdClURL.cc:330
const ParamsMap & GetParams() const
Get the URL params.
Definition: XrdClURL.hh:239
static uint32_t Calc32C(const void *data, size_t count, uint32_t prevcs=0)
Definition: XrdOucCRC.cc:190
std::pair< int, XrdCl::XRootDStatus > PReadVec(Davix::DavPosix &davix_client, DAVIX_FD *fd, const XrdCl::ChunkList &chunks, void *buffer)
std::pair< int, XRootDStatus > Read(Davix::DavPosix &davix_client, DAVIX_FD *fd, void *buffer, uint32_t size)
std::pair< int, XrdCl::XRootDStatus > PWrite(Davix::DavPosix &davix_client, DAVIX_FD *fd, uint64_t offset, uint32_t size, const void *buffer, uint16_t timeout)
std::pair< DAVIX_FD *, XRootDStatus > Open(Davix::DavPosix &davix_client, const std::string &url, int flags, uint16_t timeout)
std::pair< int, XRootDStatus > PRead(Davix::DavPosix &davix_client, DAVIX_FD *fd, void *buffer, uint32_t size, uint64_t offset)
XRootDStatus Close(Davix::DavPosix &davix_client, DAVIX_FD *fd)
XRootDStatus Unlink(Davix::DavPosix &davix_client, const std::string &url, uint16_t timeout)
XRootDStatus MkDir(Davix::DavPosix &davix_client, const std::string &path, XrdCl::MkDirFlags::Flags flags, XrdCl::Access::Mode, uint16_t timeout)
XRootDStatus Stat(Davix::DavPosix &davix_client, const std::string &url, uint16_t timeout, StatInfo *stat_info)
const uint16_t stError
An error occurred that could potentially be retried.
Definition: XrdClStatus.hh:32
const uint16_t errInvalidOp
Definition: XrdClStatus.hh:51
Davix::Context * root_davix_context_
std::vector< ChunkInfo > ChunkList
List of chunks.
Davix::DavPosix * root_davix_client_file_
void SetUpLogging(Log *logger)
static const uint64_t kLogXrdClHttp
static const int PageSize
Mode
Access mode.
Describe a data chunk for vector read.
void * buffer
length of the chunk
uint32_t length
offset in the file
@ MakePath
create the entire directory tree if it doesn't exist
Flags
Open flags, may be or'd when appropriate.
@ Read
Open only for reading.
@ Write
Open only for writing.
@ Update
Open for reading and writing.
bool IsOK() const
We're fine.
Definition: XrdClStatus.hh:124