XRootD
XrdClUtils.cc
Go to the documentation of this file.
1 //------------------------------------------------------------------------------
2 // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN)
3 // Author: Lukasz Janyst <ljanyst@cern.ch>
4 //------------------------------------------------------------------------------
5 // XRootD is free software: you can redistribute it and/or modify
6 // it under the terms of the GNU Lesser General Public License as published by
7 // the Free Software Foundation, either version 3 of the License, or
8 // (at your option) any later version.
9 //
10 // XRootD is distributed in the hope that it will be useful,
11 // but WITHOUT ANY WARRANTY; without even the implied warranty of
12 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 // GNU General Public License for more details.
14 //
15 // You should have received a copy of the GNU Lesser General Public License
16 // along with XRootD. If not, see <http://www.gnu.org/licenses/>.
17 //------------------------------------------------------------------------------
18 
19 #include "XrdCl/XrdClUtils.hh"
20 #include "XrdCl/XrdClFileSystem.hh"
21 #include "XrdCl/XrdClDefaultEnv.hh"
22 #include "XrdCl/XrdClConstants.hh"
25 #include "XrdCl/XrdClMessage.hh"
26 #include "XrdNet/XrdNetAddr.hh"
27 
28 #include <algorithm>
29 #include <iomanip>
30 #include <sstream>
31 #include <fstream>
32 #include <functional>
33 #include <cctype>
34 #include <locale>
35 #include <map>
36 #include <string>
37 #include <set>
38 #include <cctype>
39 #include <random>
40 #include <chrono>
41 
42 #include <sys/types.h>
43 #include <dirent.h>
44 
45 #if __cplusplus < 201103L
46 #include <ctime>
47 #endif
48 
49 namespace
50 {
51  bool isNotSpace( char c )
52  {
53  return c != ' ';
54  }
55 
56  //----------------------------------------------------------------------------
57  // Ordering function for sorting IP addresses
58  //----------------------------------------------------------------------------
59  struct PreferIPv6
60  {
61  bool operator() ( const XrdNetAddr &l, const XrdNetAddr &r )
62  {
63  bool rIsIPv4 = false;
64  if( r.isIPType( XrdNetAddrInfo::IPv4 ) ||
65  (r.isIPType( XrdNetAddrInfo::IPv6 ) && r.isMapped()) )
66  rIsIPv4 = true;
67 
68  if( l.isIPType( XrdNetAddrInfo::IPv6 ) && rIsIPv4 )
69  return true;
70  return false;
71  }
72  };
73 }
74 
75 namespace XrdCl
76 {
77  //----------------------------------------------------------------------------
78  // Get a parameter either from the environment or URL
79  //----------------------------------------------------------------------------
80  int Utils::GetIntParameter( const URL &url,
81  const std::string &name,
82  int defaultVal )
83  {
84  Env *env = DefaultEnv::GetEnv();
85  int value = defaultVal;
86  char *endPtr;
87  URL::ParamsMap::const_iterator it;
88 
89  env->GetInt( name, value );
90  it = url.GetParams().find( std::string("XrdCl.") + name );
91  if( it != url.GetParams().end() )
92  {
93  int urlValue = (int)strtol( it->second.c_str(), &endPtr, 0 );
94  if( !*endPtr )
95  value = urlValue;
96  }
97  return value;
98  }
99 
100  //----------------------------------------------------------------------------
101  // Get a parameter either from the environment or URL
102  //----------------------------------------------------------------------------
103  std::string Utils::GetStringParameter( const URL &url,
104  const std::string &name,
105  const std::string &defaultVal )
106  {
107  Env *env = DefaultEnv::GetEnv();
108  std::string value = defaultVal;
109  URL::ParamsMap::const_iterator it;
110 
111  env->GetString( name, value );
112  it = url.GetParams().find( std::string("XrdCl.") + name );
113  if( it != url.GetParams().end() )
114  value = it->second;
115 
116  return value;
117  }
118 
119  //----------------------------------------------------------------------------
120  // Interpret a string as address type, default to IPAll
121  //----------------------------------------------------------------------------
122  Utils::AddressType Utils::String2AddressType( const std::string &addressType )
123  {
124  if( addressType == "IPv6" )
125  return IPv6;
126  else if( addressType == "IPv4" )
127  return IPv4;
128  else if( addressType == "IPv4Mapped6" )
129  return IPv4Mapped6;
130  else if( addressType == "IPAll" )
131  return IPAll;
132  else
133  return IPAuto;
134  }
135 
136  //----------------------------------------------------------------------------
137  // Resolve IP addresses
138  //----------------------------------------------------------------------------
139  Status Utils::GetHostAddresses( std::vector<XrdNetAddr> &addresses,
140  const URL &url,
141  Utils::AddressType type )
142  {
143  Log *log = DefaultEnv::GetLog();
144  const char *err = 0;
145  int ordn;
146 
147  //--------------------------------------------------------------------------
148  // Resolve all the addresses
149  //--------------------------------------------------------------------------
150  std::ostringstream o; o << url.GetHostName() << ":" << url.GetPort();
152 
153  if( type == IPv6 ) opts = XrdNetUtils::onlyIPv6;
154  else if( type == IPv4 ) opts = XrdNetUtils::onlyIPv4;
155  else if( type == IPv4Mapped6 ) opts = XrdNetUtils::allV4Map;
156  else if( type == IPAll ) opts = XrdNetUtils::allIPMap;
158 
159  //--------------------------------------------------------------------------
160  // Check what are the preferences IPv6 or IPv4
161  //--------------------------------------------------------------------------
162  int preferIPv4 = DefaultPreferIPv4;
163  DefaultEnv::GetEnv()->GetInt( "PreferIPv4", preferIPv4 );
164 
165  //--------------------------------------------------------------------------
166  // Partition the addresses according to the preferences
167  //
168  // The preferred IP family goes to the back as it is easier to remove
169  // items from the back of the vector
170  //--------------------------------------------------------------------------
171  opts |= (preferIPv4 ? XrdNetUtils::order64 : XrdNetUtils::order46);
172 
173  //--------------------------------------------------------------------------
174  // Now get all of the properly partitioned addresses; ordn will hold the
175  // number of non-preferred addresses at the front of the table.
176  //--------------------------------------------------------------------------
177  err = XrdNetUtils::GetAddrs( o.str(), addresses, &ordn, opts );
178 
179  if( err )
180  {
181  log->Error( UtilityMsg, "Unable to resolve %s: %s", o.str().c_str(),
182  err );
183  return Status( stError, errInvalidAddr );
184  }
185 
186  if( addresses.size() == 0 )
187  {
188  log->Error( UtilityMsg, "No addresses for %s were found",
189  o.str().c_str() );
190  return Status( stError, errInvalidAddr );
191  }
192 
193  //--------------------------------------------------------------------------
194  // Shuffle each partition
195  //--------------------------------------------------------------------------
196 
197  int ipNoShuffle = DefaultIPNoShuffle;
198  Env *env = DefaultEnv::GetEnv();
199  env->GetInt( "IPNoShuffle", ipNoShuffle );
200 
201  if( !ipNoShuffle )
202  {
203 #if __cplusplus < 201103L
204  // initialize the random generator only once
205  static struct only_once_t
206  {
207  only_once_t()
208  {
209  std::srand ( unsigned ( std::time(0) ) );
210  }
211  } only_once;
212 
213  std::random_shuffle( addresses.begin(), addresses.begin() + ordn );
214  std::random_shuffle( addresses.begin() + ordn, addresses.end() );
215 #else
216  static std::default_random_engine rand_engine(
217  std::chrono::system_clock::now().time_since_epoch().count() );
218 
219  std::shuffle( addresses.begin(), addresses.begin() + ordn, rand_engine );
220  std::shuffle( addresses.begin() + ordn, addresses.end(), rand_engine );
221 #endif
222  }
223 
224  //--------------------------------------------------------------------------
225  // Return status as the result is already in the output parameter
226  //--------------------------------------------------------------------------
227  return Status();
228  }
229 
230  //----------------------------------------------------------------------------
231  // Log all the addresses on the list
232  //----------------------------------------------------------------------------
234  uint64_t type,
235  const std::string &hostId,
236  std::vector<XrdNetAddr> &addresses )
237  {
238  std::string addrStr;
239  std::vector<XrdNetAddr>::iterator it;
240  for( it = addresses.begin(); it != addresses.end(); ++it )
241  {
242  char nameBuff[256];
243  it->Format( nameBuff, 256, XrdNetAddrInfo::fmtAdv6 );
244  addrStr += nameBuff;
245  addrStr += ", ";
246  }
247  addrStr.erase( addrStr.length()-2, 2 );
248  log->Debug( type, "[%s] Found %d address(es): %s",
249  hostId.c_str(), addresses.size(), addrStr.c_str() );
250  }
251 
252  //----------------------------------------------------------------------------
253  // Convert timestamp to a string
254  //----------------------------------------------------------------------------
255  std::string Utils::TimeToString( time_t timestamp )
256  {
257  char now[30];
258  tm tsNow;
259  time_t ttNow = timestamp;
260  localtime_r( &ttNow, &tsNow );
261  strftime( now, 30, "%Y-%m-%d %H:%M:%S %z", &tsNow );
262  return now;
263  }
264 
265  //----------------------------------------------------------------------------
266  // Get the elapsed microseconds between two timevals
267  //----------------------------------------------------------------------------
268  uint64_t Utils::GetElapsedMicroSecs( timeval start, timeval end )
269  {
270  uint64_t startUSec = start.tv_sec*1000000 + start.tv_usec;
271  uint64_t endUSec = end.tv_sec*1000000 + end.tv_usec;
272  return endUSec-startUSec;
273  }
274 
275  //----------------------------------------------------------------------------
276  // Get remote checksum
277  //----------------------------------------------------------------------------
278  XRootDStatus Utils::GetRemoteCheckSum( std::string &checkSum,
279  const std::string &checkSumType,
280  const URL &url )
281  {
282  FileSystem *fs = new FileSystem( url );
283  // add the 'cks.type' cgi tag in order to
284  // select the proper checksum type in case
285  // the server supports more than one checksum
286  size_t pos = url.GetPath().find( '?' );
287  std::string cksPath = url.GetPath() + ( pos == std::string::npos ? '?' : '&' ) + "cks.type=" + checkSumType;
288  Buffer arg; arg.FromString( cksPath );
289  Buffer *cksResponse = 0;
290  XRootDStatus st;
291  Log *log = DefaultEnv::GetLog();
292 
293  st = fs->Query( QueryCode::Checksum, arg, cksResponse );
294  delete fs;
295 
296  if( !st.IsOK() )
297  {
298  std::string msg = st.GetErrorMessage();
299  msg += " Got an error while querying the checksum!";
300  st.SetErrorMessage( msg );
301  return st;
302  }
303 
304  if( !cksResponse )
305  return XRootDStatus( stError, errInternal, 0, "Got invalid response while querying the checksum!" );
306 
307  std::vector<std::string> elems;
308  Utils::splitString( elems, cksResponse->ToString(), " " );
309  delete cksResponse;
310 
311  if( elems.size() != 2 )
312  return XRootDStatus( stError, errInvalidResponse, 0, "Got invalid response while querying the checksum!" );
313 
314  if( elems[0] != checkSumType )
316 
317  checkSum = elems[0] + ":";
318  checkSum += NormalizeChecksum( elems[0], elems[1] );
319 
320  log->Dump( UtilityMsg, "Checksum for %s checksum: %s",
321  url.GetPath().c_str(), checkSum.c_str() );
322 
323  return XRootDStatus();
324  }
325 
326  //------------------------------------------------------------------------
327  // Get a checksum from local file
328  //------------------------------------------------------------------------
329  XRootDStatus Utils::GetLocalCheckSum( std::string &checkSum,
330  const std::string &checkSumType,
331  const std::string &path )
332  {
333  Log *log = DefaultEnv::GetLog();
335 
336  if( !cksMan )
337  {
338  log->Error( UtilityMsg, "Unable to get the checksum manager" );
339  return XRootDStatus( stError, errInternal );
340  }
341 
342  XrdCksData ckSum; ckSum.Set( checkSumType.c_str() );
343  bool status = cksMan->Calculate( ckSum, checkSumType, path.c_str() );
344  if( !status )
345  {
346  log->Error( UtilityMsg, "Error while calculating checksum for %s",
347  path.c_str() );
349  }
350 
351  char *cksBuffer = new char[265];
352  ckSum.Get( cksBuffer, 256 );
353  checkSum = checkSumType + ":";
354  checkSum += NormalizeChecksum( checkSumType, cksBuffer );
355  delete [] cksBuffer;
356 
357  log->Dump( UtilityMsg, "Checksum for %s is: %s", path.c_str(),
358  checkSum.c_str() );
359 
360  return XRootDStatus();
361  }
362 
363  //----------------------------------------------------------------------------
364  // Convert bytes to a human readable string
365  //----------------------------------------------------------------------------
366  std::string Utils::BytesToString( uint64_t bytes )
367  {
368  double final = bytes;
369  int i = 0;
370  char suf[3] = { 'k', 'M', 'G' };
371  for( i = 0; i < 3 && final > 1024; ++i, final /= 1024 ) {};
372  std::ostringstream o;
373  o << std::setprecision(4) << final;
374  if( i > 0 ) o << suf[i-1];
375  return o.str();
376  }
377 
378  //----------------------------------------------------------------------------
379  // Check if peer supports tpc
380  //----------------------------------------------------------------------------
381  XRootDStatus Utils::CheckTPC( const std::string &server, uint16_t timeout )
382  {
383  Log *log = DefaultEnv::GetLog();
384  log->Debug( UtilityMsg, "Checking if the data server %s supports tpc",
385  server.c_str() );
386 
387  FileSystem sourceDSFS( server );
388  Buffer queryArg; queryArg.FromString( "tpc" );
389  Buffer *queryResponse = 0;
390  XRootDStatus st;
391  st = sourceDSFS.Query( QueryCode::Config, queryArg, queryResponse,
392  timeout );
393  if( !st.IsOK() )
394  {
395  log->Error( UtilityMsg, "Cannot query source data server %s: %s",
396  server.c_str(), st.ToStr().c_str() );
397  st.status = stFatal;
398  return st;
399  }
400 
401  if( !queryResponse )
402  {
403  log->Error( UtilityMsg, "Cannot query source data server: empty response." );
404  st.status = stFatal;
405  return st;
406  }
407 
408  std::string answer = queryResponse->ToString();
409  delete queryResponse;
410  if( answer.length() == 1 || !isdigit( answer[0] ) || atoi(answer.c_str()) == 0)
411  {
412  log->Debug( UtilityMsg, "Third party copy not supported at: %s",
413  server.c_str() );
415  }
416  log->Debug( UtilityMsg, "Third party copy supported at: %s",
417  server.c_str() );
418 
419  return XRootDStatus();
420  }
421 
422  //------------------------------------------------------------------------
423  // Check if peer supports tpc / tpc lite
424  //------------------------------------------------------------------------
425  XRootDStatus Utils::CheckTPCLite( const std::string &server, uint16_t timeout )
426  {
427  Log *log = DefaultEnv::GetLog();
428  log->Debug( UtilityMsg, "Checking if the data server %s supports tpc / tpc lite",
429  server.c_str() );
430 
431  FileSystem sourceDSFS( server );
432  Buffer queryArg; queryArg.FromString( "tpc tpcdlg" );
433  Buffer *queryResponse = 0;
434  XRootDStatus st;
435  st = sourceDSFS.Query( QueryCode::Config, queryArg, queryResponse,
436  timeout );
437  if( !st.IsOK() )
438  {
439  log->Error( UtilityMsg, "Cannot query source data server %s: %s",
440  server.c_str(), st.ToStr().c_str() );
441  st.status = stFatal;
442  return st;
443  }
444 
445  if( !queryResponse )
446  {
447  log->Error( UtilityMsg, "Cannot query source data server: empty response." );
448  st.status = stFatal;
449  return st;
450  }
451 
452  std::string answer = queryResponse->ToString();
453  delete queryResponse;
454 
455  if( answer.empty() )
456  {
457  log->Error( UtilityMsg, "Cannot query source data server: empty response." );
458  st.status = stFatal;
459  return st;
460  }
461 
462  std::vector<std::string> resp;
463  Utils::splitString( resp, answer, "\n" );
464 
465  if( resp.empty() || resp[0].empty() ||
466  !isdigit( resp[0][0]) || atoi( resp[0].c_str() ) == 0 )
467  {
468  log->Debug( UtilityMsg, "Third party copy not supported at: %s",
469  server.c_str() );
471  }
472 
473  if( resp.size() == 1 || resp[1] == "tpcdlg" )
474  {
475  log->Debug( UtilityMsg, "TPC lite not supported at: %s",
476  server.c_str() );
477  return XRootDStatus( stOK, suPartial );
478  }
479 
480  log->Debug( UtilityMsg, "TPC lite supported at: %s",
481  server.c_str() );
482 
483  return XRootDStatus();
484  }
485 
486  //----------------------------------------------------------------------------
487  // Convert the fully qualified host name to country code
488  //----------------------------------------------------------------------------
489  std::string Utils::FQDNToCC( const std::string &fqdn )
490  {
491  std::vector<std::string> el;
492  Utils::splitString( el, fqdn, "." );
493  if( el.size() < 2 )
494  return "us";
495 
496  std::string cc = *el.rbegin();
497  if( cc.length() == 2 )
498  return cc;
499  return "us";
500  }
501 
502  //----------------------------------------------------------------------------
503  // Get directory entries
504  //----------------------------------------------------------------------------
505  Status Utils::GetDirectoryEntries( std::vector<std::string> &entries,
506  const std::string &path )
507  {
508  DIR *dp = opendir( path.c_str() );
509  if( !dp )
510  return Status( stError, errOSError, errno );
511 
512  dirent *dirEntry;
513 
514  while( (dirEntry = readdir(dp)) != 0 )
515  {
516  std::string entryName = dirEntry->d_name;
517  if( !entryName.compare( 0, 2, "..") )
518  continue;
519  if( !entryName.compare( 0, 1, ".") )
520  continue;
521 
522  entries.push_back( dirEntry->d_name );
523  }
524 
525  closedir(dp);
526 
527  return Status();
528  }
529 
530  //----------------------------------------------------------------------------
531  // Process a config file and return key-value pairs
532  //----------------------------------------------------------------------------
533  Status Utils::ProcessConfig( std::map<std::string, std::string> &config,
534  const std::string &file )
535  {
536  config.clear();
537  std::ifstream inFile( file.c_str() );
538  if( !inFile.good() )
539  return Status( stError, errOSError, errno );
540 
541  errno = 0;
542  std::string line;
543  while( std::getline( inFile, line ) )
544  {
545  if( line.empty() || line[0] == '#' )
546  continue;
547 
548  std::vector<std::string> elems;
549  splitString( elems, line, "=" );
550  if( elems.size() != 2 )
551  return Status( stError, errConfig );
552  std::string key = elems[0]; Trim( key );
553  std::string value = elems[1]; Trim( value );
554  config[key] = value;
555  }
556 
557  if( errno )
558  return Status( stError, errOSError, errno );
559  return Status();
560  }
561 
562  //------------------------------------------------------------------------
564  //------------------------------------------------------------------------
565  Status Utils::ProcessConfigDir( std::map<std::string, std::string> &config,
566  const std::string &dir )
567  {
568  Log *log = DefaultEnv::GetLog();
569  log->Debug( UtilityMsg, "Processing configuration files in %s...",
570  dir.c_str());
571 
572  std::vector<std::string> entries;
573  Status st = Utils::GetDirectoryEntries( entries, dir );
574  if( !st.IsOK() )
575  {
576  log->Debug( UtilityMsg, "Unable to process directory %s: %s",
577  dir.c_str(), st.ToString().c_str() );
578  return st;
579  }
580 
581  static const std::string suffix = ".conf";
582  for( auto &entry : entries )
583  {
584  std::string confFile = dir + "/" + entry;
585 
586  if( confFile.length() <= suffix.length() )
587  continue;
588  if( !std::equal( suffix.rbegin(), suffix.rend(), confFile.rbegin() ) )
589  continue;
590 
591  st = ProcessConfig( config, confFile );
592  if( !st.IsOK() )
593  {
594  log->Debug( UtilityMsg, "Unable to process configuration file %s: %s",
595  confFile.c_str(), st.ToString().c_str() );
596  }
597  }
598 
599  return Status();
600  }
601 
602  //----------------------------------------------------------------------------
603  // Trim a string
604  //----------------------------------------------------------------------------
605  void Utils::Trim( std::string &str )
606  {
607  str.erase( str.begin(),
608  std::find_if( str.begin(), str.end(), isNotSpace ) );
609  str.erase( std::find_if( str.rbegin(), str.rend(), isNotSpace ).base(),
610  str.end() );
611  }
612 
613  //----------------------------------------------------------------------------
614  // Log property list
615  //----------------------------------------------------------------------------
617  uint64_t topic,
618  const char *format,
619  const PropertyList &list )
620  {
621  PropertyList::PropertyMap::const_iterator it;
622  std::string keyVals;
623  for( it = list.begin(); it != list.end(); ++it )
624  keyVals += "'" + it->first + "' = '" + it->second + "', ";
625  keyVals.erase( keyVals.length()-2, 2 );
626  log->Dump( topic, format, keyVals.c_str() );
627  }
628 
629  //----------------------------------------------------------------------------
630  // Print a char array as hex
631  //----------------------------------------------------------------------------
632  std::string Utils::Char2Hex( uint8_t *array, uint16_t size )
633  {
634  char *hex = new char[2*size+1];
635  for( uint16_t i = 0; i < size; ++i )
636  snprintf( hex+(2*i), 3, "%02x", (int)array[i] );
637  std::string result = hex;
638  delete [] hex;
639  return result;
640  }
641 
642  //----------------------------------------------------------------------------
643  // Normalize checksum
644  //----------------------------------------------------------------------------
645  std::string Utils::NormalizeChecksum( const std::string &name,
646  const std::string &checksum )
647  {
648  if( name == "adler32" || name == "crc32" )
649  {
650  size_t i;
651  for( i = 0; i < checksum.length(); ++i )
652  if( checksum[i] != '0' )
653  break;
654  return checksum.substr(i);
655  }
656  return checksum;
657  }
658 
659  //----------------------------------------------------------------------------
660  // Get supported checksum types for given URL
661  //----------------------------------------------------------------------------
662  std::vector<std::string> Utils::GetSupportedCheckSums( const XrdCl::URL &url )
663  {
664  std::vector<std::string> ret;
665 
666  FileSystem fs( url );
667  Buffer arg; arg.FromString( "chksum" );
668  Buffer *resp = 0;
669  XRootDStatus st = fs.Query( QueryCode::Config, arg, resp );
670  if( st.IsOK() )
671  {
672  std::string response = resp->ToString();
673  if( response != "chksum" )
674  {
675  // we are expecting a response of format: '0:zcrc32,1:adler32'
676  std::vector<std::string> result;
677  Utils::splitString( result, response, "," );
678 
679  std::vector<std::string>::iterator itr = result.begin();
680  for( ; itr != result.end(); ++itr )
681  {
682  size_t pos = itr->find( ':' );
683  if( pos == std::string::npos ) continue;
684  std::string cksname = itr->substr( pos + 1 );
685  // remove all white spaces
686  cksname.erase( std::remove_if( cksname.begin(), cksname.end(), ::isspace ),
687  cksname.end() );
688  ret.push_back( std::move( cksname ) );
689  }
690  }
691  }
692 
693  return ret;
694  }
695 
696 
697  //------------------------------------------------------------------------
699  //------------------------------------------------------------------------
700  bool Utils::CheckEC( const Message *req, const URL &url )
701  {
702 #ifdef WITH_XRDEC
703  // make sure that if we will be writing it is a new file
704  ClientRequest *request = (ClientRequest*)req->GetBuffer();
705  uint16_t options = ntohs( request->open.options );
706  bool open_wrt = ( options & kXR_open_updt ) || ( options & kXR_open_wrto );
707  bool open_new = ( options & kXR_new );
708  if( open_wrt && !open_new ) return false;
709 
710  const URL::ParamsMap &params = url.GetParams();
711  // make sure all the xrdec. tokens are present and the values are sane
712  URL::ParamsMap::const_iterator itr = params.find( "xrdec.nbdta" );
713  if( itr == params.end() ) return false;
714  size_t nbdta = std::stoul( itr->second );
715 
716  itr = params.find( "xrdec.nbprt" );
717  if( itr == params.end() ) return false;
718  size_t nbprt = std::stoul( itr->second );
719 
720  itr = params.find( "xrdec.blksz" );
721  if( itr == params.end() ) return false;
722 
723  itr = params.find( "xrdec.plgr" );
724  if( itr == params.end() ) return false;
725  std::vector<std::string> plgr;
726  splitString( plgr, itr->second, "," );
727  if( plgr.size() < nbdta + nbprt ) return false;
728 
729  itr = params.find( "xrdec.objid" );
730  if( itr == params.end() ) return false;
731 
732  itr = params.find( "xrdec.format" );
733  if( itr == params.end() ) return false;
734  size_t format = std::stoul( itr->second );
735  if( format != 1 ) return false; // TODO use constant
736 
737  itr = params.find( "xrdec.dtacgi" );
738  if( itr != params.end() )
739  {
740  std::vector<std::string> dtacgi;
741  splitString( dtacgi, itr->second, "," );
742  if( plgr.size() != dtacgi.size() ) return false;
743  }
744 
745  itr = params.find( "xrdec.mdtacgi" );
746  if( itr != params.end() )
747  {
748  std::vector<std::string> mdtacgi;
749  splitString( mdtacgi, itr->second, "," );
750  if( plgr.size() != mdtacgi.size() ) return false;
751  }
752 
753  itr = params.find( "xrdec.cosc" );
754  if( itr == params.end() ) return false;
755  std::string cosc = itr->second;
756  if( cosc != "true" && cosc != "false" ) return false;
757 
758  return true;
759 #else
760  return false;
761 #endif
762  }
763 
764 
765  //----------------------------------------------------------------------------
767  //----------------------------------------------------------------------------
768  std::string Utils::InferChecksumType( const XrdCl::URL &source,
769  const XrdCl::URL &destination,
770  bool zip)
771  {
772  //--------------------------------------------------------------------------
773  // If both files are local we won't be checksumming at all
774  //--------------------------------------------------------------------------
775  if( source.IsLocalFile() && !source.IsMetalink() && destination.IsLocalFile() ) return std::string();
776 
777  // checksums supported by local files
778  std::set<std::string> local_supported;
779  local_supported.insert( "adler32" );
780  local_supported.insert( "crc32" );
781  local_supported.insert( "md5" );
782  local_supported.insert( "zcrc32" );
783 
784  std::vector<std::string> srccks;
785 
786  if( source.IsMetalink() )
787  {
788  int useMtlnCksum = DefaultZipMtlnCksum;
789  Env *env = DefaultEnv::GetEnv();
790  env->GetInt( "ZipMtlnCksum", useMtlnCksum );
791 
792  //------------------------------------------------------------------------
793  // In case of ZIP use other checksums than zcrc32 only if the user
794  // requested it explicitly.
795  //------------------------------------------------------------------------
796  if( !zip || ( zip && useMtlnCksum ) )
797  {
799  VirtualRedirector *redirector = registry.Get( source );
800  std::vector<std::string> cks = redirector->GetSupportedCheckSums();
801  srccks.insert( srccks.end(), cks.begin(), cks.end() );
802  }
803  }
804 
805  if( zip )
806  {
807  //------------------------------------------------------------------------
808  // In case of ZIP we can always extract the checksum from the archive
809  //------------------------------------------------------------------------
810  srccks.push_back( "zcrc32" );
811  }
812  else if( source.GetProtocol() == "root" || source.GetProtocol() == "xroot" )
813  {
814  //------------------------------------------------------------------------
815  // If the source is a remote endpoint query the supported checksums
816  //------------------------------------------------------------------------
817  std::vector<std::string> cks = GetSupportedCheckSums( source );
818  srccks.insert( srccks.end(), cks.begin(), cks.end() );
819  }
820 
821  std::vector<std::string> dstcks;
822 
823  if( destination.GetProtocol() == "root" ||
824  destination.GetProtocol() == "xroot" )
825  {
826  //------------------------------------------------------------------------
827  // If the destination is a remote endpoint query the supported checksums
828  //------------------------------------------------------------------------
829  std::vector<std::string> cks = GetSupportedCheckSums( destination );
830  dstcks.insert( dstcks.end(), cks.begin(), cks.end() );
831  }
832 
833  //--------------------------------------------------------------------------
834  // Now we have all the information we need, we can infer the right checksum
835  // type!!!
836  //
837  // First check if source is local
838  //--------------------------------------------------------------------------
839  if( source.IsLocalFile() && !source.IsMetalink() )
840  {
841  std::vector<std::string>::iterator itr = dstcks.begin();
842  for( ; itr != dstcks.end(); ++itr )
843  if( local_supported.count( *itr ) ) return *itr;
844  return std::string();
845  }
846 
847  //--------------------------------------------------------------------------
848  // then check if destination is local
849  //--------------------------------------------------------------------------
850  if( destination.IsLocalFile() )
851  {
852  std::vector<std::string>::iterator itr = srccks.begin();
853  for( ; itr != srccks.end(); ++itr )
854  if( local_supported.count( *itr ) ) return *itr;
855  return std::string();
856  }
857 
858  //--------------------------------------------------------------------------
859  // if both source and destination are remote look for a checksum that can
860  // satisfy both
861  //--------------------------------------------------------------------------
862  std::set<std::string> dst_supported( dstcks.begin(), dstcks.end() );
863  std::vector<std::string>::iterator itr = srccks.begin();
864  for( ; itr != srccks.end(); ++itr )
865  if( dst_supported.count( *itr ) ) return *itr;
866  return std::string();
867  }
868 
869  //----------------------------------------------------------------------------
871  //----------------------------------------------------------------------------
872  void Utils::SplitChunks( std::vector<ChunkList> &listsvec,
873  const ChunkList &chunks,
874  const uint32_t maxcs,
875  const size_t maxc )
876  {
877  listsvec.clear();
878  if( !chunks.size() ) return;
879 
880  listsvec.emplace_back();
881  ChunkList *c = &listsvec.back();
882  const size_t cs = chunks.size();
883  size_t idx = 0;
884  size_t nc = 0;
885  ChunkInfo tmpc;
886 
887  c->reserve( cs );
888 
889  while( idx < cs )
890  {
891  if( maxc && nc >= maxc )
892  {
893  listsvec.emplace_back();
894  c = &listsvec.back();
895  c->reserve( cs - idx );
896  nc = 0;
897  }
898 
899  if( tmpc.length == 0 )
900  tmpc = chunks[idx];
901 
902  if( maxcs && tmpc.length > maxcs )
903  {
904  c->emplace_back( tmpc.offset, maxcs, tmpc.buffer );
905  tmpc.offset += maxcs;
906  tmpc.length -= maxcs;
907  tmpc.buffer = static_cast<char*>( tmpc.buffer ) + maxcs;
908  }
909  else
910  {
911  c->emplace_back( tmpc.offset, tmpc.length, tmpc.buffer );
912  tmpc.length = 0;
913  ++idx;
914  }
915  ++nc;
916  }
917  }
918 }
kXR_unt16 options
Definition: XProtocol.hh:481
@ kXR_open_wrto
Definition: XProtocol.hh:469
@ kXR_open_updt
Definition: XProtocol.hh:457
@ kXR_new
Definition: XProtocol.hh:455
struct ClientOpenRequest open
Definition: XProtocol.hh:858
struct dirent * readdir(DIR *dirp)
int closedir(DIR *dirp)
DIR * opendir(const char *path)
void getline(uchar *buff, int blen)
struct myOpts opts
int Set(const char *csName)
Definition: XrdCksData.hh:81
int Get(char *Buff, int Blen)
Definition: XrdCksData.hh:69
Binary blob representation.
Definition: XrdClBuffer.hh:34
void FromString(const std::string str)
Fill the buffer from a string.
Definition: XrdClBuffer.hh:205
const char * GetBuffer(uint32_t offset=0) const
Get the message buffer.
Definition: XrdClBuffer.hh:72
std::string ToString() const
Convert the buffer to a string.
Definition: XrdClBuffer.hh:215
Manage the checksum calc objects.
bool Calculate(XrdCksData &result, const std::string &algName, const std::string &filePath)
Calculate a checksum of for a given file.
static CheckSumManager * GetCheckSumManager()
Get checksum manager.
static Log * GetLog()
Get default log.
static Env * GetEnv()
Get default client environment.
bool GetString(const std::string &key, std::string &value)
Definition: XrdClEnv.cc:31
bool GetInt(const std::string &key, int &value)
Definition: XrdClEnv.cc:89
Send file/filesystem queries to an XRootD cluster.
XRootDStatus Query(QueryCode::Code queryCode, const Buffer &arg, ResponseHandler *handler, uint16_t timeout=0) XRD_WARN_UNUSED_RESULT
Handle diagnostics.
Definition: XrdClLog.hh:101
void Error(uint64_t topic, const char *format,...)
Report an error.
Definition: XrdClLog.cc:231
void Dump(uint64_t topic, const char *format,...)
Print a dump message.
Definition: XrdClLog.cc:299
void Debug(uint64_t topic, const char *format,...)
Print a debug message.
Definition: XrdClLog.cc:282
The message representation used throughout the system.
Definition: XrdClMessage.hh:30
A key-value pair map storing both keys and values as strings.
PropertyMap::const_iterator end() const
Get the end iterator.
PropertyMap::const_iterator begin() const
Get the begin iterator.
Singleton access to URL to virtual redirector mapping.
static RedirectorRegistry & Instance()
Returns reference to the single instance.
VirtualRedirector * Get(const URL &url) const
Get a virtual redirector associated with the given URL.
URL representation.
Definition: XrdClURL.hh:31
bool IsMetalink() const
Is it a URL to a metalink.
Definition: XrdClURL.cc:451
const std::string & GetHostName() const
Get the name of the target host.
Definition: XrdClURL.hh:165
std::map< std::string, std::string > ParamsMap
Definition: XrdClURL.hh:33
const std::string & GetProtocol() const
Get the protocol.
Definition: XrdClURL.hh:113
bool IsLocalFile() const
Definition: XrdClURL.cc:460
const ParamsMap & GetParams() const
Get the URL params.
Definition: XrdClURL.hh:239
const std::string & GetPath() const
Get the path.
Definition: XrdClURL.hh:212
int GetPort() const
Get the target port.
Definition: XrdClURL.hh:183
static std::string TimeToString(time_t timestamp)
Convert timestamp to a string.
Definition: XrdClUtils.cc:255
static XRootDStatus CheckTPCLite(const std::string &server, uint16_t timeout=0)
Definition: XrdClUtils.cc:425
static void LogHostAddresses(Log *log, uint64_t type, const std::string &hostId, std::vector< XrdNetAddr > &addresses)
Log all the addresses on the list.
Definition: XrdClUtils.cc:233
static std::string NormalizeChecksum(const std::string &name, const std::string &checksum)
Normalize checksum.
Definition: XrdClUtils.cc:645
static Status ProcessConfig(std::map< std::string, std::string > &config, const std::string &file)
Process a config file and return key-value pairs.
Definition: XrdClUtils.cc:533
static Status ProcessConfigDir(std::map< std::string, std::string > &config, const std::string &dir)
Process a config directory and return key-value pairs.
Definition: XrdClUtils.cc:565
static std::string FQDNToCC(const std::string &fqdn)
Convert the fully qualified host name to country code.
Definition: XrdClUtils.cc:489
static std::string InferChecksumType(const XrdCl::URL &source, const XrdCl::URL &destination, bool zip=false)
Automatically infer the right checksum type.
Definition: XrdClUtils.cc:768
static void LogPropertyList(Log *log, uint64_t topic, const char *format, const PropertyList &list)
Log property list.
Definition: XrdClUtils.cc:616
static std::string Char2Hex(uint8_t *array, uint16_t size)
Print a char array as hex.
Definition: XrdClUtils.cc:632
static void splitString(Container &result, const std::string &input, const std::string &delimiter)
Split a string.
Definition: XrdClUtils.hh:56
static Status GetHostAddresses(std::vector< XrdNetAddr > &addresses, const URL &url, AddressType type)
Resolve IP addresses.
Definition: XrdClUtils.cc:139
static uint64_t GetElapsedMicroSecs(timeval start, timeval end)
Get the elapsed microseconds between two timevals.
Definition: XrdClUtils.cc:268
static std::vector< std::string > GetSupportedCheckSums(const XrdCl::URL &url)
Get supported checksum types for given URL.
Definition: XrdClUtils.cc:662
static AddressType String2AddressType(const std::string &addressType)
Interpret a string as address type, default to IPAll.
Definition: XrdClUtils.cc:122
static int GetIntParameter(const URL &url, const std::string &name, int defaultVal)
Get a parameter either from the environment or URL.
Definition: XrdClUtils.cc:80
static Status GetDirectoryEntries(std::vector< std::string > &entries, const std::string &path)
Get directory entries.
Definition: XrdClUtils.cc:505
static XRootDStatus GetLocalCheckSum(std::string &checkSum, const std::string &checkSumType, const std::string &path)
Get a checksum from local file.
Definition: XrdClUtils.cc:329
static std::string BytesToString(uint64_t bytes)
Convert bytes to a human readable string.
Definition: XrdClUtils.cc:366
static void Trim(std::string &str)
Trim a string.
Definition: XrdClUtils.cc:605
static bool CheckEC(const Message *req, const URL &url)
Check if this client can support given EC redirect.
Definition: XrdClUtils.cc:700
static XRootDStatus GetRemoteCheckSum(std::string &checkSum, const std::string &checkSumType, const URL &url)
Get a checksum from a remote xrootd server.
Definition: XrdClUtils.cc:278
static std::string GetStringParameter(const URL &url, const std::string &name, const std::string &defaultVal)
Get a parameter either from the environment or URL.
Definition: XrdClUtils.cc:103
static XRootDStatus CheckTPC(const std::string &server, uint16_t timeout=0)
Check if peer supports tpc.
Definition: XrdClUtils.cc:381
AddressType
Address type.
Definition: XrdClUtils.hh:87
static void SplitChunks(std::vector< ChunkList > &listsvec, const ChunkList &chunks, const uint32_t maxcs, const size_t maxc)
Split chunks in a ChunkList into one or more ChunkLists.
Definition: XrdClUtils.cc:872
An interface for metadata redirectors.
virtual std::vector< std::string > GetSupportedCheckSums() const =0
void SetErrorMessage(const std::string &message)
Set the error message.
const std::string & GetErrorMessage() const
Get error message.
std::string ToStr() const
Convert to string.
bool isMapped() const
bool isIPType(IPType ipType) const
static const char * GetAddrs(const char *hSpec, XrdNetAddr *aListP[], int &aListN, AddrOpts opts=allIPMap, int pNum=PortInSpec)
Definition: XrdNetUtils.cc:238
const uint16_t suPartial
Definition: XrdClStatus.hh:41
const uint16_t errInvalidAddr
Definition: XrdClStatus.hh:71
const uint16_t stFatal
Fatal error, it's still an error.
Definition: XrdClStatus.hh:33
const uint16_t stError
An error occurred that could potentially be retried.
Definition: XrdClStatus.hh:32
const uint16_t errInternal
Internal error.
Definition: XrdClStatus.hh:56
const uint16_t stOK
Everything went OK.
Definition: XrdClStatus.hh:31
const int DefaultIPNoShuffle
const uint16_t errConfig
System misconfigured.
Definition: XrdClStatus.hh:55
const uint16_t errOSError
Definition: XrdClStatus.hh:61
const uint64_t UtilityMsg
const uint16_t errInvalidResponse
Definition: XrdClStatus.hh:99
std::vector< ChunkInfo > ChunkList
List of chunks.
const uint16_t errNotSupported
Definition: XrdClStatus.hh:62
const int DefaultPreferIPv4
const uint16_t errCheckSumError
Definition: XrdClStatus.hh:101
const int DefaultZipMtlnCksum
@ hex
Definition: XrdSysTrace.hh:42
Describe a data chunk for vector read.
void * buffer
length of the chunk
uint32_t length
offset in the file
@ Config
Query server configuration.
@ Checksum
Query file checksum.
Procedure execution status.
Definition: XrdClStatus.hh:115
uint16_t status
Status of the execution.
Definition: XrdClStatus.hh:146
bool IsOK() const
We're fine.
Definition: XrdClStatus.hh:124
std::string ToString() const
Create a string representation.
Definition: XrdClStatus.cc:97