libzypp  17.36.1
MediaCurl.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
13 #include <iostream>
14 #include <chrono>
15 #include <list>
16 
17 #include <zypp/base/Logger.h>
18 #include <zypp/ExternalProgram.h>
19 #include <zypp/base/String.h>
20 #include <zypp/base/Gettext.h>
21 #include <utility>
22 #include <zypp-core/parser/Sysconfig>
23 #include <zypp/base/Gettext.h>
24 
25 #include <zypp/media/MediaCurl.h>
28 #include <zypp-curl/ProxyInfo>
29 #include <zypp-curl/auth/CurlAuthData>
30 #include <zypp-media/auth/CredentialManager>
31 #include <zypp-curl/CurlConfig>
33 #include <zypp/Target.h>
34 #include <zypp/ZYppFactory.h>
35 #include <zypp/ZConfig.h>
36 #include <zypp/zypp_detail/ZYppImpl.h> // for zypp_poll
37 
38 #include <cstdlib>
39 #include <sys/types.h>
40 #include <sys/stat.h>
41 #include <sys/mount.h>
42 #include <dirent.h>
43 #include <unistd.h>
44 #include <glib.h>
45 
46 using std::endl;
47 
48 namespace internal {
49  using namespace zypp;
53  struct OptionalDownloadProgressReport : public callback::ReceiveReport<media::DownloadProgressReport>
54  {
55  using TimePoint = std::chrono::steady_clock::time_point;
56 
57  OptionalDownloadProgressReport( bool isOptional=false )
58  : _oldRec { Distributor::instance().getReceiver() }
59  , _isOptional { isOptional }
60  { connect(); }
61 
66 
68  if (_oldRec)
69  Distributor::instance().setReceiver(*_oldRec);
70  else
71  Distributor::instance().noReceiver();
72  }
73 
74  void reportbegin() override
75  { if ( _oldRec ) _oldRec->reportbegin(); }
76 
77  void reportend() override
78  { if ( _oldRec ) _oldRec->reportend(); }
79 
80  void report( const UserData & userData_r = UserData() ) override
81  { if ( _oldRec ) _oldRec->report( userData_r ); }
82 
83 
84  void start( const Url & file_r, Pathname localfile_r ) override
85  {
86  if ( not _oldRec ) return;
87  if ( _isOptional ) {
88  // delay start until first data are received.
89  _startFile = file_r;
90  _startLocalfile = std::move(localfile_r);
91  return;
92  }
93  _oldRec->start( file_r, localfile_r );
94  }
95 
96  bool progress( int value_r, const Url & file_r, double dbps_avg_r = -1, double dbps_current_r = -1 ) override
97  {
98  if ( not _oldRec ) return true;
99  if ( notStarted() ) {
100  if ( not ( value_r || dbps_avg_r || dbps_current_r ) )
101  return true;
102  sendStart();
103  }
104 
105  //static constexpr std::chrono::milliseconds minfequency { 1000 }; only needed if we'd avoid sending reports without change
106  static constexpr std::chrono::milliseconds maxfequency { 100 };
107  TimePoint now { TimePoint::clock::now() };
108  TimePoint::duration elapsed { now - _lastProgressSent };
109  if ( elapsed < maxfequency )
110  return true; // continue
111  _lastProgressSent = now;
112  return _oldRec->progress( value_r, file_r, dbps_avg_r, dbps_current_r );
113  }
114 
115  Action problem( const Url & file_r, Error error_r, const std::string & description_r ) override
116  {
117  if ( not _oldRec || notStarted() ) return ABORT;
118  return _oldRec->problem( file_r, error_r, description_r );
119  }
120 
121  void finish( const Url & file_r, Error error_r, const std::string & reason_r ) override
122  {
123  if ( not _oldRec || notStarted() ) return;
124  _oldRec->finish( file_r, error_r, reason_r );
125  }
126 
127  private:
128  // _isOptional also indicates the delayed start
129  bool notStarted() const
130  { return _isOptional; }
131 
132  void sendStart()
133  {
134  if ( _isOptional ) {
135  // we know _oldRec is valid...
136  _oldRec->start( _startFile, std::move(_startLocalfile) );
137  _isOptional = false;
138  }
139  }
140 
141  private:
142  Receiver *const _oldRec;
145  Pathname _startLocalfile;
147  };
148 
150  {
151  ProgressData( CURL *curl, time_t timeout = 0, zypp::Url url = zypp::Url(),
152  zypp::ByteCount expectedFileSize_r = 0,
154 
155  void updateStats( curl_off_t dltotal = 0.0, curl_off_t dlnow = 0.0 );
156 
157  int reportProgress() const;
158 
159  CURL * curl()
160  { return _curl; }
161 
162  bool timeoutReached() const
163  { return _timeoutReached; }
164 
165  bool fileSizeExceeded() const
166  { return _fileSizeExceeded; }
167 
169  { return _expectedFileSize; }
170 
171  void expectedFileSize( ByteCount newval_r )
172  { _expectedFileSize = newval_r; }
173 
174  private:
175  CURL * _curl;
177  time_t _timeout;
182 
183  time_t _timeStart = 0;
184  time_t _timeLast = 0;
185  time_t _timeRcv = 0;
186  time_t _timeNow = 0;
187 
188  curl_off_t _dnlTotal = 0.0;
189  curl_off_t _dnlLast = 0.0;
190  curl_off_t _dnlNow = 0.0;
191 
192  int _dnlPercent= 0;
193 
194  double _drateTotal= 0.0;
195  double _drateLast = 0.0;
196  };
197 
198 
199 
201  : _curl( curl )
202  , _url(std::move( url ))
203  , _timeout( timeout )
204  , _timeoutReached( false )
205  , _fileSizeExceeded ( false )
206  , _expectedFileSize( expectedFileSize_r )
207  , report( _report )
208  {}
209 
210  void ProgressData::updateStats( curl_off_t dltotal, curl_off_t dlnow )
211  {
212  time_t now = _timeNow = time(0);
213 
214  // If called without args (0.0), recompute based on the last values seen
215  if ( dltotal && dltotal != _dnlTotal )
216  _dnlTotal = dltotal;
217 
218  if ( dlnow && dlnow != _dnlNow )
219  {
220  _timeRcv = now;
221  _dnlNow = dlnow;
222  }
223 
224  // init or reset if time jumps back
225  if ( !_timeStart || _timeStart > now )
226  _timeStart = _timeLast = _timeRcv = now;
227 
228  // timeout condition
229  if ( _timeout )
230  _timeoutReached = ( (now - _timeRcv) > _timeout );
231 
232  // check if the downloaded data is already bigger than what we expected
233  _fileSizeExceeded = _expectedFileSize > 0 && _expectedFileSize < static_cast<ByteCount::SizeType>(_dnlNow);
234 
235  // percentage:
236  if ( _dnlTotal )
237  _dnlPercent = int( _dnlNow * 100 / _dnlTotal );
238 
239  // download rates:
240  _drateTotal = double(_dnlNow) / std::max( int(now - _timeStart), 1 );
241 
242  if ( _timeLast < now )
243  {
244  _drateLast = double(_dnlNow - _dnlLast) / int(now - _timeLast);
245  // start new period
246  _timeLast = now;
247  _dnlLast = _dnlNow;
248  }
249  else if ( _timeStart == _timeLast )
251  }
252 
254  {
255  if ( _fileSizeExceeded )
256  return 1;
257  if ( _timeoutReached )
258  return 1; // no-data timeout
259  if ( report && !(*report)->progress( _dnlPercent, _url, _drateTotal, _drateLast ) )
260  return 1; // user requested abort
261  return 0;
262  }
263 
264  const char * anonymousIdHeader()
265  {
266  // we need to add the release and identifier to the
267  // agent string.
268  // The target could be not initialized, and then this information
269  // is guessed.
270  // bsc#1212187: HTTP/2 RFC 9113 forbids fields ending with a space
271  static const std::string _value( str::trim( str::form(
272  "X-ZYpp-AnonymousId: %s",
273  Target::anonymousUniqueId( Pathname()/*guess root*/ ).c_str()
274  )));
275  return _value.c_str();
276  }
277 
279  {
280  // we need to add the release and identifier to the
281  // agent string.
282  // The target could be not initialized, and then this information
283  // is guessed.
284  // bsc#1212187: HTTP/2 RFC 9113 forbids fields ending with a space
285  static const std::string _value( str::trim( str::form(
286  "X-ZYpp-DistributionFlavor: %s",
287  Target::distributionFlavor( Pathname()/*guess root*/ ).c_str()
288  )));
289  return _value.c_str();
290  }
291 
292  const char * agentString()
293  {
294  // we need to add the release and identifier to the
295  // agent string.
296  // The target could be not initialized, and then this information
297  // is guessed.
298  // bsc#1212187: HTTP/2 RFC 9113 forbids fields ending with a space
299  static const std::string _value( str::trim( str::form(
300  "ZYpp " LIBZYPP_VERSION_STRING " (curl %s) %s"
301  , curl_version_info(CURLVERSION_NOW)->version
302  , Target::targetDistribution( Pathname()/*guess root*/ ).c_str()
303  )));
304  return _value.c_str();
305  }
306 
311  {
312  public:
314  const std::string & err_r,
315  const std::string & msg_r )
316  : media::MediaCurlException( url_r, err_r, msg_r )
317  {}
318  //~MediaCurlExceptionMayRetryInternaly() noexcept {}
319  };
320 
321 }
322 
323 
324 using namespace internal;
325 using namespace zypp::base;
326 
327 namespace zypp {
328 
329  namespace media {
330 
331 Pathname MediaCurl::_cookieFile = "/var/lib/YaST2/cookies";
332 
333 // we use this define to unbloat code as this C setting option
334 // and catching exception is done frequently.
336 #define SET_OPTION(opt,val) do { \
337  ret = curl_easy_setopt ( _curl, opt, val ); \
338  if ( ret != 0) { \
339  ZYPP_THROW(MediaCurlSetOptException(_url, _curlError)); \
340  } \
341  } while ( false )
342 
343 #define SET_OPTION_OFFT(opt,val) SET_OPTION(opt,(curl_off_t)val)
344 #define SET_OPTION_LONG(opt,val) SET_OPTION(opt,(long)val)
345 #define SET_OPTION_VOID(opt,val) SET_OPTION(opt,(void*)val)
346 
347 MediaCurl::MediaCurl( const Url & url_r,
348  const Pathname & attach_point_hint_r )
349  : MediaNetworkCommonHandler( url_r, attach_point_hint_r,
350  "/", // urlpath at attachpoint
351  true ), // does_download
352  _curl( NULL ),
353  _customHeaders(0L)
354 {
355  _curlError[0] = '\0';
356 
357  MIL << "MediaCurl::MediaCurl(" << url_r << ", " << attach_point_hint_r << ")" << endl;
358 
360 
361  if( !attachPoint().empty())
362  {
363  PathInfo ainfo(attachPoint());
364  Pathname apath(attachPoint() + "XXXXXX");
365  char *atemp = ::strdup( apath.asString().c_str());
366  char *atest = NULL;
367  if( !ainfo.isDir() || !ainfo.userMayRWX() ||
368  atemp == NULL || (atest=::mkdtemp(atemp)) == NULL)
369  {
370  WAR << "attach point " << ainfo.path()
371  << " is not useable for " << url_r.getScheme() << endl;
372  setAttachPoint("", true);
373  }
374  else if( atest != NULL)
375  ::rmdir(atest);
376 
377  if( atemp != NULL)
378  ::free(atemp);
379  }
380 }
381 
383 {
385 }
386 
387 void MediaCurl::setCookieFile( const Pathname &fileName )
388 {
389  _cookieFile = fileName;
390 }
391 
392 void MediaCurl::setCurlError(const char* error)
393 {
394  // FIXME(dmllr): Use strlcpy if available for better performance
395  strncpy(_curlError, error, sizeof(_curlError)-1);
396  _curlError[sizeof(_curlError)-1] = '\0';
397 }
398 
400 
401 void MediaCurl::checkProtocol(const Url &url) const
402 {
403  curl_version_info_data *curl_info = NULL;
404  curl_info = curl_version_info(CURLVERSION_NOW);
405  // curl_info does not need any free (is static)
406  if (curl_info->protocols)
407  {
408  const char * const *proto = nullptr;
409  std::string scheme( url.getScheme());
410  bool found = false;
411  for(proto=curl_info->protocols; !found && *proto; ++proto)
412  {
413  if( scheme == std::string((const char *)*proto))
414  found = true;
415  }
416  if( !found)
417  {
418  std::string msg("Unsupported protocol '");
419  msg += scheme;
420  msg += "'";
422  }
423  }
424 }
425 
427 {
429 
430  curl_easy_setopt(_curl, CURLOPT_HEADERFUNCTION, log_redirects_curl);
431  curl_easy_setopt(_curl, CURLOPT_HEADERDATA, &_lastRedirect);
432  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_ERRORBUFFER, _curlError );
433  if ( ret != 0 ) {
434  ZYPP_THROW(MediaCurlSetOptException(_url, "Error setting error buffer"));
435  }
436 
437  SET_OPTION(CURLOPT_FAILONERROR, 1L);
438  SET_OPTION(CURLOPT_NOSIGNAL, 1L);
439 
440  // create non persistant settings
441  // so that we don't add headers twice
442  TransferSettings vol_settings(_settings);
443 
444  // add custom headers for download.opensuse.org (bsc#955801)
445  if ( _url.getHost() == "download.opensuse.org" )
446  {
447  vol_settings.addHeader(anonymousIdHeader());
448  vol_settings.addHeader(distributionFlavorHeader());
449  }
450  vol_settings.addHeader("Pragma:");
451 
453 
454  // fill some settings from url query parameters
455  try
456  {
458  }
459  catch ( const MediaException &e )
460  {
461  disconnectFrom();
462  ZYPP_RETHROW(e);
463  }
464  // if the proxy was not set (or explicitly unset) by url, then look...
465  if ( _settings.proxy().empty() )
466  {
467  // ...at the system proxy settings
469  }
470 
472  switch ( env::ZYPP_MEDIA_CURL_IPRESOLVE() )
473  {
474  case 4: SET_OPTION(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); break;
475  case 6: SET_OPTION(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6); break;
476  }
477 
481  SET_OPTION(CURLOPT_CONNECTTIMEOUT, _settings.connectTimeout());
482  // If a transfer timeout is set, also set CURLOPT_TIMEOUT to an upper limit
483  // just in case curl does not trigger its progress callback frequently
484  // enough.
485  if ( _settings.timeout() )
486  {
487  SET_OPTION(CURLOPT_TIMEOUT, 3600L);
488  }
489 
490  // follow any Location: header that the server sends as part of
491  // an HTTP header (#113275)
492  SET_OPTION(CURLOPT_FOLLOWLOCATION, 1L);
493  // 3 redirects seem to be too few in some cases (bnc #465532)
494  SET_OPTION(CURLOPT_MAXREDIRS, 6L);
495 
496  if ( _url.getScheme() == "https" )
497  {
498  if ( :: internal::setCurlRedirProtocols ( _curl ) != CURLE_OK ) {
500  }
501 
504  {
506  }
507 
509  {
510  SET_OPTION(CURLOPT_SSLCERT, _settings.clientCertificatePath().c_str());
511  }
512  if( ! _settings.clientKeyPath().empty() )
513  {
514  SET_OPTION(CURLOPT_SSLKEY, _settings.clientKeyPath().c_str());
515  }
516 
517 #ifdef CURLSSLOPT_ALLOW_BEAST
518  // see bnc#779177
519  ret = curl_easy_setopt( _curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_ALLOW_BEAST );
520  if ( ret != 0 ) {
521  disconnectFrom();
523  }
524 #endif
525  SET_OPTION(CURLOPT_SSL_VERIFYPEER, _settings.verifyPeerEnabled() ? 1L : 0L);
526  SET_OPTION(CURLOPT_SSL_VERIFYHOST, _settings.verifyHostEnabled() ? 2L : 0L);
527  // bnc#903405 - POODLE: libzypp should only talk TLS
528  SET_OPTION(CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
529  }
530 
531  SET_OPTION(CURLOPT_USERAGENT, _settings.userAgentString().c_str() );
532 
533  /* Fixes bsc#1174011 "auth=basic ignored in some cases"
534  * We should proactively add the password to the request if basic auth is configured
535  * and a password is available in the credentials but not in the URL.
536  *
537  * We will be a bit paranoid here and require that the URL has a user embedded, otherwise we go the default route
538  * and ask the server first about the auth method
539  */
540  if ( _settings.authType() == "basic"
541  && _settings.username().size()
542  && !_settings.password().size() ) {
543 
544  CredentialManager cm(CredManagerOptions(ZConfig::instance().repoManagerRoot()));
545  const auto cred = cm.getCred( _url );
546  if ( cred && cred->valid() ) {
547  if ( !_settings.username().size() )
548  _settings.setUsername(cred->username());
549  _settings.setPassword(cred->password());
550  }
551  }
552 
553  /*---------------------------------------------------------------*
554  CURLOPT_USERPWD: [user name]:[password]
555 
556  Url::username/password -> CURLOPT_USERPWD
557  If not provided, anonymous FTP identification
558  *---------------------------------------------------------------*/
559 
560  if ( _settings.userPassword().size() )
561  {
562  SET_OPTION(CURLOPT_USERPWD, _settings.userPassword().c_str());
563  std::string use_auth = _settings.authType();
564  if (use_auth.empty())
565  use_auth = "digest,basic"; // our default
566  long auth = CurlAuthData::auth_type_str2long(use_auth);
567  if( auth != CURLAUTH_NONE)
568  {
569  DBG << "Enabling HTTP authentication methods: " << use_auth
570  << " (CURLOPT_HTTPAUTH=" << auth << ")" << std::endl;
571  SET_OPTION(CURLOPT_HTTPAUTH, auth);
572  }
573  }
574 
575  if ( _settings.proxyEnabled() && ! _settings.proxy().empty() )
576  {
577  DBG << "Proxy: '" << _settings.proxy() << "'" << endl;
578  SET_OPTION(CURLOPT_PROXY, _settings.proxy().c_str());
579  SET_OPTION(CURLOPT_PROXYAUTH, CURLAUTH_BASIC|CURLAUTH_DIGEST|CURLAUTH_NTLM );
580  /*---------------------------------------------------------------*
581  * CURLOPT_PROXYUSERPWD: [user name]:[password]
582  *
583  * Url::option(proxyuser and proxypassword) -> CURLOPT_PROXYUSERPWD
584  * If not provided, $HOME/.curlrc is evaluated
585  *---------------------------------------------------------------*/
586 
587  std::string proxyuserpwd = _settings.proxyUserPassword();
588 
589  if ( proxyuserpwd.empty() )
590  {
591  CurlConfig curlconf;
592  CurlConfig::parseConfig(curlconf); // parse ~/.curlrc
593  if ( curlconf.proxyuserpwd.empty() )
594  DBG << "Proxy: ~/.curlrc does not contain the proxy-user option" << endl;
595  else
596  {
597  proxyuserpwd = curlconf.proxyuserpwd;
598  DBG << "Proxy: using proxy-user from ~/.curlrc" << endl;
599  }
600  }
601  else
602  {
603  DBG << "Proxy: using provided proxy-user '" << _settings.proxyUsername() << "'" << endl;
604  }
605 
606  if ( ! proxyuserpwd.empty() )
607  {
608  SET_OPTION(CURLOPT_PROXYUSERPWD, curlUnEscape( proxyuserpwd ).c_str());
609  }
610  }
611 #if CURLVERSION_AT_LEAST(7,19,4)
612  else if ( _settings.proxy() == EXPLICITLY_NO_PROXY )
613  {
614  // Explicitly disabled in URL (see fillSettingsFromUrl()).
615  // This should also prevent libcurl from looking into the environment.
616  DBG << "Proxy: explicitly NOPROXY" << endl;
617  SET_OPTION(CURLOPT_NOPROXY, "*");
618  }
619 #endif
620  else
621  {
622  DBG << "Proxy: not explicitly set" << endl;
623  DBG << "Proxy: libcurl may look into the environment" << endl;
624  }
625 
627  if ( _settings.minDownloadSpeed() != 0 )
628  {
629  SET_OPTION(CURLOPT_LOW_SPEED_LIMIT, _settings.minDownloadSpeed());
630  // default to 10 seconds at low speed
631  SET_OPTION(CURLOPT_LOW_SPEED_TIME, 60L);
632  }
633 
634 #if CURLVERSION_AT_LEAST(7,15,5)
635  if ( _settings.maxDownloadSpeed() != 0 )
636  SET_OPTION_OFFT(CURLOPT_MAX_RECV_SPEED_LARGE, _settings.maxDownloadSpeed());
637 #endif
638 
639  /*---------------------------------------------------------------*
640  *---------------------------------------------------------------*/
641 
643  if ( ::geteuid() == 0 || PathInfo(_currentCookieFile).owner() == ::geteuid() )
645 
646  const auto &cookieFileParam = _url.getQueryParam( "cookies" );
647  if ( !cookieFileParam.empty() && str::strToBool( cookieFileParam, true ) )
648  SET_OPTION(CURLOPT_COOKIEFILE, _currentCookieFile.c_str() );
649  else
650  MIL << "No cookies requested" << endl;
651  SET_OPTION(CURLOPT_COOKIEJAR, _currentCookieFile.c_str() );
652  SET_OPTION(CURLOPT_XFERINFOFUNCTION, &progressCallback );
653  SET_OPTION(CURLOPT_NOPROGRESS, 0L);
654 
655 #if CURLVERSION_AT_LEAST(7,18,0)
656  // bnc #306272
657  SET_OPTION(CURLOPT_PROXY_TRANSFER_MODE, 1L );
658 #endif
659  // Append settings custom headers to curl.
660  // TransferSettings assert strings are trimmed (HTTP/2 RFC 9113)
661  for ( const auto &header : vol_settings.headers() ) {
662  _customHeaders = curl_slist_append(_customHeaders, header.c_str());
663  if ( !_customHeaders )
665  }
666  SET_OPTION(CURLOPT_HTTPHEADER, _customHeaders);
667 }
668 
670 
671 
672 void MediaCurl::attachTo (bool next)
673 {
674  if ( next )
676 
677  if ( !_url.isValid() )
679 
682  {
684  }
685 
686  disconnectFrom(); // clean _curl if needed
687  _curl = curl_easy_init();
688  if ( !_curl ) {
690  }
691  try
692  {
693  setupEasy();
694  }
695  catch (Exception & ex)
696  {
697  disconnectFrom();
698  ZYPP_RETHROW(ex);
699  }
700 
701  // FIXME: need a derived class to propelly compare url's
703  setMediaSource(media);
704 }
705 
706 bool
708 {
709  return MediaHandler::checkAttachPoint( apoint, true, true);
710 }
711 
713 
715 {
716  if ( _customHeaders )
717  {
718  curl_slist_free_all(_customHeaders);
719  _customHeaders = 0L;
720  }
721 
722  if ( _curl )
723  {
724  // bsc#1201092: Strange but within global_dtors we may exceptions here.
725  try { curl_easy_cleanup( _curl ); }
726  catch (...) { ; }
727  _curl = NULL;
728  }
729 }
730 
732 
733 void MediaCurl::releaseFrom( const std::string & ejectDev )
734 {
735  disconnect();
736 }
737 
739 
740 void MediaCurl::getFile( const OnMediaLocation &file ) const
741 {
742  // Use absolute file name to prevent access of files outside of the
743  // hierarchy below the attach point.
744  getFileCopy( file, localPath(file.filename()).absolutename() );
745 }
746 
748 
749 void MediaCurl::getFileCopy( const OnMediaLocation & srcFile , const Pathname & target ) const
750 {
751 
752  const auto &filename = srcFile.filename();
753 
754  // Optional files will send no report until data are actually received (we know it exists).
755  OptionalDownloadProgressReport reportfilter( srcFile.optional() );
757 
758  Url fileurl(getFileUrl(filename));
759 
760  bool firstAuth = true; // bsc#1210870: authenticate must not return stored credentials more than once.
761  unsigned internalTry = 0;
762  static constexpr unsigned maxInternalTry = 3;
763 
764  do
765  {
766  try
767  {
768  doGetFileCopy( srcFile, target, report );
769  break; // success!
770  }
771  // retry with proper authentication data
772  catch (MediaUnauthorizedException & ex_r)
773  {
774  if ( authenticate(ex_r.hint(), firstAuth) ) {
775  firstAuth = false; // must not return stored credentials again
776  continue; // retry
777  }
778 
779  report->finish(fileurl, zypp::media::DownloadProgressReport::ACCESS_DENIED, ex_r.asUserHistory());
780  ZYPP_RETHROW(ex_r);
781  }
782  // unexpected exception
783  catch (MediaException & excpt_r)
784  {
785  if ( typeid(excpt_r) == typeid( MediaCurlExceptionMayRetryInternaly ) ) {
786  ++internalTry;
787  if ( internalTry < maxInternalTry ) {
788  // just report (NO_ERROR); no interactive request to the user
789  report->problem(fileurl, media::DownloadProgressReport::NO_ERROR, excpt_r.asUserHistory()+_("Will try again..."));
790  continue; // retry
791  }
792  excpt_r.addHistory( str::Format(_("Giving up after %1% attempts.")) % maxInternalTry );
793  }
794 
796  if( typeid(excpt_r) == typeid( media::MediaFileNotFoundException ) ||
797  typeid(excpt_r) == typeid( media::MediaNotAFileException ) )
798  {
800  }
801  report->finish(fileurl, reason, excpt_r.asUserHistory());
802  ZYPP_RETHROW(excpt_r);
803  }
804  }
805  while ( true );
806  report->finish(fileurl, zypp::media::DownloadProgressReport::NO_ERROR, "");
807 }
808 
810 
811 bool MediaCurl::getDoesFileExist( const Pathname & filename ) const
812 {
813  bool retry = false;
814 
815  do
816  {
817  try
818  {
819  return doGetDoesFileExist( filename );
820  }
821  // authentication problem, retry with proper authentication data
822  catch (MediaUnauthorizedException & ex_r)
823  {
824  if(authenticate(ex_r.hint(), !retry))
825  retry = true;
826  else
827  ZYPP_RETHROW(ex_r);
828  }
829  // unexpected exception
830  catch (MediaException & excpt_r)
831  {
832  ZYPP_RETHROW(excpt_r);
833  }
834  }
835  while (retry);
836 
837  return false;
838 }
839 
841 
843  CURLcode code,
844  bool timeout_reached) const
845 {
846  if ( code != 0 )
847  {
848  Url url;
849  if (filename.empty())
850  url = _url;
851  else
852  url = getFileUrl(filename);
853 
854  std::string err;
855  {
856  switch ( code )
857  {
858  case CURLE_UNSUPPORTED_PROTOCOL:
859  err = " Unsupported protocol";
860  if ( !_lastRedirect.empty() )
861  {
862  err += " or redirect (";
863  err += _lastRedirect;
864  err += ")";
865  }
866  break;
867  case CURLE_URL_MALFORMAT:
868  case CURLE_URL_MALFORMAT_USER:
869  err = " Bad URL";
870  break;
871  case CURLE_LOGIN_DENIED:
872  ZYPP_THROW(
873  MediaUnauthorizedException(url, "Login failed.", _curlError, ""));
874  break;
875  case CURLE_HTTP_RETURNED_ERROR:
876  {
877  long httpReturnCode = 0;
878  CURLcode infoRet = curl_easy_getinfo( _curl,
879  CURLINFO_RESPONSE_CODE,
880  &httpReturnCode );
881  if ( infoRet == CURLE_OK )
882  {
883  std::string msg = "HTTP response: " + str::numstring( httpReturnCode );
884  switch ( httpReturnCode )
885  {
886  case 401:
887  {
888  std::string auth_hint = getAuthHint();
889 
890  DBG << msg << " Login failed (URL: " << url.asString() << ")" << std::endl;
891  DBG << "MediaUnauthorizedException auth hint: '" << auth_hint << "'" << std::endl;
892 
894  url, "Login failed.", _curlError, auth_hint
895  ));
896  }
897 
898  case 502: // bad gateway (bnc #1070851)
899  case 503: // service temporarily unavailable (bnc #462545)
901  case 504: // gateway timeout
903  case 403:
904  {
905  std::string msg403;
906  if ( url.getHost().find(".suse.com") != std::string::npos )
907  msg403 = _("Visit the SUSE Customer Center to check whether your registration is valid and has not expired.");
908  else if (url.asString().find("novell.com") != std::string::npos)
909  msg403 = _("Visit the Novell Customer Center to check whether your registration is valid and has not expired.");
911  }
912  case 404:
913  case 410:
915  }
916 
917  DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
919  }
920  else
921  {
922  std::string msg = "Unable to retrieve HTTP response:";
923  DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
925  }
926  }
927  break;
928  case CURLE_FTP_COULDNT_RETR_FILE:
929 #if CURLVERSION_AT_LEAST(7,16,0)
930  case CURLE_REMOTE_FILE_NOT_FOUND:
931 #endif
932  case CURLE_FTP_ACCESS_DENIED:
933  case CURLE_TFTP_NOTFOUND:
934  err = "File not found";
936  break;
937  case CURLE_BAD_PASSWORD_ENTERED:
938  case CURLE_FTP_USER_PASSWORD_INCORRECT:
939  err = "Login failed";
940  break;
941  case CURLE_COULDNT_RESOLVE_PROXY:
942  case CURLE_COULDNT_RESOLVE_HOST:
943  case CURLE_COULDNT_CONNECT:
944  case CURLE_FTP_CANT_GET_HOST:
945  err = "Connection failed";
946  break;
947  case CURLE_WRITE_ERROR:
948  err = "Write error";
949  break;
950  case CURLE_PARTIAL_FILE:
951  case CURLE_OPERATION_TIMEDOUT:
952  timeout_reached = true; // fall though to TimeoutException
953  // fall though...
954  case CURLE_ABORTED_BY_CALLBACK:
955  if( timeout_reached )
956  {
957  err = "Timeout reached";
959  }
960  else
961  {
962  err = "User abort";
963  }
964  break;
965 
966  default:
967  err = "Curl error " + str::numstring( code );
968  break;
969  }
970 
971  // uhm, no 0 code but unknown curl exception
973  }
974  }
975  else
976  {
977  // actually the code is 0, nothing happened
978  }
979 }
980 
982 
983 bool MediaCurl::doGetDoesFileExist( const Pathname & filename ) const
984 {
985  DBG << filename.asString() << endl;
986 
987  if(!_url.isValid())
989 
990  if(_url.getHost().empty())
992 
993  Url url(getFileUrl(filename));
994 
995  DBG << "URL: " << url.asString() << endl;
996  // Use URL without options and without username and passwd
997  // (some proxies dislike them in the URL).
998  // Curl seems to need the just scheme, hostname and a path;
999  // the rest was already passed as curl options (in attachTo).
1000  Url curlUrl( clearQueryString(url) );
1001 
1002  //
1003  // See also Bug #154197 and ftp url definition in RFC 1738:
1004  // The url "ftp://user@host/foo/bar/file" contains a path,
1005  // that is relative to the user's home.
1006  // The url "ftp://user@host//foo/bar/file" (or also with
1007  // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
1008  // contains an absolute path.
1009  //
1010  _lastRedirect.clear();
1011  std::string urlBuffer( curlUrl.asString());
1012  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL,
1013  urlBuffer.c_str() );
1014  if ( ret != 0 ) {
1016  }
1017 
1018  // If no head requests allowed (?head_requests=no):
1019  // Instead of returning no data with NOBODY, we return
1020  // little data, that works with broken servers, and
1021  // works for ftp as well, because retrieving only headers
1022  // ftp will return always OK code ?
1023  // See http://curl.haxx.se/docs/knownbugs.html #58
1025  struct TempSetHeadRequest
1026  {
1027  TempSetHeadRequest(CURL *curl_r, bool doHttpHeadRequest_r)
1028  : _curl{curl_r}, _doHttpHeadRequest{doHttpHeadRequest_r} {
1029  if ( _doHttpHeadRequest ) {
1030  curl_easy_setopt( _curl, CURLOPT_NOBODY, 1L );
1031  } else {
1032  curl_easy_setopt( _curl, CURLOPT_RANGE, "0-1" );
1033  }
1034  }
1035  TempSetHeadRequest(const TempSetHeadRequest &) = delete;
1036  TempSetHeadRequest(TempSetHeadRequest &&) = delete;
1037  TempSetHeadRequest &operator=(const TempSetHeadRequest &) = delete;
1038  TempSetHeadRequest &operator=(TempSetHeadRequest &&) = delete;
1039  ~TempSetHeadRequest() {
1040  if ( _doHttpHeadRequest ) {
1041  curl_easy_setopt( _curl, CURLOPT_NOBODY, 0L);
1042  /* yes, this is why we never got to get NOBODY working before,
1043  because setting it changes this option too, and we also*
1044  need to reset it
1045  See: http://curl.haxx.se/mail/archive-2005-07/0073.html
1046  */
1047  curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1L );
1048  } else {
1049  curl_easy_setopt( _curl, CURLOPT_RANGE, NULL );
1050  }
1051  }
1052  private:
1053  CURL * _curl;
1054  bool _doHttpHeadRequest;
1055  } _guard( _curl, (_url.getScheme() == "http" || _url.getScheme() == "https") && _settings.headRequestsAllowed() );
1056 
1057 
1058  AutoFILE file { ::fopen( "/dev/null", "w" ) };
1059  if ( !file ) {
1060  ERR << "fopen failed for /dev/null" << endl;
1061  ZYPP_THROW(MediaWriteException("/dev/null"));
1062  }
1063 
1064  ret = curl_easy_setopt( _curl, CURLOPT_WRITEDATA, (*file) );
1065  if ( ret != 0 ) {
1067  }
1068 
1069  CURLcode ok = executeCurl();
1070  MIL << "perform code: " << ok << " [ " << curl_easy_strerror(ok) << " ]" << endl;
1071 
1072  // as we are not having user interaction, the user can't cancel
1073  // the file existence checking, a callback or timeout return code
1074  // will be always a timeout.
1075  try {
1076  evaluateCurlCode( filename, ok, true /* timeout */);
1077  }
1078  catch ( const MediaFileNotFoundException &e ) {
1079  // if the file did not exist then we can return false
1080  return false;
1081  }
1082  catch ( const MediaException &e ) {
1083  // some error, we are not sure about file existence, rethrw
1084  ZYPP_RETHROW(e);
1085  }
1086  // exists
1087  return ( ok == CURLE_OK );
1088 }
1089 
1091 
1092 void MediaCurl::doGetFileCopy( const OnMediaLocation &srcFile , const Pathname & target, callback::SendReport<DownloadProgressReport> & report, RequestOptions options ) const
1093 {
1094  Pathname dest = target.absolutename();
1095  if( assert_dir( dest.dirname() ) )
1096  {
1097  DBG << "assert_dir " << dest.dirname() << " failed" << endl;
1098  ZYPP_THROW( MediaSystemException(getFileUrl(srcFile.filename()), "System error on " + dest.dirname().asString()) );
1099  }
1100 
1101  ManagedFile destNew { target.extend( ".new.zypp.XXXXXX" ) };
1102  AutoFILE file;
1103  {
1104  AutoFREE<char> buf { ::strdup( (*destNew).c_str() ) };
1105  if( ! buf )
1106  {
1107  ERR << "out of memory for temp file name" << endl;
1108  ZYPP_THROW(MediaSystemException(getFileUrl(srcFile.filename()), "out of memory for temp file name"));
1109  }
1110 
1111  AutoFD tmp_fd { ::mkostemp( buf, O_CLOEXEC ) };
1112  if( tmp_fd == -1 )
1113  {
1114  ERR << "mkstemp failed for file '" << destNew << "'" << endl;
1115  ZYPP_THROW(MediaWriteException(destNew));
1116  }
1117  destNew = ManagedFile( (*buf), filesystem::unlink );
1118 
1119  file = ::fdopen( tmp_fd, "we" );
1120  if ( ! file )
1121  {
1122  ERR << "fopen failed for file '" << destNew << "'" << endl;
1123  ZYPP_THROW(MediaWriteException(destNew));
1124  }
1125  tmp_fd.resetDispose(); // don't close it here! ::fdopen moved ownership to file
1126  }
1127 
1128  DBG << "dest: " << dest << endl;
1129  DBG << "temp: " << destNew << endl;
1130 
1131  // set IFMODSINCE time condition (no download if not modified)
1132  if( PathInfo(target).isExist() && !(options & OPTION_NO_IFMODSINCE) )
1133  {
1134  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
1135  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, (long)PathInfo(target).mtime());
1136  }
1137  else
1138  {
1139  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1140  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1141  }
1142  try
1143  {
1144  doGetFileCopyFile( srcFile, dest, file, report, options);
1145  }
1146  catch (Exception &e)
1147  {
1148  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1149  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1150  ZYPP_RETHROW(e);
1151  }
1152 
1153  long httpReturnCode = 0;
1154  CURLcode infoRet = curl_easy_getinfo(_curl,
1155  CURLINFO_RESPONSE_CODE,
1156  &httpReturnCode);
1157  bool modified = true;
1158  if (infoRet == CURLE_OK)
1159  {
1160  DBG << "HTTP response: " + str::numstring(httpReturnCode);
1161  if ( httpReturnCode == 304
1162  || ( httpReturnCode == 213 && (_url.getScheme() == "ftp" || _url.getScheme() == "tftp") ) ) // not modified
1163  {
1164  DBG << " Not modified.";
1165  modified = false;
1166  }
1167  DBG << endl;
1168  }
1169  else
1170  {
1171  WAR << "Could not get the response code." << endl;
1172  }
1173 
1174  if (modified || infoRet != CURLE_OK)
1175  {
1176  // apply umask
1177  if ( ::fchmod( ::fileno(file), filesystem::applyUmaskTo( 0644 ) ) )
1178  {
1179  ERR << "Failed to chmod file " << destNew << endl;
1180  }
1181 
1182  file.resetDispose(); // we're going to close it manually here
1183  if ( ::fclose( file ) )
1184  {
1185  ERR << "Fclose failed for file '" << destNew << "'" << endl;
1186  ZYPP_THROW(MediaWriteException(destNew));
1187  }
1188 
1189  // move the temp file into dest
1190  if ( rename( destNew, dest ) != 0 ) {
1191  ERR << "Rename failed" << endl;
1193  }
1194  destNew.resetDispose(); // no more need to unlink it
1195  }
1196 
1197  DBG << "done: " << PathInfo(dest) << endl;
1198 }
1199 
1201 
1202 void MediaCurl::doGetFileCopyFile( const OnMediaLocation & srcFile, const Pathname & dest, FILE *file, callback::SendReport<DownloadProgressReport> & report, RequestOptions options ) const
1203 {
1204  DBG << srcFile.filename().asString() << endl;
1205 
1206  if(!_url.isValid())
1208 
1209  if(_url.getHost().empty())
1211 
1212  Url url(getFileUrl(srcFile.filename()));
1213 
1214  DBG << "URL: " << url.asString() << endl;
1215  // Use URL without options and without username and passwd
1216  // (some proxies dislike them in the URL).
1217  // Curl seems to need the just scheme, hostname and a path;
1218  // the rest was already passed as curl options (in attachTo).
1219  Url curlUrl( clearQueryString(url) );
1220 
1221  //
1222  // See also Bug #154197 and ftp url definition in RFC 1738:
1223  // The url "ftp://user@host/foo/bar/file" contains a path,
1224  // that is relative to the user's home.
1225  // The url "ftp://user@host//foo/bar/file" (or also with
1226  // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
1227  // contains an absolute path.
1228  //
1229  _lastRedirect.clear();
1230  std::string urlBuffer( curlUrl.asString());
1231  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL,
1232  urlBuffer.c_str() );
1233  if ( ret != 0 ) {
1235  }
1236 
1237  ret = curl_easy_setopt( _curl, CURLOPT_WRITEDATA, file );
1238  if ( ret != 0 ) {
1240  }
1241 
1242  // Set callback and perform.
1243  internal::ProgressData progressData(_curl, _settings.timeout(), url, srcFile.downloadSize(), &report);
1244  if (!(options & OPTION_NO_REPORT_START))
1245  report->start(url, dest);
1246  if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, &progressData ) != 0 ) {
1247  WAR << "Can't set CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1248  }
1249 
1250  ret = executeCurl();
1251 #if CURLVERSION_AT_LEAST(7,19,4)
1252  // bnc#692260: If the client sends a request with an If-Modified-Since header
1253  // with a future date for the server, the server may respond 200 sending a
1254  // zero size file.
1255  // curl-7.19.4 introduces CURLINFO_CONDITION_UNMET to check this condition.
1256  if ( ftell(file) == 0 && ret == 0 )
1257  {
1258  long httpReturnCode = 33;
1259  if ( curl_easy_getinfo( _curl, CURLINFO_RESPONSE_CODE, &httpReturnCode ) == CURLE_OK && httpReturnCode == 200 )
1260  {
1261  long conditionUnmet = 33;
1262  if ( curl_easy_getinfo( _curl, CURLINFO_CONDITION_UNMET, &conditionUnmet ) == CURLE_OK && conditionUnmet )
1263  {
1264  WAR << "TIMECONDITION unmet - retry without." << endl;
1265  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1266  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1267  ret = executeCurl();
1268  }
1269  }
1270  }
1271 #endif
1272 
1273  if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, NULL ) != 0 ) {
1274  WAR << "Can't unset CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1275  }
1276 
1277  if ( ret != 0 )
1278  {
1279  ERR << "curl error: " << ret << ": " << _curlError
1280  << ", temp file size " << ftell(file)
1281  << " bytes." << endl;
1282 
1283  // the timeout is determined by the progress data object
1284  // which holds whether the timeout was reached or not,
1285  // otherwise it would be a user cancel
1286  try {
1287 
1288  if ( progressData.fileSizeExceeded() )
1289  ZYPP_THROW(MediaFileSizeExceededException(url, progressData.expectedFileSize()));
1290 
1291  evaluateCurlCode( srcFile.filename(), ret, progressData.timeoutReached() );
1292  }
1293  catch ( const MediaException &e ) {
1294  // some error, we are not sure about file existence, rethrw
1295  ZYPP_RETHROW(e);
1296  }
1297  }
1298 }
1299 
1301 
1302 void MediaCurl::getDir( const Pathname & dirname, bool recurse_r ) const
1303 {
1304  filesystem::DirContent content;
1305  getDirInfo( content, dirname, /*dots*/false );
1306 
1307  for ( filesystem::DirContent::const_iterator it = content.begin(); it != content.end(); ++it ) {
1308  Pathname filename = dirname + it->name;
1309  int res = 0;
1310 
1311  switch ( it->type ) {
1312  case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
1313  case filesystem::FT_FILE:
1314  getFile( OnMediaLocation( filename ) );
1315  break;
1316  case filesystem::FT_DIR: // newer directory.yast contain at least directory info
1317  if ( recurse_r ) {
1318  getDir( filename, recurse_r );
1319  } else {
1320  res = assert_dir( localPath( filename ) );
1321  if ( res ) {
1322  WAR << "Ignore error (" << res << ") on creating local directory '" << localPath( filename ) << "'" << endl;
1323  }
1324  }
1325  break;
1326  default:
1327  // don't provide devices, sockets, etc.
1328  break;
1329  }
1330  }
1331 }
1332 
1334 
1335 void MediaCurl::getDirInfo( std::list<std::string> & retlist,
1336  const Pathname & dirname, bool dots ) const
1337 {
1338  getDirectoryYast( retlist, dirname, dots );
1339 }
1340 
1342 
1344  const Pathname & dirname, bool dots ) const
1345 {
1346  getDirectoryYast( retlist, dirname, dots );
1347 }
1348 
1350 //
1351 int MediaCurl::aliveCallback( void *clientp, curl_off_t /*dltotal*/, curl_off_t dlnow, curl_off_t /*ultotal*/, curl_off_t /*ulnow*/ )
1352 {
1353  internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>( clientp );
1354  if( pdata )
1355  {
1356  // Do not propagate dltotal in alive callbacks. MultiCurl uses this to
1357  // prevent a percentage raise while downloading a metalink file. Download
1358  // activity however is indicated by propagating the download rate (via dlnow).
1359  pdata->updateStats( 0.0, dlnow );
1360  return pdata->reportProgress();
1361  }
1362  return 0;
1363 }
1364 
1365 int MediaCurl::progressCallback( void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow )
1366 {
1367  internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>( clientp );
1368  if( pdata )
1369  {
1370  // work around curl bug that gives us old data
1371  long httpReturnCode = 0;
1372  if ( curl_easy_getinfo( pdata->curl(), CURLINFO_RESPONSE_CODE, &httpReturnCode ) != CURLE_OK || httpReturnCode == 0 )
1373  return aliveCallback( clientp, dltotal, dlnow, ultotal, ulnow );
1374 
1375  pdata->updateStats( dltotal, dlnow );
1376  return pdata->reportProgress();
1377  }
1378  return 0;
1379 }
1380 
1382 {
1383  internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>(clientp);
1384  return pdata ? pdata->curl() : 0;
1385 }
1386 
1388 
1389 std::string MediaCurl::getAuthHint() const
1390 {
1391  long auth_info = CURLAUTH_NONE;
1392 
1393  CURLcode infoRet =
1394  curl_easy_getinfo(_curl, CURLINFO_HTTPAUTH_AVAIL, &auth_info);
1395 
1396  if(infoRet == CURLE_OK)
1397  {
1398  return CurlAuthData::auth_type_long2str(auth_info);
1399  }
1400 
1401  return "";
1402 }
1403 
1408 void MediaCurl::resetExpectedFileSize(void *clientp, const ByteCount &expectedFileSize)
1409 {
1410  internal::ProgressData *data = reinterpret_cast<internal::ProgressData *>(clientp);
1411  if ( data ) {
1412  data->expectedFileSize( expectedFileSize );
1413  }
1414 }
1415 
1421 CURLcode MediaCurl::executeCurl() const
1422 {
1423  // initialize our helpers
1425  internal::CurlPollHelper::CurlPoll{ curl_multi_init() }
1426  ,[](auto &releaseMe ){ if (releaseMe._multi) curl_multi_cleanup(releaseMe._multi); }
1427  );
1428 
1429  if (!cMulti->_multi)
1431 
1432  // we could derive from that, but currently that does not make a lot of sense
1433  internal::CurlPollHelper _curlHelper(cMulti.value());
1434 
1435  // add the easy handle to the multi instance
1436  if ( curl_multi_add_handle( cMulti->_multi, _curl ) != CURLM_OK )
1437  ZYPP_THROW(MediaCurlException( _url, "curl_multi_add_handle", "unknown error"));
1438 
1439  // make sure the handle is cleanly removed from the multi handle
1440  OnScopeExit autoRemove([&](){ curl_multi_remove_handle( cMulti->_multi, _curl ); });
1441 
1442  // kickstart curl, this will cause libcurl to go over the added handles and register sockets and timeouts
1443  CURLMcode mcode = _curlHelper.handleTimout();
1444  if (mcode != CURLM_OK)
1445  ZYPP_THROW(MediaCurlException( _url, "curl_multi_socket_action", "unknown error"));
1446 
1447  bool canContinue = true;
1448  while ( canContinue ) {
1449 
1450  CURLMsg *msg = nullptr;
1451  int nqueue = 0;
1452  while ((msg = curl_multi_info_read( cMulti->_multi, &nqueue)) != 0) {
1453  if ( msg->msg != CURLMSG_DONE ) continue;
1454  if ( msg->easy_handle != _curl ) continue;
1455 
1456  return msg->data.result;
1457  }
1458 
1459  // copy watched sockets in case curl changes the vector as we go over the events later
1460  std::vector<GPollFD> requestedFds = _curlHelper.socks;
1461 
1462  int r = zypp_detail::zypp_poll( requestedFds, _curlHelper.timeout_ms.value_or( -1 ) );
1463  if ( r == -1 )
1464  ZYPP_THROW( MediaCurlException(_url, "zypp_poll() failed", "unknown error") );
1465 
1466  // run curl
1467  if ( r == 0 ) {
1468  CURLMcode mcode = _curlHelper.handleTimout();
1469  if (mcode != CURLM_OK)
1470  ZYPP_THROW(MediaCurlException(_url, "curl_multi_socket_action", "unknown error"));
1471  } else {
1472  CURLMcode mcode = _curlHelper.handleSocketActions( requestedFds );
1473  if (mcode != CURLM_OK)
1474  ZYPP_THROW(MediaCurlException(_url, "curl_multi_socket_action", "unknown error"));
1475  }
1476  }
1477  return CURLE_OK;
1478 }
1479 
1481 
1482 bool MediaCurl::authenticate(const std::string & availAuthTypes, bool firstTry) const
1483 {
1485  CredentialManager cm(CredManagerOptions(ZConfig::instance().repoManagerRoot()));
1486  CurlAuthData_Ptr credentials;
1487 
1488  // get stored credentials
1489  AuthData_Ptr cmcred = cm.getCred(_url);
1490 
1491  if (cmcred && firstTry)
1492  {
1493  credentials.reset(new CurlAuthData(*cmcred));
1494  DBG << "got stored credentials:" << endl << *credentials << endl;
1495  }
1496  // if not found, ask user
1497  else
1498  {
1499 
1500  CurlAuthData_Ptr curlcred;
1501  curlcred.reset(new CurlAuthData());
1503 
1504  // preset the username if present in current url
1505  if (!_url.getUsername().empty() && firstTry)
1506  curlcred->setUsername(_url.getUsername());
1507  // if CM has found some credentials, preset the username from there
1508  else if (cmcred)
1509  curlcred->setUsername(cmcred->username());
1510 
1511  // indicate we have no good credentials from CM
1512  cmcred.reset();
1513 
1514  std::string prompt_msg = str::Format(_("Authentication required for '%s'")) % _url.asString();
1515 
1516  // set available authentication types from the exception
1517  // might be needed in prompt
1518  curlcred->setAuthType(availAuthTypes);
1519 
1520  // ask user
1521  if (auth_report->prompt(_url, prompt_msg, *curlcred))
1522  {
1523  DBG << "callback answer: retry" << endl
1524  << "CurlAuthData: " << *curlcred << endl;
1525 
1526  if (curlcred->valid())
1527  {
1528  credentials = curlcred;
1529  // if (credentials->username() != _url.getUsername())
1530  // _url.setUsername(credentials->username());
1538  }
1539  }
1540  else
1541  {
1542  DBG << "callback answer: cancel" << endl;
1543  }
1544  }
1545 
1546  // set username and password
1547  if (credentials)
1548  {
1549  // HACK, why is this const?
1550  const_cast<MediaCurl*>(this)->_settings.setUsername(credentials->username());
1551  const_cast<MediaCurl*>(this)->_settings.setPassword(credentials->password());
1552 
1553  // set username and password
1554  CURLcode ret = curl_easy_setopt(_curl, CURLOPT_USERPWD, _settings.userPassword().c_str());
1556 
1557  // set available authentication types from the exception
1558  if (credentials->authType() == CURLAUTH_NONE)
1559  credentials->setAuthType(availAuthTypes);
1560 
1561  // set auth type (seems this must be set _after_ setting the userpwd)
1562  if (credentials->authType() != CURLAUTH_NONE)
1563  {
1564  // FIXME: only overwrite if not empty?
1565  const_cast<MediaCurl*>(this)->_settings.setAuthType(credentials->authTypeAsString());
1566  ret = curl_easy_setopt(_curl, CURLOPT_HTTPAUTH, credentials->authType());
1568  }
1569 
1570  if (!cmcred)
1571  {
1572  credentials->setUrl(_url);
1573  cm.addCred(*credentials);
1574  cm.save();
1575  }
1576 
1577  return true;
1578  }
1579 
1580  return false;
1581 }
1582 
1583 //need a out of line definiton, otherwise vtable is emitted for every translation unit
1585 
1586  } // namespace media
1587 } // namespace zypp
1588 //
std::string getScheme() const
Returns the scheme name of the URL.
Definition: Url.cc:551
long timeout() const
transfer timeout
void globalInitCurlOnce()
Definition: curlhelper.cc:64
Url getFileUrl(const Pathname &filename) const
concatenate the attach url and the filename to a complete download url
bool checkAttachPoint(const Pathname &apoint) const override
Verify if the specified directory as attach point (root) as requires by the particular media handler ...
Definition: MediaCurl.cc:707
const Pathname & certificateAuthoritiesPath() const
SSL certificate authorities path ( default: /etc/ssl/certs )
MediaCurlExceptionMayRetryInternaly(const Url &url_r, const std::string &err_r, const std::string &msg_r)
Definition: MediaCurl.cc:313
std::string targetDistribution() const
This is register.target attribute of the installed base product.
Definition: Target.cc:102
Interface to gettext.
#define SET_OPTION_OFFT(opt, val)
Definition: MediaCurl.cc:343
#define MIL
Definition: Logger.h:100
std::string curlUnEscape(const std::string &text_r)
Definition: curlhelper.cc:366
const Pathname & clientCertificatePath() const
SSL client certificate file.
bool progress(int value_r, const Url &file_r, double dbps_avg_r=-1, double dbps_current_r=-1) override
Download progress.
Definition: MediaCurl.cc:96
void addHeader(std::string &&val_r)
add a header, on the form "Foo: Bar" (trims)
int assert_dir(const Pathname &path, unsigned mode)
Like &#39;mkdir -p&#39;.
Definition: PathInfo.cc:324
#define _(MSG)
Definition: Gettext.h:39
size_t log_redirects_curl(char *ptr, size_t size, size_t nmemb, void *userdata)
Definition: curlhelper.cc:143
ProgressData()
Ctor no range [0,0](0).
Definition: progressdata.h:174
const Pathname & path() const
Return current Pathname.
Definition: PathInfo.h:251
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:424
Describes a resource file located on a medium.
static ZConfig & instance()
Singleton ctor.
Definition: ZConfig.cc:925
std::string proxyUserPassword() const
returns the proxy user and password as a user:pass string
void checkProtocol(const Url &url) const
check the url is supported by the curl library
Definition: MediaCurl.cc:401
bool authenticate(const std::string &availAuthTypes, bool firstTry) const
Definition: MediaCurl.cc:1482
Implementation class for FTP, HTTP and HTTPS MediaHandler.
Definition: MediaCurl.h:31
int assert_file_mode(const Pathname &path, unsigned mode)
Like assert_file but enforce mode even if the file already exists.
Definition: PathInfo.cc:1210
void setPassword(const std::string &val_r)
sets the auth password
std::chrono::steady_clock::time_point TimePoint
Definition: MediaCurl.cc:55
ByteCount _expectedFileSize
Definition: MediaCurl.cc:180
virtual bool checkAttachPoint(const Pathname &apoint) const
Verify if the specified directory as attach point (root) as requires by the particular media handler ...
const char * distributionFlavorHeader()
Definition: MediaCurl.cc:278
Store and operate with byte count.
Definition: ByteCount.h:31
long maxDownloadSpeed() const
Maximum download speed (bytes per second)
to not add a IFMODSINCE header if target exists
Definition: MediaCurl.h:43
Holds transfer setting.
void save()
Saves any unsaved credentials added via addUserCred() or addGlobalCred() methods. ...
const std::string & authType() const
get the allowed authentication types
bool verifyHostEnabled() const
Whether to verify host for ssl.
const std::string & proxyUsername() const
proxy auth username
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Definition: Pathname.h:175
int reportProgress() const
Definition: MediaCurl.cc:253
Url clearQueryString(const Url &url)
Definition: curlhelper.cc:373
void setAttachPoint(const Pathname &path, bool temp)
Set a new attach point.
void start(const Url &file_r, Pathname localfile_r) override
Definition: MediaCurl.cc:84
const char * anonymousIdHeader()
Definition: MediaCurl.cc:264
const char * c_str() const
String representation.
Definition: Pathname.h:112
void addHistory(const std::string &msg_r)
Add some message text to the history.
Definition: Exception.cc:176
bool isUseableAttachPoint(const Pathname &path, bool mtab=true) const
Ask media manager, if the specified path is already used as attach point or if there are another atta...
void setUsername(const std::string &val_r)
sets the auth username
Action problem(const Url &file_r, Error error_r, const std::string &description_r) override
Definition: MediaCurl.cc:115
void getFileCopy(const OnMediaLocation &srcFile, const Pathname &targetFilename) const override
Definition: MediaCurl.cc:749
MediaCurlException(const Url &url_r, std::string err_r, std::string msg_r)
time_t _timeNow
Now.
Definition: MediaCurl.cc:186
Definition: Arch.h:363
AuthData_Ptr getCred(const Url &url)
Get credentials for the specified url.
virtual void setupEasy()
initializes the curl easy handle with the data from the url
Definition: MediaCurl.cc:426
bool timeoutReached() const
Definition: MediaCurl.cc:162
Convenient building of std::string with boost::format.
Definition: String.h:252
Structure holding values of curlrc options.
Definition: curlconfig.h:26
std::string form(const char *format,...) __attribute__((format(printf
Printf style construction of std::string.
Definition: String.cc:37
CURLcode executeCurl() const
Definition: MediaCurl.cc:1421
Edition * _value
Definition: SysContent.cc:311
AutoDispose<int> calling ::close
Definition: AutoDispose.h:309
std::string _currentCookieFile
Definition: MediaCurl.h:163
const std::string & password() const
auth password
void getDir(const Pathname &dirname, bool recurse_r) const override
Call concrete handler to provide directory content (not recursive!) below attach point.
Definition: MediaCurl.cc:1302
void getFile(const OnMediaLocation &file) const override
Call concrete handler to provide file below attach point.
Definition: MediaCurl.cc:740
std::list< DirEntry > DirContent
Returned by readdir.
Definition: PathInfo.h:526
#define ERR
Definition: Logger.h:102
const std::string & username() const
auth username
bool fileSizeExceeded() const
Definition: MediaCurl.cc:165
Pathname localPath(const Pathname &pathname) const
Files provided will be available at &#39;localPath(filename)&#39;.
const Headers & headers() const
returns a list of all added headers (trimmed)
zypp::Url _url
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
Definition: ManagedFile.h:27
int ZYPP_MEDIA_CURL_IPRESOLVE()
4/6 to force IPv4/v6
Definition: curlhelper.cc:45
static void setCookieFile(const Pathname &)
Definition: MediaCurl.cc:387
void releaseFrom(const std::string &ejectDev) override
Call concrete handler to release the media.
Definition: MediaCurl.cc:733
bool verifyPeerEnabled() const
Whether to verify peer for ssl.
static void resetExpectedFileSize(void *clientp, const ByteCount &expectedFileSize)
MediaMultiCurl needs to reset the expected filesize in case a metalink file is downloaded otherwise t...
Definition: MediaCurl.cc:1408
const std::string & hint() const
comma separated list of available authentication types
bool empty() const
Test for an empty path.
Definition: Pathname.h:116
zypp::callback::UserData UserData
Definition: userrequest.h:18
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition: Exception.h:444
static int parseConfig(CurlConfig &config, const std::string &filename="")
Parse a curlrc file and store the result in the config structure.
Definition: curlconfig.cc:24
std::string asString() const
Returns a default string representation of the Url object.
Definition: Url.cc:515
std::string getQueryParam(const std::string &param, EEncoding eflag=zypp::url::E_DECODED) const
Return the value for the specified query parameter.
Definition: Url.cc:678
Bottleneck filtering all DownloadProgressReport issued from Media[Muli]Curl.
Definition: MediaCurl.cc:53
Url clearQueryString(const Url &url) const
Definition: MediaCurl.cc:382
void setAuthType(const std::string &val_r)
set the allowed authentication types
std::string trim(const std::string &s, const Trim trim_r)
Definition: String.cc:224
void finish(const Url &file_r, Error error_r, const std::string &reason_r) override
Definition: MediaCurl.cc:121
const Url _url
Url to handle.
Definition: MediaHandler.h:113
void setMediaSource(const MediaSourceRef &ref)
Set new media source reference.
const std::string & asString() const
String representation.
Definition: Pathname.h:93
Just inherits Exception to separate media exceptions.
void evaluateCurlCode(const zypp::Pathname &filename, CURLcode code, bool timeout) const
Evaluates a curl return code and throws the right MediaException filename Filename being downloaded c...
Definition: MediaCurl.cc:842
std::string asUserHistory() const
A single (multiline) string composed of asUserString and historyAsString.
Definition: Exception.cc:127
const ByteCount & downloadSize() const
The size of the resource on the server.
void disconnect()
Use concrete handler to isconnect media.
long connectTimeout() const
connection timeout
Pathname dirname() const
Return all but the last component od this path.
Definition: Pathname.h:126
do not send a start ProgressReport
Definition: MediaCurl.h:45
#define WAR
Definition: Logger.h:101
shared_ptr< AuthData > AuthData_Ptr
Definition: authdata.h:81
void getDirectoryYast(std::list< std::string > &retlist, const Pathname &dirname, bool dots=true) const
Retrieve and if available scan dirname/directory.yast.
void fillSettingsFromUrl(const Url &url, media::TransferSettings &s)
Fills the settings structure using options passed on the url for example ?timeout=x&proxy=foo.
Definition: curlhelper.cc:183
static int progressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
Callback reporting download progress.
Definition: MediaCurl.cc:1365
Receiver * _oldRec
std::string proxyuserpwd
Definition: curlconfig.h:49
bool isValid() const
Verifies the Url.
Definition: Url.cc:507
virtual bool doGetDoesFileExist(const Pathname &filename) const
Definition: MediaCurl.cc:983
const Pathname & clientKeyPath() const
SSL client key file.
void expectedFileSize(ByteCount newval_r)
Definition: MediaCurl.cc:171
const Pathname & filename() const
The path to the resource on the medium.
void attachTo(bool next=false) override
Call concrete handler to attach the media.
Definition: MediaCurl.cc:672
std::string numstring(char n, int w=0)
Definition: String.h:289
int unlink(const Pathname &path)
Like &#39;unlink&#39;.
Definition: PathInfo.cc:705
bool getDoesFileExist(const Pathname &filename) const override
Repeatedly calls doGetDoesFileExist() until it successfully returns, fails unexpectedly, or user cancels the operation.
Definition: MediaCurl.cc:811
const char * agentString()
Definition: MediaCurl.cc:292
void resetDispose()
Set no dispose function.
Definition: AutoDispose.h:171
void doGetFileCopyFile(const OnMediaLocation &srcFile, const Pathname &dest, FILE *file, callback::SendReport< DownloadProgressReport > &report, RequestOptions options=OPTION_NONE) const
Definition: MediaCurl.cc:1202
Media source internally used by MediaManager and MediaHandler.
Definition: MediaSource.h:37
static std::string auth_type_long2str(long auth_type)
Converts a long of ORed CURLAUTH_* identifiers into a string of comma separated list of authenticatio...
long minDownloadSpeed() const
Minimum download speed (bytes per second) until the connection is dropped.
int _dnlPercent
Percent completed or 0 if _dnlTotal is unknown.
Definition: MediaCurl.cc:192
curl_slist * _customHeaders
Definition: MediaCurl.h:171
bool proxyEnabled() const
proxy is enabled
constexpr std::string_view FILE("file")
#define SET_OPTION(opt, val)
Definition: MediaCurl.cc:336
Pathname absolutename() const
Return this path, adding a leading &#39;/&#39; if relative.
Definition: Pathname.h:141
Base class for Exception.
Definition: Exception.h:146
Pathname attachPoint() const
Return the currently used attach point.
std::string _lastRedirect
to log/report redirections
Definition: MediaCurl.h:167
curl_off_t _dnlNow
Bytes downloaded now.
Definition: MediaCurl.cc:190
Url url() const
Url used.
Definition: MediaHandler.h:503
shared_ptr< CurlAuthData > CurlAuthData_Ptr
Definition: curlauthdata.h:102
curl_off_t _dnlTotal
Bytes to download or 0 if unknown.
Definition: MediaCurl.cc:188
double _drateLast
Download rate in last period.
Definition: MediaCurl.cc:195
std::string getHost(EEncoding eflag=zypp::url::E_DECODED) const
Returns the hostname or IP from the URL authority.
Definition: Url.cc:606
time_t _timeStart
Start total stats.
Definition: MediaCurl.cc:183
void getDirInfo(std::list< std::string > &retlist, const Pathname &dirname, bool dots=true) const override
Call concrete handler to provide a content list of directory on media via retlist.
Definition: MediaCurl.cc:1335
void setupZYPP_MEDIA_CURL_DEBUG(CURL *curl)
Setup CURLOPT_VERBOSE and CURLOPT_DEBUGFUNCTION according to env::ZYPP_MEDIA_CURL_DEBUG.
Definition: curlhelper.cc:130
void disconnectFrom() override
Definition: MediaCurl.cc:714
static CURL * progressCallback_getcurl(void *clientp)
Definition: MediaCurl.cc:1381
Typesafe passing of user data via callbacks.
Definition: UserData.h:39
bool strToBool(const C_Str &str, bool default_r)
Parse str into a bool depending on the default value.
Definition: String.h:429
void setCurlError(const char *error)
Definition: MediaCurl.cc:392
static long auth_type_str2long(std::string &auth_type_str)
Converts a string of comma separated list of authetication type names into a long of ORed CURLAUTH_* ...
Definition: curlauthdata.cc:50
void setUserAgentString(std::string &&val_r)
sets the user agent ie: "Mozilla v3" (trims)
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:225
curl_off_t _dnlLast
Bytes downloaded at period start.
Definition: MediaCurl.cc:189
void report(const UserData &userData_r=UserData()) override
The most generic way of sending/receiving data.
Definition: MediaCurl.cc:80
AutoDispose<FILE*> calling ::fclose
Definition: AutoDispose.h:320
static Pathname _cookieFile
Definition: MediaCurl.h:164
void fillSettingsSystemProxy(const Url &url, media::TransferSettings &s)
Reads the system proxy configuration and fills the settings structure proxy information.
Definition: curlhelper.cc:331
double _drateTotal
Download rate so far.
Definition: MediaCurl.cc:194
std::string anonymousUniqueId() const
anonymous unique id
Definition: Target.cc:132
mode_t applyUmaskTo(mode_t mode_r)
Modify mode_r according to the current umask ( mode_r & ~getUmask() ).
Definition: PathInfo.h:806
std::string userPassword() const
returns the user and password as a user:pass string
std::string getAuthHint() const
Return a comma separated list of available authentication methods supported by server.
Definition: MediaCurl.cc:1389
time_t _timeRcv
Start of no-data timeout.
Definition: MediaCurl.cc:185
#define EXPLICITLY_NO_PROXY
Definition: curlhelper_p.h:23
int rename(const Pathname &oldpath, const Pathname &newpath)
Like &#39;rename&#39;.
Definition: PathInfo.cc:747
Pathname createAttachPoint() const
Try to create a default / temporary attach point.
int zypp_poll(std::vector< GPollFD > &fds, int timeout)
Small wrapper around g_poll that additionally listens to the shutdown FD returned by ZYpp::shutdownSi...
Definition: ZYppImpl.cc:313
virtual void doGetFileCopy(const OnMediaLocation &srcFile, const Pathname &targetFilename, callback::SendReport< DownloadProgressReport > &_report, RequestOptions options=OPTION_NONE) const
Definition: MediaCurl.cc:1092
void updateStats(curl_off_t dltotal=0.0, curl_off_t dlnow=0.0)
Definition: MediaCurl.cc:210
void addCred(const AuthData &cred)
Add new credentials with user callbacks.
Easy-to use interface to the ZYPP dependency resolver.
Definition: Application.cc:19
Curl HTTP authentication data.
Definition: curlauthdata.h:22
std::string distributionFlavor() const
This is flavor attribute of the installed base product but does not require the target to be loaded a...
Definition: Target.cc:127
OptionalDownloadProgressReport(bool isOptional=false)
Definition: MediaCurl.cc:57
char _curlError[CURL_ERROR_SIZE]
Definition: MediaCurl.h:165
ByteCount expectedFileSize() const
Definition: MediaCurl.cc:168
CURLcode setCurlRedirProtocols(CURL *curl)
Definition: curlhelper.cc:515
const std::string & proxy() const
proxy host
int rmdir(const Pathname &path)
Like &#39;rmdir&#39;.
Definition: PathInfo.cc:371
bool optional() const
Whether this is an optional resource.
time_t _timeLast
Start last period(~1sec)
Definition: MediaCurl.cc:184
const char * c_str() const
Definition: IdStringType.h:107
TrueBool _guard
Definition: TargetImpl.cc:1609
static int aliveCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
Callback sending just an alive trigger to the UI, without stats (e.g.
Definition: MediaCurl.cc:1351
bool userMayRWX() const
Definition: PathInfo.h:361
const std::string & userAgentString() const
user agent string (trimmed)
Url manipulation class.
Definition: Url.h:92
bool headRequestsAllowed() const
whether HEAD requests are allowed
#define DBG
Definition: Logger.h:99
Attempt to work around certain issues by autoretry in MediaCurl::getFileCopy E.g. ...
Definition: MediaCurl.cc:310
zypp::callback::SendReport< zypp::media::DownloadProgressReport > * report
Definition: MediaCurl.cc:181
std::string getUsername(EEncoding eflag=zypp::url::E_DECODED) const
Returns the username from the URL authority.
Definition: Url.cc:590