libzypp  17.36.7
commitpackagepreloader.cc
Go to the documentation of this file.
5 #include <zypp/media/MediaCurl2.h> // for shared logic like authenticate
6 #include <zypp/media/MediaHandlerFactory.h> // to detect the URL type
10 #include <zypp-core/fs/TmpPath.h>
14 #include <zypp/MediaSetAccess.h>
15 #include <zypp/Package.h>
16 #include <zypp/SrcPackage.h>
17 #include <zypp/ZConfig.h>
18 
19 namespace zypp {
20 
21  namespace {
22 
23  inline bool preloadEnabled()
24  {
25  const char *val = ::getenv("ZYPP_PCK_PRELOAD");
26 
27  // opt out for now
28  if ( !val )
29  return false;
30 
31  return ( std::string_view( val ) == "1" );
32  }
33 
34  zypp::Pathname pckCachedLocation ( const PoolItem &pck ) {
35  if ( pck.isKind<Package>() ) {
36  return pck->asKind<Package>()->cachedLocation();
37  } else if ( pck.isKind<SrcPackage>() ) {
38  return pck->asKind<SrcPackage>()->cachedLocation();
39  }
40  return {};
41  }
42 
43  }
44 
45  struct RepoUrl {
48  };
49 
51  public:
52  enum State {
55  //ZckHead,
56  //ZckData,
58  };
59 
61 
62  bool finished ( ) const {
63  return (_s == Finished);
64  }
65 
66  void nextJob () {
67 
68  // clean state vars
69  _started = false;
70  _firstAuth = true;
71  _notFoundRetry = 0;
72  _tmpFile.reset();
73  _lastByteCount = 0;
74  _taintedMirrors.clear();
75 
76  if ( _parent._requiredDls.empty() ) {
77 
78  if ( _myMirror ) {
79  _myMirror->refs--;
80  _myMirror = nullptr;
82  }
83 
84  MIL << "No more jobs pending, exiting worker" << std::endl;
85  // exit!
86  _s = Finished;
87  _sigFinished.emit();
88  return;
89  }
90 
91  _job = _parent._requiredDls.front();
92  _parent._requiredDls.pop_front();
93 
94  auto loc = _job.lookupLocation();
96 
97  // select a mirror we want to use
98  if ( !prepareMirror( ) ) {
99  callback::UserData userData( "CommitPreloadReport/fileDone" );
100  userData.set( "description", _("No mirror found") );
102  return nextJob();
103  }
104 
105  if ( filesystem::assert_dir( _targetPath.dirname()) != 0 ) {
106  ERR << "Failed to create target dir for file: " << _targetPath << std::endl;
107  callback::UserData userData( "CommitPreloadReport/fileDone" );
108  userData.set( "description", _("Could not create target File") );
110  return nextJob();
111  }
112 
113 
114  media::TransferSettings settings;
115  zypp::Url url;
116  makeJobUrl ( url, settings );
117 
118  // check if the file is there already
119  {
120  PathInfo pathInfo(_targetPath);
121  if ( pathInfo.isExist() ) {
122  // just in case there is something else that is not a file we delete it
123  if ( !pathInfo.isFile() ) {
124  if ( pathInfo.isDir () )
126  else
128 
129  } else if ( is_checksum( _targetPath, loc.checksum() ) ) {
130 
131  // if we have the file already, no need to download again
132  callback::UserData userData( "CommitPreloadReport/fileDone" );
133  userData.set( "Url", url );
134  userData.set( "description", _("Already in Cache") );
136 
137  return nextJob();
138 
139  } else {
140  // everything else we delete
142  }
143  }
144  }
145 
146  // we download into a temp file so that we don't leave broken files in case of errors or a crash
148 
149  if ( _s == Pending ) {
150  // init case, set up request
151  _req = std::make_shared<zyppng::NetworkRequest>( url, _tmpFile );
155  } else {
156  _req->resetRequestRanges();
157  _req->setUrl( url );
158  _req->setTargetFilePath( _tmpFile );
159  }
160 
161  // TODO check for zchunk
162 
163  _s = SimpleDl;
164  _req->transferSettings() = settings;
165  _parent._dispatcher->enqueue(_req);
166  }
167 
169  return _sigFinished;
170  }
171 
172  private:
173 
174  // TODO some smarter logic that selects mirrors
175  bool prepareMirror( ) {
176 
177  const auto &pi = _job;
178 
179  if ( _myMirror ) {
180  if ( _currentRepoId == pi.repository().id() ) {
181  return true;
182  }
184  _myMirror->refs--;
185  _myMirror = nullptr;
186  }
187 
189  if ( !_myMirror )
190  return false;
191 
192  _currentRepoId = pi.repository().id();
193  _myMirror->refs++;
194  return true;
195  }
196 
201 
202  if ( _myMirror ) {
203  _myMirror->miss++;
204  _taintedMirrors.insert( _myMirror );
205  }
206 
207  // try to find another mirror
208  auto mirrPtr = findUsableMirror ( _myMirror, false );
209  if ( mirrPtr ) {
210  if ( _myMirror ) {
211  _myMirror->refs--;
212  }
213  _myMirror = mirrPtr;
214  _myMirror->refs++;
215  return true;
216  }
217  return false;
218  }
219 
223  RepoUrl *findUsableMirror( RepoUrl *skip = nullptr, bool allowTainted = true ) {
224  auto &repoDlInfo = _parent._dlRepoInfo.at( _job.repository().id() );
225 
226  std::vector<RepoUrl>::iterator curr = repoDlInfo._baseUrls.end();
227  int currentSmallestRef = INT_MAX;
228 
229  for ( auto i = repoDlInfo._baseUrls.begin(); i != repoDlInfo._baseUrls.end(); i++ ) {
230  auto mirrorPtr = &(*i);
231 
232  if ( skip == mirrorPtr )
233  continue;
234 
235  if ( !allowTainted && _taintedMirrors.find(mirrorPtr) != _taintedMirrors.end() )
236  continue;
237 
238  // we are adding the file misses on top of the refcount
239  // that way we will use mirrors that often miss a file less
240  if ( ( i->refs + i->miss ) < currentSmallestRef ) {
241  currentSmallestRef = ( i->refs + i->miss );
242  curr = i;
243  }
244  }
245 
246  if ( curr == repoDlInfo._baseUrls.end() )
247  return nullptr;
248  return &(*curr);
249  }
250 
252  MIL << "Request for " << req.url() << " started" << std::endl;
253  }
254 
256  if ( !_started ) {
257  _started = true;
258 
259  callback::UserData userData( "CommitPreloadReport/fileStart" );
260  userData.set( "Url", _req->url() );
261  _parent._report->fileStart( _targetPath, userData );
262  }
263 
264  ByteCount downloaded;
265  if ( _lastByteCount == 0 )
266  downloaded = count;
267  else
268  downloaded = count - _lastByteCount;
269  _lastByteCount = count;
270 
271  _parent.reportBytesDownloaded( downloaded );
272  }
273 
275  MIL << "Request for " << req.url() << " finished. (" << err.toString() << ")" << std::endl;
276  if ( !req.hasError() ) {
277  if ( filesystem::rename( _tmpFile, _targetPath ) != 0 ) {
278  // error
279  failCurrentJob ( _targetPath, req.url(), media::CommitPreloadReport::ERROR, _("Failed to rename temporary file.") );
280  } else {
281  callback::UserData userData( "CommitPreloadReport/fileDone" );
282  userData.set( "Url", req.url() );
283  userData.set( "description", _("Finished") );
285  }
286  } else {
287  // handle errors and auth
288  const auto &error = req.error();
289  switch ( error.type() ) {
307  break;
308  }
311 
312  //in case we got a auth hint from the server the error object will contain it
313  std::string authHint = error.extraInfoValue("authHint", std::string());
314 
316  bool newCreds = media::MediaCurl2::authenticate( _myMirror->baseUrl, cm, req.transferSettings(), authHint, _firstAuth );
317  if ( newCreds) {
318  _firstAuth = false;
319  _parent._dispatcher->enqueue( _req );
320  return;
321  }
322 
324  break;
325  }
327 
328  MIL << "Download from mirror failed for file " << req.url () << " trying to taint mirror and move on" << std::endl;
329 
330  if ( taintCurrentMirror() ) {
331  _notFoundRetry++;
332 
333  media::TransferSettings settings;
334  zypp::Url url;
335  makeJobUrl ( url, settings );
336 
337  MIL << "Found new mirror: " << url << " recovering, retry count: " << _notFoundRetry << std::endl;
338 
339  _req->setUrl( url );
340  _req->transferSettings () = settings;
341 
342  _parent._dispatcher->enqueue( _req );
343  return;
344  }
345 
347  break;
348  }
350  // should never happen
351  DBG << "BUG: Download error flag is set , but Error code is NoError" << std::endl;
352  break;
353  }
354  }
355  nextJob();
356  }
357 
358  void failCurrentJob( const zypp::Pathname &localPath, const std::optional<zypp::Url> &url, media::CommitPreloadReport::Error e, const std::optional<std::string> &errorMessage ) {
359 
360  callback::UserData userData( "CommitPreloadReport/fileDone" );
361  if ( url )
362  userData.set( "Url", url );
363  if ( errorMessage )
364  userData.set( "description", _("Already in Cache") );
365 
366  _parent._missedDownloads = true;
367  _parent._report->fileDone( localPath, e, userData );
368  }
369 
370  void makeJobUrl ( zypp::Url &resultUrl, media::TransferSettings &resultSet ) {
371 
372  // rewrite Url
373  zypp::Url url = _myMirror->baseUrl;
374  media::TransferSettings settings;
375 
376  // TODO share logic with MediaCurl2
377  ::internal::fillSettingsFromUrl( url, settings );
378 
379  // if the proxy was not set (or explicitly unset) by url, then look...
380  if ( settings.proxy().empty() )
381  ::internal::fillSettingsSystemProxy( url, settings );
382 
383  // remove extra options from the URL
384  url = ::internal::clearQueryString( url );
385 
386  const auto &loc = _job.lookupLocation();
387 
388  // rewrite URL for media handle
389  url = MediaSetAccess::rewriteUrl( url ,loc.medianr() );
390 
391  // append path to file
392  url.appendPathName( loc.filename() );
393 
394  // add extra headers
395  for ( const auto & el : _myMirror->headers ) {
396  std::string header { el.first };
397  header += ": ";
398  header += el.second;
399  MIL << "Added custom header -> " << header << std::endl;
400  settings.addHeader( std::move(header) );
401  }
402 
403  resultUrl = url;
404  resultSet = settings;
405  }
406 
407  private:
410  zyppng::NetworkRequestRef _req;
411 
415  bool _started = false;
416  bool _firstAuth = true;
417  RepoUrl *_myMirror = nullptr;
420 
421  // retry handling
422  int _notFoundRetry = 0;
423  std::set<RepoUrl *> _taintedMirrors; //< mirrors that returned 404 for the current request
424 
426 
427  };
428 
430  {}
431 
432  void CommitPackagePreloader::preloadTransaction( const std::vector<sat::Transaction::Step> &steps)
433  {
434  if ( !preloadEnabled() ) {
435  MIL << "CommitPackagePreloader disabled" << std::endl;
436  return;
437  }
438 
439  // preload happens only if someone handles the report
440  if ( !_report->connected() ) {
441  MIL << "No receiver for the CommitPreloadReport, skipping preload phase" << std::endl;
442  return;
443  }
444 
445  auto ev = zyppng::EventLoop::create();
446  _dispatcher = std::make_shared<zyppng::NetworkRequestDispatcher>();
447  _dispatcher->setMaximumConcurrentConnections( MediaConfig::instance().download_max_concurrent_connections() );
448  _dispatcher->setAgentString ( str::asString( media::MediaCurl2::agentString () ) );
449  _dispatcher->setHostSpecificHeader ("download.opensuse.org", "X-ZYpp-DistributionFlavor", str::asString(media::MediaCurl2::distributionFlavorHeader()) );
450  _dispatcher->setHostSpecificHeader ("download.opensuse.org", "X-ZYpp-AnonymousId", str::asString(media::MediaCurl2::anonymousIdHeader()) );
451  _dispatcher->run();
452 
453  _pTracker = std::make_shared<internal::ProgressTracker>();
454  _requiredBytes = 0;
455  _downloadedBytes = 0;
456  _missedDownloads = false;
457 
458  zypp_defer {
459  _dispatcher.reset();
460  _pTracker.reset();
461  };
462 
463  for ( const auto &step : steps ) {
464  switch ( step.stepType() )
465  {
468  // proceed: only install actions may require download.
469  break;
470 
471  default:
472  // next: no download for non-packages and delete actions.
473  continue;
474  break;
475  }
476 
477  PoolItem pi(step.satSolvable());
478 
479  if ( !pi->isKind<Package>() && !pi->isKind<SrcPackage>() )
480  continue;
481 
482  // no checksum ,no predownload, Fetcher would ignore it
483  if ( pi->lookupLocation().checksum().empty() )
484  continue;
485 
486  // check if Package is cached already
487  if( !pckCachedLocation(pi).empty() )
488  continue;
489 
490  auto repoDlsIter = _dlRepoInfo.find( pi.repository().id() );
491  if ( repoDlsIter == _dlRepoInfo.end() ) {
492 
493  // make sure download path for this repo exists
494  if ( filesystem::assert_dir( pi.repoInfo().predownloadPath() ) != 0 ) {
495  ERR << "Failed to create predownload cache for repo " << pi.repoInfo().alias() << std::endl;
496  return;
497  }
498 
499  // filter base URLs that do not download
500  std::vector<RepoUrl> repoUrls;
501  const auto bu = pi.repoInfo().baseUrls();
502  std::for_each( bu.begin(), bu.end(), [&]( const zypp::Url &u ) {
504  Url url = media::UrlResolverPlugin::resolveUrl(u, custom_headers);
505  MIL << "Trying scheme '" << url.getScheme() << "'" << std::endl;
506 
508  return;
509 
510  repoUrls.push_back( RepoUrl {
511  .baseUrl = std::move(url),
512  .headers = std::move(custom_headers)
513  } );
514  });
515 
516  // skip this solvable if it has no downloading base URLs
517  if( repoUrls.empty() ) {
518  MIL << "Skipping predownload for " << step.satSolvable() << " no downloading URL" << std::endl;
519  continue;
520  }
521 
522  // TODO here we could block to fetch mirror informations, either if the RepoInfo has a metalink or mirrorlist entry
523  // or if the hostname of the repo is d.o.o
524  if ( repoUrls.begin()->baseUrl.getHost() == "download.opensuse.org" ){
525  //auto req = std::make_shared<zyppng::NetworkRequest>( );
526  }
527 
528  _dlRepoInfo.insert( std::make_pair(
529  pi.repository().id(),
531  ._baseUrls = std::move(repoUrls)
532  }
533  ));
534  }
535 
536 
537  _requiredBytes += pi.lookupLocation().downloadSize();
538  _requiredDls.push_back( pi );
539  }
540 
541  if ( _requiredDls.empty() )
542  return;
543 
544  // order by repo
545  std::sort( _requiredDls.begin(), _requiredDls.end(), []( const PoolItem &a , const PoolItem &b ) { return a.repository() < b.repository(); });
546 
547  const auto &workerDone = [&, this](){
548  if ( std::all_of( _workers.begin(), _workers.end(), []( const auto &w ) { return w->finished();} ) )
549  ev->quit();
550  };
551 
552  _report->start();
553  zypp_defer {
555  };
556 
557  MIL << "Downloading packages via " << MediaConfig::instance().download_max_concurrent_connections() << " connections." << std::endl;
558 
559  // we start a worker for each configured connection
560  for ( int i = 0; i < MediaConfig::instance().download_max_concurrent_connections() ; i++ ) {
561  // if we run out of jobs before we started all workers, stop
562  if (_requiredDls.empty())
563  break;
564  auto worker = std::make_shared<PreloadWorker>(*this);
565  worker->sigWorkerFinished().connect(workerDone);
566  worker->nextJob();
567  _workers.push_back( std::move(worker) );
568  }
569 
570  if( std::any_of( _workers.begin(), _workers.end(), []( const auto &w ) { return !w->finished(); } ) ) {
571  MIL << "Running preload event loop!" << std::endl;
572  ev->run();
573  }
574 
575  MIL << "Preloading done, returning" << std::endl;
576  }
577 
579  {
580  if ( !preloadEnabled() ) {
581  MIL << "CommitPackagePreloader disabled" << std::endl;
582  return;
583  }
584  std::for_each( _dlRepoInfo.begin (), _dlRepoInfo.end(), []( const auto &elem ){
586  });
587  }
588 
590  {
591  return _missedDownloads;
592  }
593 
595  {
596  _downloadedBytes += newBytes;
597  _pTracker->updateStats( _requiredBytes, _downloadedBytes );
598 
599  callback::UserData userData( "CommitPreloadReport/progress" );
600  userData.set( "dbps_avg" , static_cast<double>( _pTracker->_drateTotal ) );
601  userData.set( "dbps_current", static_cast<double>( _pTracker->_drateLast ) );
602  userData.set( "bytesReceived", static_cast<double>( _pTracker->_dnlNow ) );
603  userData.set( "bytesRequired", static_cast<double>( _pTracker->_dnlTotal ) );
604  if ( !_report->progress( _pTracker->_dnlPercent, userData ) ) {
605  _missedDownloads = true;
606  _requiredDls.clear();
607  _dispatcher->cancelAll( _("Cancelled by user."));
608  }
609  }
610 
611 }
std::string getScheme() const
Returns the scheme name of the URL.
Definition: Url.cc:551
#define MIL
Definition: Logger.h:100
RepoUrl * findUsableMirror(RepoUrl *skip=nullptr, bool allowTainted=true)
Tries to find a usable mirror
void addHeader(std::string &&val_r)
add a header, on the form "Foo: Bar" (trims)
int assert_dir(const Pathname &path, unsigned mode)
Like &#39;mkdir -p&#39;.
Definition: PathInfo.cc:324
Pathname cachedLocation(const OnMediaLocation &loc_r, const RepoInfo &repo_r)
Definition: Package.cc:99
#define _(MSG)
Definition: Gettext.h:39
void onRequestStarted(zyppng::NetworkRequest &req)
[M] Install(multiversion) item (
Definition: Transaction.h:67
static ZConfig & instance()
Singleton ctor.
Definition: ZConfig.cc:940
int clean_dir(const Pathname &path)
Like &#39;rm -r DIR/ *&#39;.
Definition: PathInfo.cc:447
SignalProxy< void(NetworkRequest &req, zypp::ByteCount count)> sigBytesDownloaded()
Signals that new data has been downloaded, this is only the payload and does not include control data...
Definition: request.cc:1056
unsigned short b
void appendPathName(const Pathname &path_r, EEncoding eflag_r=zypp::url::E_DECODED)
Extend the path name.
Definition: Url.cc:804
void reset()
Reset to default Ctor values.
Definition: AutoDispose.h:150
Store and operate with byte count.
Definition: ByteCount.h:31
static Url resolveUrl(const Url &url, HeaderList &headers)
Resolves an url using the installed plugins If no plugin is found the url is resolved as its current ...
Holds transfer setting.
Url clearQueryString(const Url &url)
Definition: curlhelper.cc:380
static const RepoIdType noRepoId(0)
Id to denote Repo::noRepository.
std::string toString() const
toString Returns a string representation of the error
media::UrlResolverPlugin::HeaderList headers
const std::string & asString(const std::string &t)
Global asString() that works with std::string too.
Definition: String.h:139
media::UrlResolverPlugin::HeaderList headers
zyppng::NetworkRequestDispatcherRef _dispatcher
static Url rewriteUrl(const Url &url_r, const media::MediaNr medianr)
Replaces media number in specified url with given medianr.
std::string basename() const
Return the last component of this path.
Definition: Pathname.h:130
TransferSettings & transferSettings()
Definition: request.cc:982
#define zypp_defer
Definition: AutoDispose.h:293
callback::SendReport< media::CommitPreloadReport > _report
void preloadTransaction(const std::vector< sat::Transaction::Step > &steps)
#define ERR
Definition: Logger.h:102
OnMediaLocation lookupLocation() const
Definition: SolvableType.h:160
bool hasError() const
Checks if there was a error with the request.
Definition: request.cc:1022
void failCurrentJob(const zypp::Pathname &localPath, const std::optional< zypp::Url > &url, media::CommitPreloadReport::Error e, const std::optional< std::string > &errorMessage)
bool isKind(const Resolvable::constPtr &p)
Test whether a Resolvable::Ptr is of a certain Kind.
Definition: Resolvable.h:99
void onRequestFinished(zyppng::NetworkRequest &req, const zyppng::NetworkRequestError &err)
bool taintCurrentMirror()
Taints the current mirror, returns true if a alternative was found
WeakPtr parent() const
Definition: base.cc:26
void makeJobUrl(zypp::Url &resultUrl, media::TransferSettings &resultSet)
bool set(const std::string &key_r, AnyType val_r)
Set the value for key (nonconst version always returns true).
Definition: UserData.h:119
zyppng::Ref< internal::ProgressTracker > _pTracker
bool isExist() const
Return whether valid stat info exists.
Definition: PathInfo.h:286
std::vector< zyppng::Ref< PreloadWorker > > _workers
Package interface.
Definition: Package.h:33
RepoInfo repoInfo() const
Definition: SolvableType.h:76
Pathname dirname() const
Return all but the last component od this path.
Definition: Pathname.h:126
long download_max_concurrent_connections() const
Definition: mediaconfig.cc:109
void onRequestProgress(zyppng::NetworkRequest &req, zypp::ByteCount count)
RepoInfo info() const
Return any associated RepoInfo.
Definition: Repository.cc:274
The NetworkRequestError class Represents a error that occured in.
static std::optional< MediaHandlerType > handlerType(const Url &url)
NetworkRequestError error() const
Returns the last set Error.
Definition: request.cc:1006
std::string extendedErrorString() const
In some cases, curl can provide extended error information collected at runtime.
Definition: request.cc:1014
void fillSettingsFromUrl(const Url &url, media::TransferSettings &s)
Fills the settings structure using options passed on the url for example ?timeout=x&proxy=foo.
Definition: curlhelper.cc:183
sat::detail::RepoIdType IdType
Definition: Repository.h:44
std::map< Repository::IdType, RepoDownloadData > _dlRepoInfo
static Ptr create()
const Pathname & filename() const
The path to the resource on the medium.
int unlink(const Pathname &path)
Like &#39;unlink&#39;.
Definition: PathInfo.cc:705
Pathname predownloadPath() const
Path where this repo packages are predownloaded.
Definition: RepoInfo.cc:597
SrcPackage interface.
Definition: SrcPackage.h:29
bool any_of(const Container &c, Fnc &&cb)
Definition: Algorithm.h:76
Typesafe passing of user data via callbacks.
Definition: UserData.h:39
unsigned short a
std::multimap< std::string, std::string > HeaderList
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:225
void reportBytesDownloaded(ByteCount newBytes)
Combining sat::Solvable and ResStatus.
Definition: PoolItem.h:50
SignalProxy< void(NetworkRequest &req, const NetworkRequestError &err)> sigFinished()
Signals that the download finished.
Definition: request.cc:1066
void fillSettingsSystemProxy(const Url &url, media::TransferSettings &s)
Reads the system proxy configuration and fills the settings structure proxy information.
Definition: curlhelper.cc:338
SignalProxy< void(NetworkRequest &req)> sigStarted()
Signals that the dispatcher dequeued the request and actually starts downloading data.
Definition: request.cc:1051
int rename(const Pathname &oldpath, const Pathname &newpath)
Like &#39;rename&#39;.
Definition: PathInfo.cc:747
static bool authenticate(const Url &url, CredentialManager &cm, TransferSettings &settings, const std::string &availAuthTypes, bool firstTry)
Definition: MediaCurl2.cc:682
IdType id() const
Expert backdoor.
Definition: Repository.h:321
Easy-to use interface to the ZYPP dependency resolver.
Definition: Application.cc:19
const std::string & proxy() const
proxy host
int rmdir(const Pathname &path)
Like &#39;rmdir&#39;.
Definition: PathInfo.cc:371
Repository repository() const
Definition: SolvableType.h:75
Url manipulation class.
Definition: Url.h:92
static MediaConfig & instance()
Definition: mediaconfig.cc:46
#define DBG
Definition: Logger.h:99
bool is_checksum(const Pathname &file, const CheckSum &checksum)
check files checksum
Definition: PathInfo.cc:1068
static ManagedFile asManagedFile()
Create a temporary file and convert it to a automatically cleaned up ManagedFile. ...
Definition: TmpPath.cc:240