libzypp 17.37.10
commitpackagepreloader.cc
Go to the documentation of this file.
5#include <zypp/media/MediaCurl2.h> // for shared logic like authenticate
6#include <zypp/media/MediaHandlerFactory.h> // to detect the URL type
14#include <zypp/MediaSetAccess.h>
15#include <zypp/Package.h>
16#include <zypp/SrcPackage.h>
17#include <zypp/ZConfig.h>
18#include <zypp/base/Env.h>
19
20namespace zypp {
21
22 namespace {
23
24 inline bool preloadEnabled()
25 {
26 TriBool envstate = env::getenvBool( "ZYPP_PCK_PRELOAD" );
27 if ( indeterminate(envstate) ) {
28#if APIConfig(LIBZYPP_CONFIG_USE_SERIAL_PACKAGE_DOWNLOAD_BY_DEFAULT)
29 return false;
30#else
31 return true;
32#endif
33 }
34 return bool(envstate);
35 }
36
37 zypp::Pathname pckCachedLocation ( const PoolItem &pck ) {
38 if ( pck.isKind<Package>() ) {
39 return pck->asKind<Package>()->cachedLocation();
40 } else if ( pck.isKind<SrcPackage>() ) {
41 return pck->asKind<SrcPackage>()->cachedLocation();
42 }
43 return {};
44 }
45
46 }
47
52
54 public:
55 enum State {
58 //ZckHead,
59 //ZckData,
61 };
62
64
65 bool finished ( ) const {
66 return (_s == Finished);
67 }
68
69 void nextJob () {
70
71 // clean state vars
72 _started = false;
73 _firstAuth = true;
75 _tmpFile.reset();
77 _taintedMirrors.clear();
78
79 if ( _parent._requiredDls.empty() ) {
80
81 if ( _myMirror ) {
82 _myMirror->refs--;
83 _myMirror = nullptr;
85 }
86
87 MIL << "No more jobs pending, exiting worker" << std::endl;
88 // exit!
89 _s = Finished;
90 _sigFinished.emit();
91 return;
92 }
93
94 _job = _parent._requiredDls.front();
95 _parent._requiredDls.pop_front();
96
97 auto loc = _job.lookupLocation();
98 _targetPath = _job.repoInfo().predownloadPath() / _job.lookupLocation().filename();
99
100 // select a mirror we want to use
101 if ( !prepareMirror( ) ) {
102 finishCurrentJob ( _targetPath, {}, media::CommitPreloadReport::ERROR, asString( _("no mirror found") ), true );
103 return nextJob();
104 }
105
106 if ( filesystem::assert_dir( _targetPath.dirname()) != 0 ) {
107 ERR << "Failed to create target dir for file: " << _targetPath << std::endl;
108 finishCurrentJob ( _targetPath, {}, media::CommitPreloadReport::ERROR, asString( _("could not create target file") ), true );
109 return nextJob();
110 }
111
112
115 makeJobUrl ( url, settings );
116
117 // check if the file is there already
118 {
119 PathInfo pathInfo(_targetPath);
120 if ( pathInfo.isExist() ) {
121 // just in case there is something else that is not a file we delete it
122 if ( !pathInfo.isFile() ) {
123 if ( pathInfo.isDir () )
125 else
127
128 } else if ( is_checksum( _targetPath, loc.checksum() ) ) {
129 // if we have the file already, no need to download again
131 return nextJob();
132
133 } else {
134 // everything else we delete
136 }
137 }
138 }
139
140 // we download into a temp file so that we don't leave broken files in case of errors or a crash
142
143 if ( _s == Pending ) {
144 // init case, set up request
145 _req = std::make_shared<zyppng::NetworkRequest>( url, _tmpFile );
149 } else {
150 _req->resetRequestRanges();
151 _req->setUrl( url );
152 _req->setTargetFilePath( _tmpFile );
153 }
154
155 // TODO check for zchunk
156
157 _s = SimpleDl;
158 _req->transferSettings() = settings;
159 _parent._dispatcher->enqueue(_req);
160 }
161
165
166 private:
167
168 // TODO some smarter logic that selects mirrors
170
171 const auto &pi = _job;
172
173 if ( _myMirror ) {
174 if ( _currentRepoId == pi.repository().id() ) {
175 return true;
176 }
178 _myMirror->refs--;
179 _myMirror = nullptr;
180 }
181
183 if ( !_myMirror )
184 return false;
185
186 _currentRepoId = pi.repository().id();
187 _myMirror->refs++;
188 return true;
189 }
190
195
196 if ( _myMirror ) {
197 _myMirror->miss++;
198 _taintedMirrors.insert( _myMirror );
199 }
200
201 // try to find another mirror
202 auto mirrPtr = findUsableMirror ( _myMirror, false );
203 if ( mirrPtr ) {
204 if ( _myMirror ) {
205 _myMirror->refs--;
206 }
207 _myMirror = mirrPtr;
208 _myMirror->refs++;
209 return true;
210 }
211 return false;
212 }
213
217 RepoUrl *findUsableMirror( RepoUrl *skip = nullptr, bool allowTainted = true ) {
218 auto &repoDlInfo = _parent._dlRepoInfo.at( _job.repository().id() );
219
220 std::vector<RepoUrl>::iterator curr = repoDlInfo._baseUrls.end();
221 int currentSmallestRef = INT_MAX;
222
223 for ( auto i = repoDlInfo._baseUrls.begin(); i != repoDlInfo._baseUrls.end(); i++ ) {
224 auto mirrorPtr = &(*i);
225
226 if ( skip == mirrorPtr )
227 continue;
228
229 if ( !allowTainted && _taintedMirrors.find(mirrorPtr) != _taintedMirrors.end() )
230 continue;
231
232 // we are adding the file misses on top of the refcount
233 // that way we will use mirrors that often miss a file less
234 if ( ( i->refs + i->miss ) < currentSmallestRef ) {
235 currentSmallestRef = ( i->refs + i->miss );
236 curr = i;
237 }
238 }
239
240 if ( curr == repoDlInfo._baseUrls.end() )
241 return nullptr;
242 return &(*curr);
243 }
244
246 MIL << "Request for " << req.url() << " started" << std::endl;
247 }
248
250 if ( !_started ) {
251 _started = true;
252
253 callback::UserData userData( "CommitPreloadReport/fileStart" );
254 userData.set( "Url", _req->url() );
255 _parent._report->fileStart( _targetPath, userData );
256 }
257
258 ByteCount downloaded;
259 if ( _lastByteCount == 0 )
260 downloaded = count;
261 else
262 downloaded = count - _lastByteCount;
263 _lastByteCount = count;
264
265 _parent.reportBytesDownloaded( downloaded );
266 }
267
269 MIL << "Request for " << req.url() << " finished. (" << err.toString() << ")" << std::endl;
270 if ( !req.hasError() ) {
271 // apply umask and move the _tmpFile into _targetPath
273 _tmpFile.resetDispose(); // rename consumed the file, no need to unlink.
275 } else {
276 // error
277 finishCurrentJob ( _targetPath, req.url(), media::CommitPreloadReport::ERROR, _("failed to rename temporary file."), true );
278 }
279 } else {
280 // handle errors and auth
281 const auto &error = req.error();
282 switch ( error.type() ) {
299 MIL << "Download from mirror failed for file " << req.url () << " trying to taint mirror and move on" << std::endl;
300
301 if ( taintCurrentMirror() ) {
303
304 const auto str = zypp::str::Format(_("Error: \"%1%\", trying next mirror.")) % req.extendedErrorString();
306
309 makeJobUrl ( url, settings );
310
311 MIL << "Found new mirror: " << url << " recovering, retry count: " << _notFoundRetry << std::endl;
312
313 _req->setUrl( url );
314 _req->transferSettings () = settings;
315
316 _parent._dispatcher->enqueue( _req );
317 return;
318 }
319
321 break;
322 }
325
326 //in case we got a auth hint from the server the error object will contain it
327 std::string authHint = error.extraInfoValue("authHint", std::string());
328
330 bool newCreds = media::MediaNetworkCommonHandler::authenticate( _myMirror->baseUrl, cm, req.transferSettings(), authHint, _firstAuth );
331 if ( newCreds) {
332 _firstAuth = false;
333 _parent._dispatcher->enqueue( _req );
334 return;
335 }
336
338 break;
339
342 break;
343 }
345 // should never happen
346 DBG << "BUG: Download error flag is set , but Error code is NoError" << std::endl;
347 break;
348 }
349 }
350 nextJob();
351 }
352
353 void finishCurrentJob( const zypp::Pathname &localPath, const std::optional<zypp::Url> &url, media::CommitPreloadReport::Error e, const std::optional<std::string> &errorMessage, bool fatal ) {
354
355 callback::UserData userData( "CommitPreloadReport/fileDone" );
356 if ( url )
357 userData.set( "Url", *url );
358 if ( errorMessage )
359 userData.set( "description", *errorMessage );
360
361 if ( e != media::CommitPreloadReport::NO_ERROR && fatal )
362 _parent._missedDownloads = true;
363
364 _parent._report->fileDone( localPath, e, userData );
365 }
366
367 void makeJobUrl ( zypp::Url &resultUrl, media::TransferSettings &resultSet ) {
368
369 // rewrite Url
370 zypp::Url url = _myMirror->baseUrl;
371
374
375 const auto &loc = _job.lookupLocation();
376
377 // rewrite URL for media handle
378 if ( loc.medianr() > 1 )
379 url = MediaSetAccess::rewriteUrl( url ,loc.medianr() );
380
381 // append path to file
382 url.appendPathName( loc.filename() );
383
384 // add extra headers
385 for ( const auto & el : _myMirror->headers ) {
386 std::string header { el.first };
387 header += ": ";
388 header += el.second;
389 MIL << "Added custom header -> " << header << std::endl;
390 settings.addHeader( std::move(header) );
391 }
392
393 resultUrl = url;
394 resultSet = settings;
395 }
396
397 private:
400 zyppng::NetworkRequestRef _req;
401
405 bool _started = false;
406 bool _firstAuth = true;
407 RepoUrl *_myMirror = nullptr;
410
411 // retry handling
413 std::set<RepoUrl *> _taintedMirrors; //< mirrors that returned 404 for the current request
414
416
417 };
418
421
422 void CommitPackagePreloader::preloadTransaction( const std::vector<sat::Transaction::Step> &steps)
423 {
424 if ( !preloadEnabled() ) {
425 MIL << "CommitPackagePreloader disabled" << std::endl;
426 return;
427 }
428
429 // preload happens only if someone handles the report
430 if ( !_report->connected() ) {
431 MIL << "No receiver for the CommitPreloadReport, skipping preload phase" << std::endl;
432 return;
433 }
434
435 auto ev = zyppng::EventLoop::create();
436 _dispatcher = std::make_shared<zyppng::NetworkRequestDispatcher>();
437 _dispatcher->setMaximumConcurrentConnections( MediaConfig::instance().download_max_concurrent_connections() );
439 _dispatcher->setHostSpecificHeader ("download.opensuse.org", "X-ZYpp-DistributionFlavor", str::asString(media::MediaCurl2::distributionFlavorHeader()) );
440 _dispatcher->setHostSpecificHeader ("download.opensuse.org", "X-ZYpp-AnonymousId", str::asString(media::MediaCurl2::anonymousIdHeader()) );
441 _dispatcher->run();
442
443 _pTracker = std::make_shared<internal::ProgressTracker>();
444 _requiredBytes = 0;
446 _missedDownloads = false;
447 _lastProgressUpdate.reset();
448
449 zypp_defer {
450 _dispatcher.reset();
451 _pTracker.reset();
452 };
453
454 for ( const auto &step : steps ) {
455 switch ( step.stepType() )
456 {
459 // proceed: only install actions may require download.
460 break;
461
462 default:
463 // next: no download for non-packages and delete actions.
464 continue;
465 break;
466 }
467
468 PoolItem pi(step.satSolvable());
469
470 if ( !pi->isKind<Package>() && !pi->isKind<SrcPackage>() )
471 continue;
472
473 // no checksum ,no predownload, Fetcher would ignore it
474 if ( pi->lookupLocation().checksum().empty() )
475 continue;
476
477 // check if Package is cached already
478 if( !pckCachedLocation(pi).empty() )
479 continue;
480
481 auto repoDlsIter = _dlRepoInfo.find( pi.repository().id() );
482 if ( repoDlsIter == _dlRepoInfo.end() ) {
483
484 // make sure download path for this repo exists
485 if ( filesystem::assert_dir( pi.repoInfo().predownloadPath() ) != 0 ) {
486 ERR << "Failed to create predownload cache for repo " << pi.repoInfo().alias() << std::endl;
487 return;
488 }
489
490 // filter base URLs that do not download
491 std::vector<RepoUrl> repoUrls;
492 const auto origins = pi.repoInfo().repoOrigins();
493 for ( const auto &origin: origins ) {
494 std::for_each( origin.begin(), origin.end(), [&]( const zypp::OriginEndpoint &u ) {
495 media::UrlResolverPlugin::HeaderList custom_headers;
496 Url url = media::UrlResolverPlugin::resolveUrl(u.url(), custom_headers);
497
498 if ( media::MediaHandlerFactory::handlerType(url) != media::MediaHandlerFactory::MediaCURLType )
499 return;
500
501 // use geo IP if available
502 {
503 const auto rewriteUrl = media::MediaNetworkCommonHandler::findGeoIPRedirect( url );
504 if ( rewriteUrl.isValid () )
505 url = rewriteUrl;
506 }
507
508 MIL << "Adding Url: " << url << " to the mirror set" << std::endl;
509
510 repoUrls.push_back( RepoUrl {
511 .baseUrl = std::move(url),
512 .headers = std::move(custom_headers)
513 } );
514 });
515 }
516
517 // skip this solvable if it has no downloading base URLs
518 if( repoUrls.empty() ) {
519 MIL << "Skipping predownload for " << step.satSolvable() << " no downloading URL" << std::endl;
520 continue;
521 }
522
523 // TODO here we could block to fetch mirror informations, either if the RepoInfo has a metalink or mirrorlist entry
524 // or if the hostname of the repo is d.o.o
525 if ( repoUrls.begin()->baseUrl.getHost() == "download.opensuse.org" ){
526 //auto req = std::make_shared<zyppng::NetworkRequest>( );
527 }
528
529 _dlRepoInfo.insert( std::make_pair(
530 pi.repository().id(),
532 ._baseUrls = std::move(repoUrls)
533 }
534 ));
535 }
536
537
538 _requiredBytes += pi.lookupLocation().downloadSize();
539 _requiredDls.push_back( pi );
540 }
541
542 if ( _requiredDls.empty() )
543 return;
544
545 // order by repo
546 std::sort( _requiredDls.begin(), _requiredDls.end(), []( const PoolItem &a , const PoolItem &b ) { return a.repository() < b.repository(); });
547
548 const auto &workerDone = [&, this](){
549 if ( std::all_of( _workers.begin(), _workers.end(), []( const auto &w ) { return w->finished();} ) )
550 ev->quit();
551 };
552
553 _report->start();
554 zypp_defer {
555 _report->finish( _missedDownloads ? media::CommitPreloadReport::MISS : media::CommitPreloadReport::SUCCESS );
556 };
557
558 MIL << "Downloading packages via " << MediaConfig::instance().download_max_concurrent_connections() << " connections." << std::endl;
559
560 // we start a worker for each configured connection
561 for ( int i = 0; i < MediaConfig::instance().download_max_concurrent_connections() ; i++ ) {
562 // if we run out of jobs before we started all workers, stop
563 if (_requiredDls.empty())
564 break;
565 auto worker = std::make_shared<PreloadWorker>(*this);
566 worker->sigWorkerFinished().connect(workerDone);
567 worker->nextJob();
568 _workers.push_back( std::move(worker) );
569 }
570
571 if( std::any_of( _workers.begin(), _workers.end(), []( const auto &w ) { return !w->finished(); } ) ) {
572 MIL << "Running preload event loop!" << std::endl;
573 ev->run();
574 }
575
576 MIL << "Preloading done, mirror stats: " << std::endl;
577 for ( const auto &elem : _dlRepoInfo ) {
578 std::for_each ( elem.second._baseUrls.begin (), elem.second._baseUrls.end(), []( const RepoUrl &repoUrl ){
579 MIL << "url: " << repoUrl.baseUrl << " misses: " << repoUrl.miss << std::endl;
580 });
581 }
582 MIL << "Preloading done, mirror stats end" << std::endl;
583 }
584
586 {
587 if ( !preloadEnabled() ) {
588 MIL << "CommitPackagePreloader disabled" << std::endl;
589 return;
590 }
591 std::for_each( _dlRepoInfo.begin (), _dlRepoInfo.end(), []( const auto &elem ){
592 filesystem::clean_dir ( Repository(elem.first).info().predownloadPath() );
593 });
594 }
595
597 {
598 return _missedDownloads;
599 }
600
602 {
603 // throttle progress updates to one time per second
604 const auto now = clock::now();
605 bool canUpdate = false;
606 if ( _lastProgressUpdate ) {
607 const auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now - *_lastProgressUpdate);
608 canUpdate = (duration >= std::chrono::milliseconds(500));
609 } else {
610 canUpdate = true;
611 }
612
613 _downloadedBytes += newBytes;
615
616 // update progress one time per second
617 if( canUpdate ) {
619 callback::UserData userData( "CommitPreloadReport/progress" );
620 userData.set( "dbps_avg" , static_cast<double>( _pTracker->_drateTotal ) );
621 userData.set( "dbps_current", static_cast<double>( _pTracker->_drateLast ) );
622 userData.set( "bytesReceived", static_cast<double>( _pTracker->_dnlNow ) );
623 userData.set( "bytesRequired", static_cast<double>( _pTracker->_dnlTotal ) );
624 if ( !_report->progress( _pTracker->_dnlPercent, userData ) ) {
625 _missedDownloads = true;
626 _requiredDls.clear();
627 _dispatcher->cancelAll( _("Cancelled by user."));
628 }
629 }
630 }
631
632}
Store and operate with byte count.
Definition ByteCount.h:32
void onRequestProgress(zyppng::NetworkRequest &req, zypp::ByteCount count)
RepoUrl * findUsableMirror(RepoUrl *skip=nullptr, bool allowTainted=true)
Tries to find a usable mirror.
void makeJobUrl(zypp::Url &resultUrl, media::TransferSettings &resultSet)
void onRequestStarted(zyppng::NetworkRequest &req)
bool taintCurrentMirror()
Taints the current mirror, returns true if a alternative was found.
void onRequestFinished(zyppng::NetworkRequest &req, const zyppng::NetworkRequestError &err)
void finishCurrentJob(const zypp::Pathname &localPath, const std::optional< zypp::Url > &url, media::CommitPreloadReport::Error e, const std::optional< std::string > &errorMessage, bool fatal)
callback::SendReport< media::CommitPreloadReport > _report
std::optional< clock::time_point > _lastProgressUpdate
zyppng::Ref< internal::ProgressTracker > _pTracker
std::map< Repository::IdType, RepoDownloadData > _dlRepoInfo
void reportBytesDownloaded(ByteCount newBytes)
void preloadTransaction(const std::vector< sat::Transaction::Step > &steps)
zyppng::NetworkRequestDispatcherRef _dispatcher
long download_max_concurrent_connections() const
static MediaConfig & instance()
static Url rewriteUrl(const Url &url_r, const media::MediaNr medianr)
Replaces media number in specified url with given medianr.
Represents a single, configurable network endpoint, combining a URL with specific access settings.
Package interface.
Definition Package.h:34
Combining sat::Solvable and ResStatus.
Definition PoolItem.h:51
Pathname predownloadPath() const
Path where this repo packages are predownloaded.
Definition RepoInfo.cc:715
MirroredOriginSet repoOrigins() const
The repodata origins.
Definition RepoInfo.cc:689
IdType id() const
Expert backdoor.
Definition Repository.h:321
sat::detail::RepoIdType IdType
Definition Repository.h:44
SrcPackage interface.
Definition SrcPackage.h:30
Url manipulation class.
Definition Url.h:93
static ZConfig & instance()
Singleton ctor.
Definition ZConfig.cc:940
Typesafe passing of user data via callbacks.
Definition UserData.h:40
bool set(const std::string &key_r, AnyType val_r)
Set the value for key (nonconst version always returns true).
Definition UserData.h:119
Wrapper class for stat/lstat.
Definition PathInfo.h:226
bool isExist() const
Return whether valid stat info exists.
Definition PathInfo.h:286
static ManagedFile asManagedFile()
Create a temporary file and convert it to a automatically cleaned up ManagedFile.
Definition TmpPath.cc:240
bool authenticate(const Url &url, TransferSettings &settings, const std::string &availAuthTypes, bool firstTry)
Holds transfer setting.
void addHeader(std::string &&val_r)
add a header, on the form "Foo: Bar" (trims)
std::multimap< std::string, std::string > HeaderList
std::string alias() const
unique identifier for this source.
@ TRANSACTION_MULTIINSTALL
[M] Install(multiversion) item (
Definition Transaction.h:67
@ TRANSACTION_INSTALL
[+] Install(update) item
Definition Transaction.h:66
WeakPtr parent() const
Definition base.cc:26
static Ptr create()
The NetworkRequestError class Represents a error that occured in.
std::string toString() const
toString Returns a string representation of the error
bool hasError() const
Checks if there was a error with the request.
Definition request.cc:1033
SignalProxy< void(NetworkRequest &req, const NetworkRequestError &err)> sigFinished()
Signals that the download finished.
Definition request.cc:1077
SignalProxy< void(NetworkRequest &req, zypp::ByteCount count)> sigBytesDownloaded()
Signals that new data has been downloaded, this is only the payload and does not include control data...
Definition request.cc:1067
std::string extendedErrorString() const
In some cases, curl can provide extended error information collected at runtime.
Definition request.cc:1025
NetworkRequestError error() const
Returns the last set Error.
Definition request.cc:1017
SignalProxy< void(NetworkRequest &req)> sigStarted()
Signals that the dispatcher dequeued the request and actually starts downloading data.
Definition request.cc:1062
TransferSettings & transferSettings()
Definition request.cc:993
unsigned short a
unsigned short b
void prepareSettingsAndUrl(zypp::Url &url_r, zypp::media::TransferSettings &s)
String related utilities and Regular expression matching.
TriBool getenvBool(const C_Str &var_r)
If the environment variable var_r is set to a legal true or false string return bool,...
Definition Env.h:32
int rmdir(const Pathname &path)
Like 'rmdir'.
Definition PathInfo.cc:371
int unlink(const Pathname &path)
Like 'unlink'.
Definition PathInfo.cc:705
int assert_dir(const Pathname &path, unsigned mode)
Like 'mkdir -p'.
Definition PathInfo.cc:324
int rename(const Pathname &oldpath, const Pathname &newpath)
Like 'rename'.
Definition PathInfo.cc:747
int chmodApplyUmask(const Pathname &path, mode_t mode)
Similar to 'chmod', but mode is modified by the process's umask in the usual way.
Definition PathInfo.cc:1106
static const RepoIdType noRepoId(0)
Id to denote Repo::noRepository.
const std::string & asString(const std::string &t)
Global asString() that works with std::string too.
Definition String.h:140
Url details namespace.
Definition UrlBase.cc:58
Easy-to use interface to the ZYPP dependency resolver.
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
Definition ManagedFile.h:27
boost::logic::tribool TriBool
3-state boolean logic (true, false and indeterminate).
Definition String.h:31
std::string asString(const Patch::Category &obj)
Definition Patch.cc:122
Pathname cachedLocation(const OnMediaLocation &loc_r, const RepoInfo &repo_r)
Definition Package.cc:99
media::UrlResolverPlugin::HeaderList headers
RepoInfo repoInfo() const
Repository repository() const
Convenient building of std::string with boost::format.
Definition String.h:254
#define zypp_defer
#define _(MSG)
Definition Gettext.h:39
#define DBG
Definition Logger.h:99
#define MIL
Definition Logger.h:100
#define ERR
Definition Logger.h:102