libzypp 17.34.0
susetags.cc
Go to the documentation of this file.
1/*---------------------------------------------------------------------\
2| ____ _ __ __ ___ |
3| |__ / \ / / . \ . \ |
4| / / \ V /| _/ _/ |
5| / /__ | | | | | | |
6| /_____||_| |_| |_| |
7| |
8\---------------------------------------------------------------------*/
9#include "susetags.h"
11#include <zypp-core/zyppng/ui/ProgressObserver>
12#include <zypp-media/ng/ProvideSpec>
13#include <zypp/ng/Context>
14
15#include <zypp-core/parser/ParseException>
16
19
24
26
27 namespace {
28
29 using namespace zyppng::operators;
30
36 template<class Executor, class OpType>
37 struct StatusLogic : public LogicBase<Executor, OpType>{
39
40 public:
41
42 using DlContextRefType = std::conditional_t<zyppng::detail::is_async_op_v<OpType>, repo::AsyncDownloadContextRef, repo::SyncDownloadContextRef>;
43 using ZyppContextType = typename DlContextRefType::element_type::ContextType;
44 using ProvideType = typename ZyppContextType::ProvideType;
45 using MediaHandle = typename ProvideType::MediaHandle;
46 using ProvideRes = typename ProvideType::Res;
47
48 StatusLogic( DlContextRefType ctx, MediaHandle &&media )
49 : _ctx(std::move(ctx))
50 , _handle(std::move(media))
51 {}
52
54 return _ctx->zyppContext()->provider()->provide( _handle, _ctx->repoInfo().path() / "content" , ProvideFileSpec() )
56
57 // mandatory master index is missing -> stay empty
58 if ( !contentFile )
60
61 zypp::RepoStatus status ( contentFile->file() );
62
63 if ( !status.empty() /* && _ctx->repoInfo().requireStatusWithMediaFile() */ ) {
64 return _ctx->zyppContext()->provider()->provide( _handle, "/media.1/media" , ProvideFileSpec())
65 | [status = std::move(status)]( expected<ProvideRes> mediaFile ) mutable {
66 if ( mediaFile ) {
67 return make_expected_success(status && zypp::RepoStatus( mediaFile->file()) );
68 }
69 return make_expected_success( std::move(status) );
70 };
71 }
72 return makeReadyResult( make_expected_success(std::move(status)) );
73 };
74 }
75
76 DlContextRefType _ctx;
77 MediaHandle _handle;
78 };
79 }
80
81 AsyncOpRef<expected<zypp::RepoStatus> > repoStatus(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle)
82 {
83 return SimpleExecutor< StatusLogic, AsyncOp<expected<zypp::RepoStatus>> >::run( std::move(dl), std::move(mediaHandle) );
84 }
85
86 expected<zypp::RepoStatus> repoStatus(repo::SyncDownloadContextRef dl, SyncMediaHandle mediaHandle)
87 {
88 return SimpleExecutor< StatusLogic, SyncOp<expected<zypp::RepoStatus>> >::run( std::move(dl), std::move(mediaHandle) );
89 }
90
91
92
93 namespace {
94
95 using namespace zyppng::operators;
96
97 // search old repository file file to run the delta algorithm on
98 static zypp::Pathname search_deltafile( const zypp::Pathname &dir, const zypp::Pathname &file )
99 {
100 zypp::Pathname deltafile(dir + file.basename());
101 if (zypp::PathInfo(deltafile).isExist())
102 return deltafile;
103 return zypp::Pathname();
104 }
105
106
107 template<class Executor, class OpType>
108 struct DlLogic : public LogicBase<Executor, OpType> {
109
111 public:
112
113 using DlContextRefType = std::conditional_t<zyppng::detail::is_async_op_v<OpType>, repo::AsyncDownloadContextRef, repo::SyncDownloadContextRef>;
114 using ZyppContextType = typename DlContextRefType::element_type::ContextType;
115 using ProvideType = typename ZyppContextType::ProvideType;
116 using MediaHandle = typename ProvideType::MediaHandle;
117 using ProvideRes = typename ProvideType::Res;
118
119 DlLogic( DlContextRefType ctx, MediaHandle &&mediaHandle, ProgressObserverRef &&progressObserver )
120 : _ctx( std::move(ctx))
121 , _mediaHandle(std::move(mediaHandle))
123 {}
124
125 auto execute() {
126 // download media file here
127 return RepoDownloaderWorkflow::downloadMediaInfo( _mediaHandle, _ctx->destDir() )
128 | [this]( expected<zypp::ManagedFile> &&mediaInfo ) {
129
130 // remember the media info if we had one
131 if ( mediaInfo ) _ctx->files().push_back ( std::move(mediaInfo.get()) );
132
133 if ( _progressObserver ) _progressObserver->inc();
134
135 return RepoDownloaderWorkflow::downloadMasterIndex( _ctx, _mediaHandle, _ctx->repoInfo().path() / "content" )
136 | inspect( incProgress( _progressObserver ) )
137 | and_then( [this] ( DlContextRefType && ) {
138
139 zypp::Pathname contentPath = _ctx->files().front();
140 std::vector<zypp::OnMediaLocation> requiredFiles;
141
142 // Content file first to get the repoindex
143 try {
144 const zypp::Pathname &inputfile = contentPath;
146 content.setRepoIndexConsumer( [this]( const auto & data_r ) {
147 MIL << "Consuming repo index" << std::endl;
149 });
150 content.parse( inputfile );
151
152 if ( ! _repoindex ) {
153 ZYPP_THROW( zypp::parser::ParseException( ( _ctx->destDir() / _ctx->repoInfo().path() ).asString() + ": " + "No repository index in content file." ) );
154 }
155
156 MIL << "RepoIndex: " << _repoindex << std::endl;
157 if ( _repoindex->metaFileChecksums.empty() ) {
158 ZYPP_THROW( zypp::parser::ParseException( ( _ctx->destDir() / _ctx->repoInfo().path() ).asString() + ": " + "No metadata checksums in content file." ) );
159 }
160
161 if ( _repoindex->signingKeys.empty() ) {
162 WAR << "No signing keys defined." << std::endl;
163 }
164
165 // Prepare parsing
166 zypp::Pathname descr_dir = _repoindex->descrdir; // path below reporoot
167 //_datadir = _repoIndex->datadir; // path below reporoot
168
169 std::map<std::string,zypp::parser::susetags::RepoIndex::FileChecksumMap::const_iterator> availablePackageTranslations;
170
171 for_( it, _repoindex->metaFileChecksums.begin(), _repoindex->metaFileChecksums.end() )
172 {
173 // omit unwanted translations
174 if ( zypp::str::hasPrefix( it->first, "packages" ) )
175 {
176 static const zypp::str::regex rx_packages( "^packages((.gz)?|(.([^.]*))(.gz)?)$" );
178
179 if ( zypp::str::regex_match( it->first, what, rx_packages ) ) {
180 if ( what[4].empty() // packages(.gz)?
181 || what[4] == "DU"
182 || what[4] == "en" )
183 { ; /* always downloaded */ }
184 else if ( what[4] == "FL" )
185 { continue; /* never downloaded */ }
186 else
187 {
188 // remember and decide later
190 continue;
191 }
192 }
193 else
194 continue; // discard
195
196 } else if ( it->first == "patterns.pat"
197 || it->first == "patterns.pat.gz" ) {
198 // take all patterns in one go
199
200 } else if ( zypp::str::endsWith( it->first, ".pat" )
201 || zypp::str::endsWith( it->first, ".pat.gz" ) ) {
202
203 // *** see also zypp/parser/susetags/RepoParser.cc ***
204
205 // omit unwanted patterns, see https://bugzilla.novell.com/show_bug.cgi?id=298716
206 // expect "<name>.<arch>.pat[.gz]", <name> might contain additional dots
207 // split at dots, take .pat or .pat.gz into account
208
209 std::vector<std::string> patparts;
210 unsigned archpos = 2;
211 // expect "<name>.<arch>.pat[.gz]", <name> might contain additional dots
212 unsigned count = zypp::str::split( it->first, std::back_inserter(patparts), "." );
213 if ( patparts[count-1] == "gz" )
214 archpos++;
215
216 if ( count > archpos ) {
217 try { // might by an invalid architecture
219 if ( !patarch.compatibleWith( zConfig().systemArchitecture() ) ) {
220 // discard, if not compatible
221 MIL << "Discarding pattern " << it->first << std::endl;
222 continue;
223 }
224
225 } catch ( const zypp::Exception & excpt ) {
226 WAR << "Pattern file name does not contain recognizable architecture: " << it->first << std::endl;
227 // keep .pat file if it doesn't contain an recognizable arch
228 }
229 }
230 }
231
232 MIL << "adding job " << it->first << std::endl;
233 auto location = zypp::OnMediaLocation( repoInfo().path() + descr_dir + it->first, 1 )
234 .setChecksum( it->second )
235 .setDeltafile( search_deltafile( _ctx->deltaDir() + descr_dir, it->first) );
236
237 requiredFiles.push_back( std::move(location) );
238 }
239
240
241 // check whether to download more package translations:
242 {
243 auto fnc_checkTransaltions( [&]( const zypp::Locale & locale_r ) {
244 for ( zypp::Locale toGet( locale_r ); toGet; toGet = toGet.fallback() ) {
245 auto it( availablePackageTranslations.find( toGet.code() ) );
246 if ( it != availablePackageTranslations.end() ) {
247 auto mit( it->second );
248 MIL << "adding job " << mit->first << std::endl;
249 requiredFiles.push_back( zypp::OnMediaLocation( repoInfo().path() + descr_dir + mit->first, 1 )
250 .setChecksum( mit->second )
251 .setDeltafile( search_deltafile( deltaDir() + descr_dir, mit->first) ));
252 break;
253 }
254 }
255 });
256
257 for ( const zypp::Locale & it : zConfig().repoRefreshLocales() ) {
259 }
260 fnc_checkTransaltions( zConfig().textLocale() );
261 }
262
263 for( const auto &it : _repoindex->mediaFileChecksums ) {
264 // Repo adopts license files listed in HASH
265 if ( it.first != "license.tar.gz" )
266 continue;
267
268 MIL << "adding job " << it.first << std::endl;
269 requiredFiles.push_back( zypp::OnMediaLocation ( repoInfo().path() + it.first, 1 )
270 .setChecksum( it.second )
271 .setDeltafile( search_deltafile( deltaDir(), it.first ) ));
272 }
273
274 for( const auto &it : _repoindex->signingKeys ) {
275 MIL << "adding job " << it.first << std::endl;
276 zypp::OnMediaLocation location( repoInfo().path() + it.first, 1 );
277 location.setChecksum( it.second );
278 requiredFiles.push_back( std::move(location) );
279 }
280
281 } catch ( const zypp::Exception &e ) {
282 ZYPP_CAUGHT( e );
283 return makeReadyResult(expected<DlContextRefType>::error( std::make_exception_ptr(e) ) );
284 } catch ( ... ) {
285 return makeReadyResult(expected<DlContextRefType>::error( std::current_exception() ) );
286 }
287
288 // add the required files to the base steps
289 if ( _progressObserver ) _progressObserver->setBaseSteps ( _progressObserver->baseSteps () + requiredFiles.size() );
290
291 return transform_collect ( std::move(requiredFiles), [this]( zypp::OnMediaLocation file ) {
292
293 return DownloadWorkflow::provideToCacheDir( _ctx, _mediaHandle, file.filename(), ProvideFileSpec(file) )
294 | inspect ( incProgress( _progressObserver ) );
295
296 }) | and_then ( [this]( std::vector<zypp::ManagedFile> &&dlFiles ) {
297 auto &downloadedFiles = _ctx->files();
298 downloadedFiles.insert( downloadedFiles.end(), std::make_move_iterator(dlFiles.begin()), std::make_move_iterator(dlFiles.end()) );
299 return expected<DlContextRefType>::success( std::move(_ctx) );
300 });
301 });
302
303 } | finishProgress( _progressObserver );
304 }
305
306 private:
307
308 const zypp::RepoInfo &repoInfo() const {
309 return _ctx->repoInfo();
310 }
311
312 const zypp::filesystem::Pathname &deltaDir() const {
313 return _ctx->deltaDir();
314 }
315
317 return _ctx->zyppContext()->config();
318 }
319
320 DlContextRefType _ctx;
321 zypp::parser::susetags::RepoIndex_Ptr _repoindex;
322 MediaHandle _mediaHandle;
323 ProgressObserverRef _progressObserver;
324 };
325 }
326
327 AsyncOpRef<expected<repo::AsyncDownloadContextRef> > download(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle, ProgressObserverRef progressObserver)
328 {
329 return SimpleExecutor< DlLogic, AsyncOp<expected<repo::AsyncDownloadContextRef>> >::run( std::move(dl), std::move(mediaHandle), std::move(progressObserver) );
330 }
331
332 expected<repo::SyncDownloadContextRef> download(repo::SyncDownloadContextRef dl, SyncMediaHandle mediaHandle, ProgressObserverRef progressObserver)
333 {
334 return SimpleExecutor< DlLogic, SyncOp<expected<repo::SyncDownloadContextRef>> >::run( std::move(dl), std::move(mediaHandle), std::move(progressObserver) );
335 }
336
337}
Pathname deltafile
Architecture.
Definition Arch.h:37
Reference counted access to a Tp object calling a custom Dispose function when the last AutoDispose h...
Definition AutoDispose.h:95
Base class for Exception.
Definition Exception.h:147
'Language[_Country]' codes.
Definition Locale.h:51
Describes a resource file located on a medium.
const Pathname & filename() const
The path to the resource on the medium.
OnMediaLocation & setChecksum(CheckSum val_r)
Set the checksum.
OnMediaLocation & setDeltafile(Pathname path)
Set the deltafile.
What is known about a repository.
Definition RepoInfo.h:72
Track changing files or directories.
Definition RepoStatus.h:41
bool empty() const
Whether the status is empty (empty checksum)
Interim helper class to collect global options and settings.
Definition ZConfig.h:64
Wrapper class for stat/lstat.
Definition PathInfo.h:222
std::string basename() const
Return the last component of this path.
Definition Pathname.h:130
Parse repoindex part from a content file.
virtual void parse(const InputStream &imput_r, const ProgressData::ReceiverFnc &fnc_r=ProgressData::ReceiverFnc())
Parse the stream.
void setRepoIndexConsumer(const RepoIndexConsumer &fnc_r)
Consumer to call when repo index was parsed.
Regular expression.
Definition Regex.h:95
Regular expression match result.
Definition Regex.h:168
A ProvideRes object is a reference counted ownership of a resource in the cache provided by a Provide...
Definition provideres.h:36
static expected success(ConsParams &&...params)
Definition expected.h:115
#define ZYPP_ENABLE_LOGIC_BASE(Executor, OpType)
Definition Arch.h:364
typename conditional< B, T, F >::type conditional_t
Definition TypeTraits.h:39
bool hasPrefix(const C_Str &str_r, const C_Str &prefix_r)
Return whether str_r has prefix prefix_r.
Definition String.h:1026
bool endsWith(const C_Str &str_r, const C_Str &prefix_r)
alias for hasSuffix
Definition String.h:1091
bool regex_match(const std::string &s, smatch &matches, const regex &regex)
\relates regex \ingroup ZYPP_STR_REGEX \relates regex \ingroup ZYPP_STR_REGEX
Definition Regex.h:70
unsigned split(const C_Str &line_r, TOutputIterator result_r, const C_Str &sepchars_r=" \t", const Trim trim_r=NO_TRIM)
Split line_r into words.
Definition String.h:531
AsyncOpRef< expected< zypp::ManagedFile > > provideToCacheDir(AsyncCacheProviderContextRef cacheContext, ProvideMediaHandle medium, zypp::Pathname file, ProvideFileSpec filespec)
AsyncOpRef< expected< repo::AsyncDownloadContextRef > > downloadMasterIndex(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle, zypp::filesystem::Pathname masterIndex_r)
auto downloadMediaInfo(MediaHandle &&mediaHandle, const zypp::filesystem::Pathname &destdir)
Download workflow namespace for SUSETags (YaST2) repositories Encapsulates all the knowledge of which...
Definition susetags.cc:25
AsyncOpRef< expected< repo::AsyncDownloadContextRef > > download(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle, ProgressObserverRef progressObserver)
Definition susetags.cc:327
AsyncOpRef< expected< zypp::RepoStatus > > repoStatus(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle)
Definition susetags.cc:81
auto transform_collect(Transformation &&f)
Definition expected.h:637
auto and_then(Fun &&function)
Definition expected.h:554
auto finishProgress(ProgressObserverRef progressObserver)
auto incProgress(ProgressObserverRef progressObserver, double progrIncrease=1.0, std::optional< std::string > newStr={})
auto inspect(Fun &&function)
Definition expected.h:568
std::conditional_t< isAsync, AsyncOpRef< T >, T > makeReadyResult(T &&result)
Definition asyncop.h:297
static expected< std::decay_t< Type >, Err > make_expected_success(Type &&t)
Definition expected.h:341
MediaHandle _mediaHandle
Definition rpmmd.cc:163
DlContextRefType _ctx
Definition rpmmd.cc:66
MediaHandle _handle
Definition rpmmd.cc:67
ProgressObserverRef _progressObserver
Definition rpmmd.cc:164
zypp::parser::susetags::RepoIndex_Ptr _repoindex
Definition susetags.cc:321
#define for_(IT, BEG, END)
Convenient for-loops using iterator.
Definition Easy.h:28
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Definition Exception.h:437
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition Exception.h:429
#define MIL
Definition Logger.h:98
#define WAR
Definition Logger.h:99