source: appstream-generator/src/asgen/extractor.d @ 4841

Last change on this file since 4841 was 4841, checked in by Juanma, 2 years ago

Initial release

File size: 9.3 KB
Line 
1/*
2 * Copyright (C) 2016 Matthias Klumpp <matthias@tenstral.net>
3 *
4 * Licensed under the GNU Lesser General Public License Version 3
5 *
6 * This program is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU Lesser General Public License as published by
8 * the Free Software Foundation, either version 3 of the license, or
9 * (at your option) any later version.
10 *
11 * This software is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14 * GNU Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public License
17 * along with this software.  If not, see <http://www.gnu.org/licenses/>.
18 */
19
20module asgen.extractor;
21
22import std.stdio;
23import std.string;
24import std.path : baseName;
25import std.algorithm : canFind;
26import std.typecons : scoped;
27import appstream.Component;
28import appstream.Metadata;
29
30import asgen.config;
31import asgen.hint;
32import asgen.result;
33import asgen.backends.interfaces;
34import asgen.datastore;
35import asgen.handlers;
36import asgen.utils : componentGetStockIcon;
37
38
39final class DataExtractor
40{
41
42private:
43    Component[] cpts;
44    GeneratorHint[] hints;
45
46    DataStore dstore;
47    IconHandler iconh;
48    Config conf;
49    DataType dtype;
50
51public:
52
53    this (DataStore db, IconHandler iconHandler)
54    {
55        dstore = db;
56        iconh = iconHandler;
57        conf = Config.get ();
58        dtype = conf.metadataType;
59    }
60
61    GeneratorResult processPackage (Package pkg)
62    {
63        // create a new result container
64        auto gres = new GeneratorResult (pkg);
65
66        // prepare a list of metadata files which interest us
67        string[string] desktopFiles;
68        string[] metadataFiles;
69        foreach (ref fname; pkg.contents) {
70            if ((fname.startsWith ("/usr/share/applications")) && (fname.endsWith (".desktop"))) {
71                desktopFiles[baseName (fname)] = fname;
72                continue;
73            }
74            if ((fname.startsWith ("/usr/share/metainfo/")) && (fname.endsWith (".xml"))) {
75                metadataFiles ~= fname;
76                continue;
77            }
78            if ((fname.startsWith ("/usr/share/appdata/")) && (fname.endsWith (".xml"))) {
79                metadataFiles ~= fname;
80                continue;
81            }
82        }
83
84        // create new AppStream metadata parser
85        auto mdata = scoped!Metadata ();
86        mdata.setLocale ("ALL");
87        mdata.setFormatStyle (FormatStyle.METAINFO);
88
89        // now process metainfo XML files
90        foreach (ref mfname; metadataFiles) {
91            auto dataBytes = pkg.getFileData (mfname);
92            auto data = cast(string) dataBytes;
93
94            mdata.clearComponents ();
95            auto cpt = parseMetaInfoFile (mdata, gres, data);
96            if (cpt is null)
97                continue;
98
99            // check if we need to extend this component's data with data from its .desktop file
100            auto cid = cpt.getId ();
101            if (cid.empty) {
102                gres.addHint (null, "metainfo-no-id", ["fname": mfname]);
103                continue;
104            }
105
106            // check for legacy path
107            if (mfname.startsWith ("/usr/share/appdata/")) {
108                gres.addHint (null, "legacy-metainfo-directory", ["fname": mfname.baseName]);
109            }
110
111            // we need to add the version to re-download screenshot on every new upload.
112            // otherwise, screenshots would only get updated if the actual metadata file was touched.
113            gres.updateComponentGCID (cpt, pkg.ver);
114
115            // if we have a desktop-application, find the .desktop file
116            // and merge its metadata with the metainfo file data.
117            // having no .desktop file present is a bug.
118            if (cpt.getKind () == ComponentKind.DESKTOP_APP) {
119                auto dfp = cid in desktopFiles;
120                if (dfp is null)
121                    dfp = (cid ~ ".desktop") in desktopFiles;
122                if (dfp is null) {
123                    if (componentGetStockIcon (cpt).isNull) {
124                        // no .desktop file was found and this component does not
125                        // define an icon - this means that a .desktop file is required
126                        // and can not be omitted, so we stop processing here.
127                        // Otherwise we take the data and see how far we get.
128
129                        // finalize GCID checksum and continue
130                        gres.updateComponentGCID (cpt, data);
131
132                        gres.addHint (cpt.getId (), "missing-desktop-file");
133                        // we have a DESKTOP_APP component, but no .desktop file. This is a bug.
134                        continue;
135                    }
136                } else {
137                    // update component with .desktop file data, ignoring NoDisplay field
138                    auto ddataBytes = pkg.getFileData (*dfp);
139                    auto ddata = cast(string) ddataBytes;
140                    parseDesktopFile (gres, *dfp, ddata, true);
141
142                    // update GCID checksum
143                    gres.updateComponentGCID (cpt, ddata);
144
145                    // drop the .desktop file from the list, it has been handled
146                    desktopFiles.remove (cid);
147                }
148            }
149
150            // do a validation of the file. Validation is slow, so we allow
151            // the user to disable this feature.
152            if (conf.featureEnabled (GeneratorFeature.VALIDATE)) {
153                if (!dstore.metadataExists (dtype, gres.gcidForComponent (cpt)))
154                    validateMetaInfoFile (gres, cpt, data);
155            }
156        }
157
158        // process the remaining .desktop files
159        foreach (ref dfname; desktopFiles.byValue ()) {
160            auto ddataBytes = pkg.getFileData (dfname);
161            auto ddata = cast(string) ddataBytes;
162            auto cpt = parseDesktopFile (gres, dfname, ddata, false);
163            if (cpt !is null)
164                gres.updateComponentGCID (cpt, ddata);
165        }
166
167        auto hasFontComponent = false;
168        foreach (ref cpt; gres.getComponents ()) {
169            auto gcid = gres.gcidForComponent (cpt);
170
171            // don't run expensive operations if the metadata already exists
172            auto existingMData = dstore.getMetadata (dtype, gcid);
173            if (existingMData !is null) {
174                // To account for packages which change their package name, we
175                // also need to check if the package this component is associated
176                // with matches ours.
177                // If it doesn't, we can't just link the package to the component.
178                bool samePkg = false;
179                if (dtype == DataType.YAML) {
180                    if (existingMData.canFind (format ("Package: %s\n", pkg.name)))
181                        samePkg = true;
182                } else {
183                    if (existingMData.canFind (format ("<pkgname>%s</pkgname>", pkg.name)))
184                        samePkg = true;
185                }
186
187                if (!samePkg) {
188                    // The exact same metadata exists in a different package already, we emit an error hint.
189                    // ATTENTION: This does not cover the case where *different* metadata (as in, different summary etc.)
190                    // but with the *same ID* exists.
191                    // We only catch that kind of problem later.
192
193                    auto cdata = new Metadata ();
194                    cdata.setFormatStyle (FormatStyle.COLLECTION);
195                    cdata.setFormatVersion (conf.formatVersion);
196
197                    if (dtype == DataType.YAML)
198                        cdata.parse (existingMData, FormatKind.YAML);
199                    else
200                        cdata.parse (existingMData, FormatKind.XML);
201                    auto ecpt = cdata.getComponent ();
202
203                    gres.addHint (cpt.getId (), "metainfo-duplicate-id", ["cid": cpt.getId (), "pkgname": ecpt.getPkgnames ()[0]]);
204                }
205
206                continue;
207            }
208
209            // find & store icons
210            iconh.process (gres, cpt);
211            if (gres.isIgnored (cpt))
212                continue;
213
214            // download and resize screenshots.
215            // we don't even need to call this if no downloads are allowed.
216            if (!conf.featureEnabled (GeneratorFeature.NO_DOWNLOADS))
217                processScreenshots (gres, cpt, dstore.mediaExportPoolDir);
218
219            // we don't want to run expensive font processing if we don't have a font component.
220            // since the font handler needs to load all font data prior to processing the component,
221            // for efficiency we only record whether we need to process fonts here and then handle
222            // them at a later step.
223            // This improves performance for a package that contains multiple font components.
224            if (cpt.getKind () == ComponentKind.FONT)
225                hasFontComponent = true;
226        }
227
228        // render font previews and extract font metadata (if any of the components is a font)
229        if (conf.featureEnabled (GeneratorFeature.PROCESS_FONTS)) {
230            if (hasFontComponent)
231                processFontData (gres, dstore.mediaExportPoolDir);
232        }
233
234        // this removes invalid components and cleans up the result
235        gres.finalize ();
236        pkg.close ();
237
238        return gres;
239    }
240}
Note: See TracBrowser for help on using the repository browser.