source: calamares/trunk/fuentes/src/modules/unpackfs/main.py @ 7538

Last change on this file since 7538 was 7538, checked in by kbut, 17 months ago

sync with github

File size: 10.2 KB
Line 
1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3#
4# === This file is part of Calamares - <https://github.com/calamares> ===
5#
6#   Copyright 2014, Teo Mrnjavac <teo@kde.org>
7#   Copyright 2014, Daniel Hillenbrand <codeworkx@bbqlinux.org>
8#   Copyright 2014, Philip Müller <philm@manjaro.org>
9#   Copyright 2017, Alf Gaida <agaida@siduction.org>
10#
11#   Calamares is free software: you can redistribute it and/or modify
12#   it under the terms of the GNU General Public License as published by
13#   the Free Software Foundation, either version 3 of the License, or
14#   (at your option) any later version.
15#
16#   Calamares is distributed in the hope that it will be useful,
17#   but WITHOUT ANY WARRANTY; without even the implied warranty of
18#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19#   GNU General Public License for more details.
20#
21#   You should have received a copy of the GNU General Public License
22#   along with Calamares. If not, see <http://www.gnu.org/licenses/>.
23
24import os
25import re
26import shutil
27import subprocess
28import sys
29import tempfile
30
31from libcalamares import *
32
33
34class UnpackEntry:
35    """
36    Extraction routine using rsync.
37
38    :param source:
39    :param sourcefs:
40    :param destination:
41    """
42    __slots__ = ['source', 'sourcefs', 'destination', 'copied', 'total']
43
44    def __init__(self, source, sourcefs, destination):
45        self.source = source
46        self.sourcefs = sourcefs
47        self.destination = destination
48        self.copied = 0
49        self.total = 0
50
51
52ON_POSIX = 'posix' in sys.builtin_module_names
53
54
55def list_excludes(destination):
56    """
57    List excludes for rsync.
58
59    :param destination:
60    :return:
61    """
62    lst = []
63    extra_mounts = globalstorage.value("extraMounts")
64
65    for extra_mount in extra_mounts:
66        mount_point = extra_mount["mountPoint"]
67
68        if mount_point:
69            lst.extend(['--exclude', mount_point + '/'])
70
71    return lst
72
73
74def file_copy(source, dest, progress_cb):
75    """
76    Extract given image using rsync.
77
78    :param source:
79    :param dest:
80    :param progress_cb:
81    :return:
82    """
83    # Environment used for executing rsync properly
84    # Setting locale to C (fix issue with tr_TR locale)
85    at_env = os.environ
86    at_env["LC_ALL"] = "C"
87
88    # `source` *must* end with '/' otherwise a directory named after the source
89    # will be created in `dest`: ie if `source` is "/foo/bar" and `dest` is
90    # "/dest", then files will be copied in "/dest/bar".
91    source += "/"
92
93    args = ['rsync', '-aHAXr']
94    args.extend(list_excludes(dest))
95    args.extend(['--progress', source, dest])
96    process = subprocess.Popen(
97        args, env=at_env, bufsize=1, stdout=subprocess.PIPE, close_fds=ON_POSIX
98        )
99
100    for line in iter(process.stdout.readline, b''):
101        # small comment on this regexp.
102        # rsync outputs three parameters in the progress.
103        # xfer#x => i try to interpret it as 'file copy try no. x'
104        # to-check=x/y, where:
105        #  - x = number of files yet to be checked
106        #  - y = currently calculated total number of files.
107        # but if you're copying directory with some links in it, the xfer#
108        # might not be a reliable counter (for one increase of xfer, many
109        # files may be created).
110        # In case of manjaro, we pre-compute the total number of files.
111        # therefore we can easily subtract x from y in order to get real files
112        # copied / processed count.
113        m = re.findall(r'xfr#(\d+), ir-chk=(\d+)/(\d+)', line.decode())
114
115        if m:
116            # we've got a percentage update
117            num_files_remaining = int(m[0][1])
118            num_files_total_local = int(m[0][2])
119            # adjusting the offset so that progressbar can be continuesly drawn
120            num_files_copied = num_files_total_local - num_files_remaining
121
122            # I guess we're updating every 100 files...
123            if num_files_copied % 100 == 0:
124                progress_cb(num_files_copied)
125
126    process.wait()
127
128    # 23 is the return code rsync returns if it cannot write extended
129    # attributes (with -X) because the target file system does not support it,
130    # e.g., the FAT EFI system partition. We need -X because distributions
131    # using file system capabilities and/or SELinux require the extended
132    # attributes. But distributions using SELinux may also have SELinux labels
133    # set on files under /boot/efi, and rsync complains about those. The only
134    # clean way would be to split the rsync into one with -X and
135    # --exclude /boot/efi and a separate one without -X for /boot/efi, but only
136    # if /boot/efi is actually an EFI system partition. For now, this hack will
137    # have to do. See also:
138    # https://bugzilla.redhat.com/show_bug.cgi?id=868755#c50
139    # for the same issue in Anaconda, which uses a similar workaround.
140    if process.returncode != 0 and process.returncode != 23:
141        return "rsync failed with error code {}.".format(process.returncode)
142
143    return None
144
145
146class UnpackOperation:
147    """
148    Extraction routine using unsquashfs.
149
150    :param entries:
151    """
152
153    def __init__(self, entries):
154        self.entries = entries
155        self.entry_for_source = dict((x.source, x) for x in self.entries)
156
157    def report_progress(self):
158        """
159        Pass progress to user interface
160        """
161        progress = float(0)
162
163        for entry in self.entries:
164            if entry.total == 0:
165                continue
166
167            partialprogress = 0.05  # Having a total !=0 gives 5%
168
169            partialprogress += 0.95 * (entry.copied / float(entry.total))
170            progress += partialprogress / len(self.entries)
171
172        job.setprogress(progress)
173
174    def run(self):
175        """
176        Extract given image using unsquashfs.
177
178        :return:
179        """
180        source_mount_path = tempfile.mkdtemp()
181
182        try:
183            for entry in self.entries:
184                imgbasename = os.path.splitext(
185                    os.path.basename(entry.source))[0]
186                imgmountdir = os.path.join(source_mount_path, imgbasename)
187                os.mkdir(imgmountdir)
188
189                self.mount_image(entry, imgmountdir)
190
191                fslist = ""
192
193                if entry.sourcefs == "squashfs":
194                    if shutil.which("unsquashfs") is None:
195                        msg = ("Failed to find unsquashfs, make sure you have "
196                               "the squashfs-tools package installed")
197                        print(msg)
198                        return ("Failed to unpack image",
199                                msg)
200
201                    fslist = subprocess.check_output(
202                        ["unsquashfs", "-l", entry.source]
203                        )
204
205                if entry.sourcefs == "ext4":
206                    fslist = subprocess.check_output(
207                        ["find", imgmountdir, "-type", "f"]
208                        )
209
210                entry.total = len(fslist.splitlines())
211
212                self.report_progress()
213                error_msg = self.unpack_image(entry, imgmountdir)
214
215                if error_msg:
216                    return ("Failed to unpack image {}".format(entry.source),
217                            error_msg)
218
219            return None
220        finally:
221            shutil.rmtree(source_mount_path, ignore_errors=True, onerror=None)
222
223    def mount_image(self, entry, imgmountdir):
224        """
225        Mount given image as loop device.
226
227        :param entry:
228        :param imgmountdir:
229        """
230        if os.path.isdir(entry.source):
231            subprocess.check_call(["mount",
232                                   "--bind", entry.source,
233                                   imgmountdir])
234        else:
235            subprocess.check_call(["mount",
236                                   entry.source,
237                                   imgmountdir,
238                                   "-t", entry.sourcefs,
239                                   "-o", "loop"
240                                   ])
241
242    def unpack_image(self, entry, imgmountdir):
243        """
244        Unpacks image.
245
246        :param entry:
247        :param imgmountdir:
248        :return:
249        """
250        def progress_cb(copied):
251            """ Copies file to given destination target.
252
253            :param copied:
254            """
255            entry.copied = copied
256            self.report_progress()
257
258        try:
259            return file_copy(imgmountdir, entry.destination, progress_cb)
260        finally:
261            subprocess.check_call(["umount", "-l", imgmountdir])
262
263
264def run():
265    """
266    Unsquash filesystem.
267    """
268    PATH_PROCFS = '/proc/filesystems'
269
270    root_mount_point = globalstorage.value("rootMountPoint")
271
272    if not root_mount_point:
273        return ("No mount point for root partition in globalstorage",
274                "globalstorage does not contain a \"rootMountPoint\" key, "
275                "doing nothing")
276
277    if not os.path.exists(root_mount_point):
278        return ("Bad mount point for root partition in globalstorage",
279                "globalstorage[\"rootMountPoint\"] is \"{}\", which does not "
280                "exist, doing nothing".format(root_mount_point))
281
282    unpack = list()
283
284    for entry in job.configuration["unpack"]:
285        source = os.path.abspath(entry["source"])
286
287        sourcefs = entry["sourcefs"]
288
289        # Get supported filesystems
290        fs_is_supported = False
291
292        if os.path.isfile(PATH_PROCFS) and os.access(PATH_PROCFS, os.R_OK):
293            with open(PATH_PROCFS, 'r') as procfile:
294                filesystems = procfile.read()
295                filesystems = filesystems.replace(
296                    "nodev", "").replace("\t", "").splitlines()
297
298                # Check if the source filesystem is supported
299                for fs in filesystems:
300                    if fs == sourcefs:
301                        fs_is_supported = True
302
303        if not fs_is_supported:
304            return "Bad filesystem", "sourcefs=\"{}\"".format(sourcefs)
305
306        destination = os.path.abspath(root_mount_point + entry["destination"])
307
308        if not os.path.exists(source):
309            return "Bad source", "source=\"{}\"".format(source)
310
311        if not os.path.isdir(destination):
312            return "Bad destination", "destination=\"{}\"".format(destination)
313
314        unpack.append(UnpackEntry(source, sourcefs, destination))
315
316    unpackop = UnpackOperation(unpack)
317
318    return unpackop.run()
Note: See TracBrowser for help on using the repository browser.