<commit_before>from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division

import io
import logging
import os
import random
import subprocess
import re
import tempfile
import threading
import shutil
import time

from imagemounter import _util, FILE_SYSTEM_TYPES, VOLUME_SYSTEM_TYPES
from imagemounter.exceptions import CommandNotFoundError, NoMountpointAvailableError, SubsystemError, \
    NoLoopbackAvailableError, UnsupportedFilesystemError, NotMountedError, IncorrectFilesystemError, ArgumentError, \
    ImageMounterError, KeyInvalidError
from imagemounter.volume_system import VolumeSystem

logger = logging.getLogger(__name__)


FILE_SYSTEM_GUIDS = {
    '2AE031AA-0F40-DB11-9590-000C2911D1B8': 'vmfs',
    '8053279D-AD40-DB11-BF97-000C2911D1B8': 'vmkcore-diagnostics',
    '6A898CC3-1DD2-11B2-99A6-080020736631': 'zfs-member',
    'C38C896A-D21D-B211-99A6-080020736631': 'zfs-member',
    '0FC63DAF-8483-4772-8E79-3D69D8477DE4': 'linux',
    'E6D6D379-F507-44C2-A23C-238F2A3DF928': 'lvm',
    '79D3D6E6-07F5-C244-A23C-238F2A3DF928': 'lvm',
    'CA7D7CCB-63ED-4C53-861C-1742536059CC': 'luks'
}


class Volume(object):
    """Information about a volume. Note that every detected volume gets their own Volume object, though it may or may
    not be mounted. This can be seen through the :attr:`mountpoint` attribute -- if it is not set, perhaps the
    :attr:`exception` attribute is set with an exception.
    """

    def __init__(self, disk, parent=None, index="0", size=0, offset=0, flag='alloc', slot=0, fstype="", key="",
                 vstype='', volume_detector='auto'):
        """Creates a Volume object that is not mounted yet.

        Only use arguments as keyword arguments.

        :param disk: the parent disk
        :type disk: :class:`Disk`
        :param parent: the parent volume or disk.
        :param str index: the volume index within its volume system, see the attribute documentation.
        :param int size: the volume size, see the attribute documentation.
        :param int offset: the volume offset, see the attribute documentation.
        :param str flag: the volume flag, see the attribute documentation.
        :param int slot: the volume slot, see the attribute documentation.
        :param str fstype: the fstype you wish to use for this Volume. May be ?<fstype> as a fallback value. If not
                           specified, will be retrieved from the ImageParser instance instead.
        :param str key: the key to use for this Volume.
        :param str vstype: the volume system type to use.
        :param str volume_detector: the volume system detection method to use
        """

        self.parent = parent
        self.disk = disk

        # Should be filled somewhere
        self.size = size
        self.offset = offset
        self.index = index
        self.slot = slot
        self.flag = flag
        self.block_size = self.disk.block_size

        self.volumes = VolumeSystem(parent=self, vstype=vstype, volume_detector=volume_detector)

        self.fstype = fstype
        self._get_fstype_from_parser(fstype)

        if key:
            self.key = key
        elif self.index in self.disk.parser.keys:
            self.key = self.disk.parser.keys[self.index]
        else:
            self.key = ""

        self.info = {}
        self._paths = {}

        self.mountpoint = ""
        self.loopback = ""
        self.was_mounted = False
        self.is_mounted = False

    def __unicode__(self):
        return '{0}:{1}'.format(self.index, self.info.get('fsdescription') or '-')

    def __str__(self):
        return str(self.__unicode__())

    def __getitem__(self, item):
        return self.volumes[item]

    def _get_fstype_from_parser(self, fstype=None):
        """Load fstype information from the parser instance."""
        if fstype:
            self.fstype = fstype
        elif self.index in self.disk.parser.fstypes:
            self.fstype = self.disk.parser.fstypes[self.index]
        elif '*' in self.disk.parser.fstypes:
            self.fstype = self.disk.parser.fstypes['*']
        elif '?' in self.disk.parser.fstypes and self.disk.parser.fstypes['?'] is not None:
            self.fstype = "?" + self.disk.parser.fstypes['?']
        else:
            self.fstype = ""

        if self.fstype in VOLUME_SYSTEM_TYPES:
            self.volumes.vstype = self.fstype
            self.fstype = 'volumesystem'

    def get_description(self, with_size=True, with_index=True):
        """Obtains a generic description of the volume, containing the file system type, index, label and NTFS version.
        If *with_size* is provided, the volume size is also included.
        """

        desc = ''

        if with_size and self.size:
            desc += '{0} '.format(self.get_formatted_size())

        s = self.info.get('statfstype') or self.info.get('fsdescription') or '-'
        if with_index:
            desc += '{1}:{0}'.format(s, self.index)
        else:
            desc += s

        if self.info.get('label'):
            desc += ' {0}'.format(self.info.get('label'))

        if self.info.get('version'):  # NTFS
            desc += ' [{0}]'.format(self.info.get('version'))

        return desc

    def get_formatted_size(self):
        """Obtains the size of the volume in a human-readable format (i.e. in TiBs, GiBs or MiBs)."""

        if self.size is not None:
            if self.size < 1024:
                return "{0} B".format(self.size)
            elif self.size < 1024 ** 2:
                return "{0} KiB".format(round(self.size / 1024, 2))
            elif self.size < 1024 ** 3:
                return "{0} MiB".format(round(self.size / 1024 ** 2, 2))
            elif self.size < 1024 ** 4:
                return "{0} GiB".format(round(self.size / 1024 ** 3, 2))
            else:
                return "{0} TiB".format(round(self.size / 1024 ** 4, 2))
        else:
            return self.size

    def _get_blkid_type(self):
        """Retrieves the FS type from the blkid command."""
        try:
            result = _util.check_output_(['blkid', '-p', '-O', str(self.offset), self.get_raw_path()])
            if not result:
                return None

            # noinspection PyTypeChecker
            blkid_result = dict(re.findall(r'([A-Z]+)="(.+?)"', result))

            self.info['blkid_data'] = blkid_result

            if 'PTTYPE' in blkid_result and 'TYPE' not in blkid_result:
                return blkid_result.get('PTTYPE')
            else:
                return blkid_result.get('TYPE')

        except Exception:
            return None  # returning None is better here, since we do not care about the exception in determine_fs_type

    def _get_magic_type(self):
        """Checks the volume for its magic bytes and returns the magic."""

        with io.open(self.disk.get_fs_path(), "rb") as file:
            file.seek(self.offset)
            fheader = file.read(min(self.size, 4096) if self.size else 4096)

        # TODO fallback to img-cat image -s blocknum | file -
        # if we were able to load the module magic
        try:
            # noinspection PyUnresolvedReferences
            import magic

            if hasattr(magic, 'from_buffer'):
                # using https://github.com/ahupp/python-magic
                logger.debug("Using python-magic Python package for file type magic")
                result = magic.from_buffer(fheader).decode()
                self.info['magic_data'] = result
                return result

            elif hasattr(magic, 'open'):
                # using Magic file extensions by <NAME> (Ubuntu python-magic module)
                logger.debug("Using python-magic system package for file type magic")
                ms = magic.open(magic.NONE)
                ms.load()
                result = ms.buffer(fheader)
                ms.close()
                self.info['magic_data'] = result
                return result

            else:
                logger.warning("The python-magic module is not available, but another module named magic was found.")

        except ImportError:
            logger.warning("The python-magic module is not available.")
        except AttributeError:
            logger.warning("The python-magic module is not available, but another module named magic was found.")
        return None  # returning None is better here, since we do not care about the exception in determine_fs_type

    def get_raw_path(self, include_self=False):
        """Retrieves the base mount path of the volume. Typically equals to :func:`Disk.get_fs_path` but may also be the
        path to a logical volume. This is used to determine the source path for a mount call.

        The value returned is normally based on the parent's paths, e.g. if this volume is mounted to a more specific
        path, only its children return the more specific path, this volume itself will keep returning the same path.
        This makes for consistent use of the offset attribute. If you do not need this behaviour, you can override this
        with the include_self argument.

        This behavior, however, is not retained for paths that directly affect the volume itself, not the child volumes.
        This includes VSS stores and LV volumes.
        """

        v = self
        if not include_self:
            # lv / vss_store are exceptions, as it covers the volume itself, not the child volume
            if v._paths.get('lv'):
                return v._paths['lv']
            elif v._paths.get('vss_store'):
                return v._paths['vss_store']
            elif v.parent and v.parent != self.disk:
                v = v.parent
            else:
                return self.disk.get_fs_path()

        while True:
            if v._paths.get('lv'):
                return v._paths['lv']
            elif v._paths.get('bde'):
                return v._paths['bde'] + '/bde1'
            elif v._paths.get('luks'):
                return '/dev/mapper/' + v._paths['luks']
            elif v._paths.get('md'):
                return v._paths['md']
            elif v._paths.get('vss_store'):
                return v._paths['vss_store']

            # Only if the volume has a parent that is not a disk, we try to check the parent for a location.
            if v.parent and v.parent != self.disk:
                v = v.parent
            else:
                break
        return self.disk.get_fs_path()

    def get_safe_label(self):
        """Returns a label that is safe to add to a path in the mountpoint for this volume."""

        if self.info.get('label') == '/':
            return 'root'

        suffix = re.sub(r"[/ \(\)]+", "_", self.info.get('label')) if self.info.get('label') else ""
        if suffix and suffix[0] == '_':
            suffix = suffix[1:]
        if len(suffix) > 2 and suffix[-1] == '_':
            suffix = suffix[:-1]
        return suffix

    def carve(self, freespace=True):
        """Call this method to carve the free space of the volume for (deleted) files. Note that photorec has its
        own interface that temporarily takes over the shell.

        :param freespace: indicates whether the entire volume should be carved (False) or only the free space (True)
        :type freespace: bool
        :return: string to the path where carved data is available
        :raises CommandNotFoundError: if the underlying command does not exist
        :raises SubsystemError: if the underlying command fails
        :raises NoMountpointAvailableError: if there is no mountpoint available
        :raises NoLoopbackAvailableError: if there is no loopback available (only when volume has no slot number)
        """

        if not _util.command_exists('photorec'):
            logger.warning("photorec is not installed, could not carve volume")
            raise CommandNotFoundError("photorec")

        self._make_mountpoint(var_name='carve', suffix="carve", in_paths=True)

        # if no slot, we need to make a loopback that we can use to carve the volume
        loopback_was_created_for_carving = False
        if not self.slot:
            if not self.loopback:
                self._find_loopback()
                #Can't carve if volume has no slot number and can't be mounted on loopback.
                loopback_was_created_for_carving = True

            # noinspection PyBroadException
            try:
                _util.check_call_(["photorec", "/d", self._paths['carve'] + os.sep, "/cmd", self.loopback,
                                  ("freespace," if freespace else "") + "search"])

                # clean out the loop device if we created it specifically for carving
                if loopback_was_created_for_carving:
                    # noinspection PyBroadException
                    try:
                        _util.check_call_(['losetup', '-d', self.loopback])
                    except Exception:
                        pass
                    else:
                        self.loopback = ""

                return self._paths['carve']
            except Exception as e:
                logger.exception("Failed carving the volume.")
                raise SubsystemError(e)
        else:
            # noinspection PyBroadException
            try:
                _util.check_call_(["photorec", "/d", self._paths['carve'] + os.sep, "/cmd", self.get_raw_path(),
                                  str(self.slot) + (",freespace" if freespace else "") + ",search"])
                return self._paths['carve']

            except Exception as e:
                logger.exception("Failed carving the volume.")
                raise SubsystemError(e)

    def detect_volume_shadow_copies(self):
        """Method to call vshadowmount and mount NTFS volume shadow copies.

        :return: iterable with the :class:`Volume` objects of the VSS
        :raises CommandNotFoundError: if the underlying command does not exist
        :raises SubSystemError: if the underlying command fails
        :raises NoMountpointAvailableError: if there is no mountpoint available
        """

        if not _util.command_exists('vshadowmount'):
            logger.warning("vshadowmount is not installed, could not mount volume shadow copies")
            raise CommandNotFoundError('vshadowmount')

        self._make_mountpoint(var_name='vss', suffix="vss", in_paths=True)

        try:
            _util.check_call_(["vshadowmount", "-o", str(self.offset), self.get_raw_path(), self._paths['vss']])
        except Exception as e:
            logger.exception("Failed mounting the volume shadow copies.")
            raise SubsystemError(e)
        else:
            return self.volumes.detect_volumes(vstype='vss')

    def _should_mount(self, only_mount=None):
        """Indicates whether this volume should be mounted. Internal method, used by imount.py"""

        return only_mount is None or \
            self.index in only_mount or \
            self.info.get('lastmountpoint') in only_mount or \
            self.info.get('label') in only_mount

    def init(self, only_mount=None, swallow_exceptions=True):
        """Generator that mounts this volume and either yields itself or recursively generates its subvolumes.

        More specifically, this function will call :func:`load_fsstat_data` (iff *no_stats* is False), followed by
        :func:`mount`, followed by a call to :func:`detect_mountpoint`, after which ``self`` is yielded, or the result
        of the :func:`init` call on each subvolume is yielded

        :param only_mount: if specified, only volume indexes in this list are mounted. Volume indexes are strings.
        :param swallow_exceptions: if True, any error occuring when mounting the volume is swallowed and added as an
            exception attribute to the yielded objects.
        """
        if swallow_exceptions:
            self.exception = None

        try:
            if not self._should_mount(only_mount):
                yield self
                return

            if not self.init_volume():
                yield self
                return

        except ImageMounterError as e:
            if swallow_exceptions:
                self.exception = e
            else:
                raise

        if not self.volumes:
            yield self
        else:
            for v in self.volumes:
                for s in v.init(only_mount, swallow_exceptions):
                    yield s

    def init_volume(self, fstype=None):
        """Initializes a single volume. You should use this method instead of :func:`mount` if you want some sane checks
        before mounting.
        """

        logger.debug("Initializing volume {0}".format(self))

        if not self._should_mount():
            return False

        if self.flag != 'alloc':
            return False

        if self.info.get('raid_status') == 'waiting':
            logger.info("RAID array %s not ready for mounting", self)
            return False

        if self.is_mounted:
            logger.info("%s is currently mounted, not mounting it again", self)
            return False

        logger.info("Mounting volume {0}".format(self))
        self.mount(fstype=fstype)
        self.detect_mountpoint()

        return True

    def _make_mountpoint(self, casename=None, var_name='mountpoint', suffix='', in_paths=False):
        """Creates a directory that can be used as a mountpoint. The directory is stored in :attr:`mountpoint`,
        or the varname as specified by the argument. If in_paths is True, the path is stored in the :attr:`_paths`
        attribute instead.

        :returns: the mountpoint path
        :raises NoMountpointAvailableError: if no mountpoint could be made
        """
        parser = self.disk.parser

        if parser.mountdir and not os.path.exists(parser.mountdir):
            os.makedirs(parser.mountdir)

        if parser.pretty:
            md = parser.mountdir or tempfile.gettempdir()
            case_name = casename or self.disk.parser.casename or \
                        ".".join(os.path.basename(self.disk.paths[0]).split('.')[0:-1]) or \
                        os.path.basename(self.disk.paths[0])
            if self.disk.parser.casename == case_name:  # the casename is already in the path in this case
                pretty_label = "{0}-{1}".format(self.index, self.get_safe_label() or self.fstype or 'volume')
            else:
                pretty_label = "{0}-{1}-{2}".format(case_name, self.index,
                                                    self.get_safe_label() or self.fstype or 'volume')
            if suffix:
                pretty_label += "-" + suffix
            path = os.path.join(md, pretty_label)

            # check if path already exists, otherwise try to find another nice path
            if os.path.exists(path):
                for i in range(2, 100):
                    path = os.path.join(md, pretty_label + "-" + str(i))
                    if not os.path.exists(path):
                        break
                else:
                    logger.error("Could not find free mountdir.")
                    raise NoMountpointAvailableError()

            # noinspection PyBroadException
            try:
                os.mkdir(path, 777)
                if in_paths:
                    self._paths[var_name] = path
                else:
                    setattr(self, var_name, path)
                return path
            except Exception:
                logger.exception("Could not create mountdir.")
                raise NoMountpointAvailableError()
        else:
            t = tempfile.mkdtemp(prefix='im_' + self.index + '_',
                                 suffix='_' + self.get_safe_label() + ("_" + suffix if suffix else ""),
                                 dir=parser.mountdir)
            if in_paths:
                self._paths[var_name] = t
            else:
                setattr(self, var_name, t)
            return t

    def _find_loopback(self, use_loopback=True, var_name='loopback'):
        """Finds a free loopback device that can be used. The loopback is stored in :attr:`loopback`. If *use_loopback*
        is True, the loopback will also be used directly.

        :returns: the loopback address
        :raises NoLoopbackAvailableError: if no loopback could be found
        """

        # noinspection PyBroadException
        try:
            loopback = _util.check_output_(['losetup', '-f']).strip()
            setattr(self, var_name, loopback)
        except Exception:
            logger.warning("No free loopback device found.", exc_info=True)
            raise NoLoopbackAvailableError()

        # noinspection PyBroadException
        if use_loopback:
            try:
                cmd = ['losetup', '-o', str(self.offset), '--sizelimit', str(self.size),
                       loopback, self.get_raw_path()]
                if not self.disk.read_write:
                    cmd.insert(1, '-r')
                _util.check_call_(cmd, stdout=subprocess.PIPE)
            except Exception:
                logger.exception("Loopback device could not be mounted.")
                raise NoLoopbackAvailableError()
        return loopback

    def _free_loopback(self, var_name='loopback'):
        if getattr(self, var_name):
            _util.check_call_(['losetup', '-d', getattr(self, var_name)], wrap_error=True)
            setattr(self, var_name, "")

    def determine_fs_type(self):
        """Determines the FS type for this partition. This function is used internally to determine which mount system
        to use, based on the file system description. Return values include *ext*, *ufs*, *ntfs*, *lvm* and *luks*.

        Note: does not do anything if fstype is already set to something sensible.
        """

        fstype_fallback = self.fstype[1:] if self.fstype and self.fstype.startswith("?") else ""

        # Determine fs type. If forced, always use provided type.
        if self.fstype in FILE_SYSTEM_TYPES:
            pass  # already correctly set
        elif self.fstype in VOLUME_SYSTEM_TYPES:
            self.volumes.vstype = self.fstype
            self.fstype = 'volumesystem'
        else:
            last_resort = None  # use this if we can't determine the FS type more reliably
            # we have two possible sources for determining the FS type: the description given to us by the detection
            # method, and the type given to us by the stat function
            for fsdesc in (self.info.get('fsdescription'), self.info.get('guid'),
                           self._get_blkid_type, self.info.get('statfstype'), self._get_magic_type):
                # For efficiency reasons, not all functions are called instantly.
                if callable(fsdesc):
                    fsdesc = fsdesc()
                logger.debug("Trying to determine fs type from '{}'".format(fsdesc))
                if not fsdesc:
                    continue
                fsdesc = fsdesc.lower()

                # for the purposes of this function, logical volume is nothing, and 'primary' is rather useless info
                if fsdesc in ('logical volume', 'luks volume', 'bde volume', 'raid volume',
                              'primary', 'basic data partition', 'vss store'):
                    continue

                if fsdesc == 'directory':
                    self.fstype = 'dir'  # dummy fs type
                elif re.search(r'\bext[0-9]*\b', fsdesc):
                    self.fstype = 'ext'
                elif 'bsd' in fsdesc or 'ufs 2' in fsdesc:
                    self.fstype = 'ufs'
                elif '0x07' in fsdesc or 'ntfs' in fsdesc or 'exfat' in fsdesc:
                    # Use blkid to differentiate between ntfs and exfat
                    self.fstype = self._get_blkid_type()
                    logger.info("Detected filesystem type: '{}'".format(self.fstype))
                elif '0x8e' in fsdesc or 'lvm' in fsdesc:
                    self.fstype = 'lvm'
                elif 'hfs+' in fsdesc:
                    self.fstype = 'hfs+'
                elif 'hfs' in fsdesc:
                    self.fstype = 'hfs'
                elif 'luks' in fsdesc:
                    self.fstype = 'luks'
                elif 'fat' in fsdesc or 'efi system partition' in fsdesc:
                    # based on http://en.wikipedia.org/wiki/EFI_System_partition, efi is always fat.
                    self.fstype = 'fat'
                elif 'iso 9660' in fsdesc:
                    self.fstype = 'iso'
                elif 'linux compressed rom file system' in fsdesc or 'cramfs' in fsdesc:
                    self.fstype = 'cramfs'
                elif fsdesc.startswith("sgi xfs") or re.search(r'\bxfs\b', fsdesc):
                    self.fstype = "xfs"
                elif 'swap file' in fsdesc or 'linux swap' in fsdesc or 'linux-swap' in fsdesc:
                    self.fstype = 'swap'
                elif 'squashfs' in fsdesc:<commit_msg>Remove the forced _get_blkid_type() call from the ntfs/exfat condition.<commit_after>from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division

import io
import logging
import os
import random
import subprocess
import re
import tempfile
import threading
import shutil
import time

from imagemounter import _util, FILE_SYSTEM_TYPES, VOLUME_SYSTEM_TYPES
from imagemounter.exceptions import CommandNotFoundError, NoMountpointAvailableError, SubsystemError, \
    NoLoopbackAvailableError, UnsupportedFilesystemError, NotMountedError, IncorrectFilesystemError, ArgumentError, \
    ImageMounterError, KeyInvalidError
from imagemounter.volume_system import VolumeSystem

logger = logging.getLogger(__name__)


FILE_SYSTEM_GUIDS = {
    '2AE031AA-0F40-DB11-9590-000C2911D1B8': 'vmfs',
    '8053279D-AD40-DB11-BF97-000C2911D1B8': 'vmkcore-diagnostics',
    '6A898CC3-1DD2-11B2-99A6-080020736631': 'zfs-member',
    'C38C896A-D21D-B211-99A6-080020736631': 'zfs-member',
    '0FC63DAF-8483-4772-8E79-3D69D8477DE4': 'linux',
    'E6D6D379-F507-44C2-A23C-238F2A3DF928': 'lvm',
    '79D3D6E6-07F5-C244-A23C-238F2A3DF928': 'lvm',
    'CA7D7CCB-63ED-4C53-861C-1742536059CC': 'luks'
}


class Volume(object):
    """Information about a volume. Note that every detected volume gets their own Volume object, though it may or may
    not be mounted. This can be seen through the :attr:`mountpoint` attribute -- if it is not set, perhaps the
    :attr:`exception` attribute is set with an exception.
    """

    def __init__(self, disk, parent=None, index="0", size=0, offset=0, flag='alloc', slot=0, fstype="", key="",
                 vstype='', volume_detector='auto'):
        """Creates a Volume object that is not mounted yet.

        Only use arguments as keyword arguments.

        :param disk: the parent disk
        :type disk: :class:`Disk`
        :param parent: the parent volume or disk.
        :param str index: the volume index within its volume system, see the attribute documentation.
        :param int size: the volume size, see the attribute documentation.
        :param int offset: the volume offset, see the attribute documentation.
        :param str flag: the volume flag, see the attribute documentation.
        :param int slot: the volume slot, see the attribute documentation.
        :param str fstype: the fstype you wish to use for this Volume. May be ?<fstype> as a fallback value. If not
                           specified, will be retrieved from the ImageParser instance instead.
        :param str key: the key to use for this Volume.
        :param str vstype: the volume system type to use.
        :param str volume_detector: the volume system detection method to use
        """

        self.parent = parent
        self.disk = disk

        # Should be filled somewhere
        self.size = size
        self.offset = offset
        self.index = index
        self.slot = slot
        self.flag = flag
        self.block_size = self.disk.block_size

        self.volumes = VolumeSystem(parent=self, vstype=vstype, volume_detector=volume_detector)

        self.fstype = fstype
        self._get_fstype_from_parser(fstype)

        if key:
            self.key = key
        elif self.index in self.disk.parser.keys:
            self.key = self.disk.parser.keys[self.index]
        else:
            self.key = ""

        self.info = {}
        self._paths = {}

        self.mountpoint = ""
        self.loopback = ""
        self.was_mounted = False
        self.is_mounted = False

    def __unicode__(self):
        return '{0}:{1}'.format(self.index, self.info.get('fsdescription') or '-')

    def __str__(self):
        return str(self.__unicode__())

    def __getitem__(self, item):
        return self.volumes[item]

    def _get_fstype_from_parser(self, fstype=None):
        """Load fstype information from the parser instance."""
        if fstype:
            self.fstype = fstype
        elif self.index in self.disk.parser.fstypes:
            self.fstype = self.disk.parser.fstypes[self.index]
        elif '*' in self.disk.parser.fstypes:
            self.fstype = self.disk.parser.fstypes['*']
        elif '?' in self.disk.parser.fstypes and self.disk.parser.fstypes['?'] is not None:
            self.fstype = "?" + self.disk.parser.fstypes['?']
        else:
            self.fstype = ""

        if self.fstype in VOLUME_SYSTEM_TYPES:
            self.volumes.vstype = self.fstype
            self.fstype = 'volumesystem'

    def get_description(self, with_size=True, with_index=True):
        """Obtains a generic description of the volume, containing the file system type, index, label and NTFS version.
        If *with_size* is provided, the volume size is also included.
        """

        desc = ''

        if with_size and self.size:
            desc += '{0} '.format(self.get_formatted_size())

        s = self.info.get('statfstype') or self.info.get('fsdescription') or '-'
        if with_index:
            desc += '{1}:{0}'.format(s, self.index)
        else:
            desc += s

        if self.info.get('label'):
            desc += ' {0}'.format(self.info.get('label'))

        if self.info.get('version'):  # NTFS
            desc += ' [{0}]'.format(self.info.get('version'))

        return desc

    def get_formatted_size(self):
        """Obtains the size of the volume in a human-readable format (i.e. in TiBs, GiBs or MiBs)."""

        if self.size is not None:
            if self.size < 1024:
                return "{0} B".format(self.size)
            elif self.size < 1024 ** 2:
                return "{0} KiB".format(round(self.size / 1024, 2))
            elif self.size < 1024 ** 3:
                return "{0} MiB".format(round(self.size / 1024 ** 2, 2))
            elif self.size < 1024 ** 4:
                return "{0} GiB".format(round(self.size / 1024 ** 3, 2))
            else:
                return "{0} TiB".format(round(self.size / 1024 ** 4, 2))
        else:
            return self.size

    def _get_blkid_type(self):
        """Retrieves the FS type from the blkid command."""
        try:
            result = _util.check_output_(['blkid', '-p', '-O', str(self.offset), self.get_raw_path()])
            if not result:
                return None

            # noinspection PyTypeChecker
            blkid_result = dict(re.findall(r'([A-Z]+)="(.+?)"', result))

            self.info['blkid_data'] = blkid_result

            if 'PTTYPE' in blkid_result and 'TYPE' not in blkid_result:
                return blkid_result.get('PTTYPE')
            else:
                return blkid_result.get('TYPE')

        except Exception:
            return None  # returning None is better here, since we do not care about the exception in determine_fs_type

    def _get_magic_type(self):
        """Checks the volume for its magic bytes and returns the magic."""

        with io.open(self.disk.get_fs_path(), "rb") as file:
            file.seek(self.offset)
            fheader = file.read(min(self.size, 4096) if self.size else 4096)

        # TODO fallback to img-cat image -s blocknum | file -
        # if we were able to load the module magic
        try:
            # noinspection PyUnresolvedReferences
            import magic

            if hasattr(magic, 'from_buffer'):
                # using https://github.com/ahupp/python-magic
                logger.debug("Using python-magic Python package for file type magic")
                result = magic.from_buffer(fheader).decode()
                self.info['magic_data'] = result
                return result

            elif hasattr(magic, 'open'):
                # using Magic file extensions by <NAME> (Ubuntu python-magic module)
                logger.debug("Using python-magic system package for file type magic")
                ms = magic.open(magic.NONE)
                ms.load()
                result = ms.buffer(fheader)
                ms.close()
                self.info['magic_data'] = result
                return result

            else:
                logger.warning("The python-magic module is not available, but another module named magic was found.")

        except ImportError:
            logger.warning("The python-magic module is not available.")
        except AttributeError:
            logger.warning("The python-magic module is not available, but another module named magic was found.")
        return None  # returning None is better here, since we do not care about the exception in determine_fs_type

    def get_raw_path(self, include_self=False):
        """Retrieves the base mount path of the volume. Typically equals to :func:`Disk.get_fs_path` but may also be the
        path to a logical volume. This is used to determine the source path for a mount call.

        The value returned is normally based on the parent's paths, e.g. if this volume is mounted to a more specific
        path, only its children return the more specific path, this volume itself will keep returning the same path.
        This makes for consistent use of the offset attribute. If you do not need this behaviour, you can override this
        with the include_self argument.

        This behavior, however, is not retained for paths that directly affect the volume itself, not the child volumes.
        This includes VSS stores and LV volumes.
        """

        v = self
        if not include_self:
            # lv / vss_store are exceptions, as it covers the volume itself, not the child volume
            if v._paths.get('lv'):
                return v._paths['lv']
            elif v._paths.get('vss_store'):
                return v._paths['vss_store']
            elif v.parent and v.parent != self.disk:
                v = v.parent
            else:
                return self.disk.get_fs_path()

        while True:
            if v._paths.get('lv'):
                return v._paths['lv']
            elif v._paths.get('bde'):
                return v._paths['bde'] + '/bde1'
            elif v._paths.get('luks'):
                return '/dev/mapper/' + v._paths['luks']
            elif v._paths.get('md'):
                return v._paths['md']
            elif v._paths.get('vss_store'):
                return v._paths['vss_store']

            # Only if the volume has a parent that is not a disk, we try to check the parent for a location.
            if v.parent and v.parent != self.disk:
                v = v.parent
            else:
                break
        return self.disk.get_fs_path()

    def get_safe_label(self):
        """Returns a label that is safe to add to a path in the mountpoint for this volume."""

        if self.info.get('label') == '/':
            return 'root'

        suffix = re.sub(r"[/ \(\)]+", "_", self.info.get('label')) if self.info.get('label') else ""
        if suffix and suffix[0] == '_':
            suffix = suffix[1:]
        if len(suffix) > 2 and suffix[-1] == '_':
            suffix = suffix[:-1]
        return suffix

    def carve(self, freespace=True):
        """Call this method to carve the free space of the volume for (deleted) files. Note that photorec has its
        own interface that temporarily takes over the shell.

        :param freespace: indicates whether the entire volume should be carved (False) or only the free space (True)
        :type freespace: bool
        :return: string to the path where carved data is available
        :raises CommandNotFoundError: if the underlying command does not exist
        :raises SubsystemError: if the underlying command fails
        :raises NoMountpointAvailableError: if there is no mountpoint available
        :raises NoLoopbackAvailableError: if there is no loopback available (only when volume has no slot number)
        """

        if not _util.command_exists('photorec'):
            logger.warning("photorec is not installed, could not carve volume")
            raise CommandNotFoundError("photorec")

        self._make_mountpoint(var_name='carve', suffix="carve", in_paths=True)

        # if no slot, we need to make a loopback that we can use to carve the volume
        loopback_was_created_for_carving = False
        if not self.slot:
            if not self.loopback:
                self._find_loopback()
                #Can't carve if volume has no slot number and can't be mounted on loopback.
                loopback_was_created_for_carving = True

            # noinspection PyBroadException
            try:
                _util.check_call_(["photorec", "/d", self._paths['carve'] + os.sep, "/cmd", self.loopback,
                                  ("freespace," if freespace else "") + "search"])

                # clean out the loop device if we created it specifically for carving
                if loopback_was_created_for_carving:
                    # noinspection PyBroadException
                    try:
                        _util.check_call_(['losetup', '-d', self.loopback])
                    except Exception:
                        pass
                    else:
                        self.loopback = ""

                return self._paths['carve']
            except Exception as e:
                logger.exception("Failed carving the volume.")
                raise SubsystemError(e)
        else:
            # noinspection PyBroadException
            try:
                _util.check_call_(["photorec", "/d", self._paths['carve'] + os.sep, "/cmd", self.get_raw_path(),
                                  str(self.slot) + (",freespace" if freespace else "") + ",search"])
                return self._paths['carve']

            except Exception as e:
                logger.exception("Failed carving the volume.")
                raise SubsystemError(e)

    def detect_volume_shadow_copies(self):
        """Method to call vshadowmount and mount NTFS volume shadow copies.

        :return: iterable with the :class:`Volume` objects of the VSS
        :raises CommandNotFoundError: if the underlying command does not exist
        :raises SubSystemError: if the underlying command fails
        :raises NoMountpointAvailableError: if there is no mountpoint available
        """

        if not _util.command_exists('vshadowmount'):
            logger.warning("vshadowmount is not installed, could not mount volume shadow copies")
            raise CommandNotFoundError('vshadowmount')

        self._make_mountpoint(var_name='vss', suffix="vss", in_paths=True)

        try:
            _util.check_call_(["vshadowmount", "-o", str(self.offset), self.get_raw_path(), self._paths['vss']])
        except Exception as e:
            logger.exception("Failed mounting the volume shadow copies.")
            raise SubsystemError(e)
        else:
            return self.volumes.detect_volumes(vstype='vss')

    def _should_mount(self, only_mount=None):
        """Indicates whether this volume should be mounted. Internal method, used by imount.py"""

        return only_mount is None or \
            self.index in only_mount or \
            self.info.get('lastmountpoint') in only_mount or \
            self.info.get('label') in only_mount

    def init(self, only_mount=None, swallow_exceptions=True):
        """Generator that mounts this volume and either yields itself or recursively generates its subvolumes.

        More specifically, this function will call :func:`load_fsstat_data` (iff *no_stats* is False), followed by
        :func:`mount`, followed by a call to :func:`detect_mountpoint`, after which ``self`` is yielded, or the result
        of the :func:`init` call on each subvolume is yielded

        :param only_mount: if specified, only volume indexes in this list are mounted. Volume indexes are strings.
        :param swallow_exceptions: if True, any error occuring when mounting the volume is swallowed and added as an
            exception attribute to the yielded objects.
        """
        if swallow_exceptions:
            self.exception = None

        try:
            if not self._should_mount(only_mount):
                yield self
                return

            if not self.init_volume():
                yield self
                return

        except ImageMounterError as e:
            if swallow_exceptions:
                self.exception = e
            else:
                raise

        if not self.volumes:
            yield self
        else:
            for v in self.volumes:
                for s in v.init(only_mount, swallow_exceptions):
                    yield s

    def init_volume(self, fstype=None):
        """Initializes a single volume. You should use this method instead of :func:`mount` if you want some sane checks
        before mounting.
        """

        logger.debug("Initializing volume {0}".format(self))

        if not self._should_mount():
            return False

        if self.flag != 'alloc':
            return False

        if self.info.get('raid_status') == 'waiting':
            logger.info("RAID array %s not ready for mounting", self)
            return False

        if self.is_mounted:
            logger.info("%s is currently mounted, not mounting it again", self)
            return False

        logger.info("Mounting volume {0}".format(self))
        self.mount(fstype=fstype)
        self.detect_mountpoint()

        return True

    def _make_mountpoint(self, casename=None, var_name='mountpoint', suffix='', in_paths=False):
        """Creates a directory that can be used as a mountpoint. The directory is stored in :attr:`mountpoint`,
        or the varname as specified by the argument. If in_paths is True, the path is stored in the :attr:`_paths`
        attribute instead.

        :returns: the mountpoint path
        :raises NoMountpointAvailableError: if no mountpoint could be made
        """
        parser = self.disk.parser

        if parser.mountdir and not os.path.exists(parser.mountdir):
            os.makedirs(parser.mountdir)

        if parser.pretty:
            md = parser.mountdir or tempfile.gettempdir()
            case_name = casename or self.disk.parser.casename or \
                        ".".join(os.path.basename(self.disk.paths[0]).split('.')[0:-1]) or \
                        os.path.basename(self.disk.paths[0])
            if self.disk.parser.casename == case_name:  # the casename is already in the path in this case
                pretty_label = "{0}-{1}".format(self.index, self.get_safe_label() or self.fstype or 'volume')
            else:
                pretty_label = "{0}-{1}-{2}".format(case_name, self.index,
                                                    self.get_safe_label() or self.fstype or 'volume')
            if suffix:
                pretty_label += "-" + suffix
            path = os.path.join(md, pretty_label)

            # check if path already exists, otherwise try to find another nice path
            if os.path.exists(path):
                for i in range(2, 100):
                    path = os.path.join(md, pretty_label + "-" + str(i))
                    if not os.path.exists(path):
                        break
                else:
                    logger.error("Could not find free mountdir.")
                    raise NoMountpointAvailableError()

            # noinspection PyBroadException
            try:
                os.mkdir(path, 777)
                if in_paths:
                    self._paths[var_name] = path
                else:
                    setattr(self, var_name, path)
                return path
            except Exception:
                logger.exception("Could not create mountdir.")
                raise NoMountpointAvailableError()
        else:
            t = tempfile.mkdtemp(prefix='im_' + self.index + '_',
                                 suffix='_' + self.get_safe_label() + ("_" + suffix if suffix else ""),
                                 dir=parser.mountdir)
            if in_paths:
                self._paths[var_name] = t
            else:
                setattr(self, var_name, t)
            return t

    def _find_loopback(self, use_loopback=True, var_name='loopback'):
        """Finds a free loopback device that can be used. The loopback is stored in :attr:`loopback`. If *use_loopback*
        is True, the loopback will also be used directly.

        :returns: the loopback address
        :raises NoLoopbackAvailableError: if no loopback could be found
        """

        # noinspection PyBroadException
        try:
            loopback = _util.check_output_(['losetup', '-f']).strip()
            setattr(self, var_name, loopback)
        except Exception:
            logger.warning("No free loopback device found.", exc_info=True)
            raise NoLoopbackAvailableError()

        # noinspection PyBroadException
        if use_loopback:
            try:
                cmd = ['losetup', '-o', str(self.offset), '--sizelimit', str(self.size),
                       loopback, self.get_raw_path()]
                if not self.disk.read_write:
                    cmd.insert(1, '-r')
                _util.check_call_(cmd, stdout=subprocess.PIPE)
            except Exception:
                logger.exception("Loopback device could not be mounted.")
                raise NoLoopbackAvailableError()
        return loopback

    def _free_loopback(self, var_name='loopback'):
        if getattr(self, var_name):
            _util.check_call_(['losetup', '-d', getattr(self, var_name)], wrap_error=True)
            setattr(self, var_name, "")

    def determine_fs_type(self):
        """Determines the FS type for this partition. This function is used internally to determine which mount system
        to use, based on the file system description. Return values include *ext*, *ufs*, *ntfs*, *lvm* and *luks*.

        Note: does not do anything if fstype is already set to something sensible.
        """

        fstype_fallback = self.fstype[1:] if self.fstype and self.fstype.startswith("?") else ""

        # Determine fs type. If forced, always use provided type.
        if self.fstype in FILE_SYSTEM_TYPES:
            pass  # already correctly set
        elif self.fstype in VOLUME_SYSTEM_TYPES:
            self.volumes.vstype = self.fstype
            self.fstype = 'volumesystem'
        else:
            last_resort = None  # use this if we can't determine the FS type more reliably
            # we have two possible sources for determining the FS type: the description given to us by the detection
            # method, and the type given to us by the stat function
            for fsdesc in (self.info.get('fsdescription'), self.info.get('guid'),
                           self._get_blkid_type, self.info.get('statfstype'), self._get_magic_type):
                # For efficiency reasons, not all functions are called instantly.
                if callable(fsdesc):
                    fsdesc = fsdesc()
                logger.debug("Trying to determine fs type from '{}'".format(fsdesc))
                if not fsdesc:
                    continue
                fsdesc = fsdesc.lower()

                # for the purposes of this function, logical volume is nothing, and 'primary' is rather useless info
                if fsdesc in ('logical volume', 'luks volume', 'bde volume', 'raid volume',
                              'primary', 'basic data partition', 'vss store'):
                    continue

                if fsdesc == 'directory':
                    self.fstype = 'dir'  # dummy fs type
                elif re.search(r'\bext[0-9]*\b', fsdesc):
                    self.fstype = 'ext'
                elif 'bsd' in fsdesc or 'ufs 2' in fsdesc:
                    self.fstype = 'ufs'
                elif '0x07' in fsdesc or 'ntfs / exfat' in fsdesc:
                    last_resort = 'ntfs'
                    continue
                elif 'ntfs' in fsdesc:
                    self.fstype = 'ntfs'
                elif 'exfat' in fsdesc:
                    self.fstype = 'exfat'
                elif '0x8e' in fsdesc or 'lvm' in fsdesc:
                    self.fstype = 'lvm'
                elif 'hfs+' in fsdesc:
                    self.fstype = 'hfs+'
                elif 'hfs' in fsdesc:
                    self.fstype = 'hfs'
                elif 'luks' in fsdesc:
                    self.fstype = 'luks'
                elif 'fat' in fsdesc or 'efi system partition' in fsdesc:
                    # based on http://en.wikipedia.org/wiki/EFI_System_partition, efi is always fat.
                    self.fstype = 'fat'
                elif 'iso 9660' in fsdesc:
                    self.fstype = 'iso'
                elif 'linux compressed rom file system' in fsdesc or 'cramfs' in fsdesc:
                    self.fstype = 'cramfs'
                elif fsdesc.startswith("sgi xfs") or re.search(r'\bxfs\b', fsdesc):
                    self.fstype = "xfs"
                elif 'swap file' in fsdesc or 'linux swap' in fsdesc or 'linux-swap' in fsdesc:
                    self.fstype = 'swap'
                elif 'squashfs' in fsdesc:
{'fraction_non_alphanumeric': 0.09337265372813135, 'fraction_numerical': 0.011238193918622724, 'mean_word_length': 1.9402425449645557, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1512786', 'n_tokens_mistral': 14386, 'n_tokens_neox': 13508, 'n_words': 4369}