<commit_before>import logging
import os
import re
import shutil
import subprocess
import sys

from .constants import drivetypes
from kolibri.utils.android import on_android

logger = logging.getLogger(__name__)

# Regex parser for the output of `mount` on OSX, which contains rows that looks like:
#  /dev/disk1s1 on /Volumes/HP v125w (msdos, local, nodev, nosuid, noowners)
OSX_MOUNT_PARSER = re.compile(
    r"^(?P<device>\S+) on (?P<path>.+) \((?P<filesystem>[^, ]+)", flags=re.MULTILINE
)

# Regex parser for the output of `mount` on Linux, which contains rows that looks like:
#  /dev/sdb2 on /media/user/KEEPOD type ext4 (rw,nosuid,nodev,uhelper=udisks2)
LINUX_MOUNT_PARSER = re.compile(
    r"^(?P<device>\S+) on (?P<path>.+) type (?P<filesystem>\S+)", flags=re.MULTILINE
)

# Regex parser for the output of 'mount' on Android, which contains rows that looks like:
#  /dev/block/bootdevice/by-name/userdata /data ext4 rw,seclabel,nosuid,nodev,noatime,noauto_da_alloc,data=ordered 0 0
# Note that access to /proc/ is restricted in later versions of Android. Will break the app.
RAW_MOUNT_PARSER = re.compile(
    r"^(?P<device>\S+) (?P<path>\S+) (?P<filesystem>\S+)", flags=re.MULTILINE
)


FILESYSTEM_BLACKLIST = set(
    [
        "anon_inodefs",
        "bdev",
        "binfmt_misc",
        "cgroup",
        "cpuset",
        "debugfs",
        "devpts",
        "devtmpfs",
        "ecryptfs",
        "fuse",
        "fuse.gvfsd-fuse",
        "fusectl",
        "hugetlbfs",
        "mqueue",
        "nfs",
        "nfs4",
        "nfsd",
        "pipefs",
        "proc",
        "pstore",
        "ramfs",
        "rootfs",
        "rpc_pipefs",
        "securityfs",
        "sockfs",
        "sysfs",
        "tmpfs",
        "cgmfs",
    ]
)

# These paths can be mounted as separate drives/partitions,
# so they should not be shown in the list of import/export drives.
PATH_PREFIX_BLACKLIST = ["/proc", "/sys", "/tmp", "/var", "/boot", "/dev"]


def get_drive_list():
    """
    Gets a list of drives and metadata by parsing the output of `mount`, and adding additional info from various commands.
    Disk size/usage comes from shutil.disk_usage or os.statvfs, and name/type info from dbus (Linux) or diskutil (OSX).
    """

    if sys.platform == "darwin":
        MOUNT_PARSER = OSX_MOUNT_PARSER
    elif on_android():
        MOUNT_PARSER = RAW_MOUNT_PARSER
    else:
        MOUNT_PARSER = LINUX_MOUNT_PARSER

    try:
        drivelist = subprocess.Popen("mount", shell=True, stdout=subprocess.PIPE)
        drivelisto, err = drivelist.communicate()
    except OSError:  # couldn't run `mount`, let's try reading the /etc/mounts listing directly
        with open("/proc/mounts") as f:
            drivelisto = f.read()
        MOUNT_PARSER = RAW_MOUNT_PARSER

    drives = []

    for drivematch in MOUNT_PARSER.finditer(drivelisto.decode()):

        drive = drivematch.groupdict()<commit_msg>Fix for mount point detection on Android with latest toolchain.<commit_after>import logging
import os
import re
import shutil
import subprocess
import sys

from .constants import drivetypes
from kolibri.utils.android import on_android

logger = logging.getLogger(__name__)

# Regex parser for the output of `mount` on OSX, which contains rows that looks like:
#  /dev/disk1s1 on /Volumes/HP v125w (msdos, local, nodev, nosuid, noowners)
OSX_MOUNT_PARSER = re.compile(
    r"^(?P<device>\S+) on (?P<path>.+) \((?P<filesystem>[^, ]+)", flags=re.MULTILINE
)

# Regex parser for the output of `mount` on Linux, which contains rows that looks like:
#  /dev/sdb2 on /media/user/KEEPOD type ext4 (rw,nosuid,nodev,uhelper=udisks2)
LINUX_MOUNT_PARSER = re.compile(
    r"^(?P<device>\S+) on (?P<path>.+) type (?P<filesystem>\S+)", flags=re.MULTILINE
)

# Regex parser for the output of 'mount' on Android, which contains rows that looks like:
#  /dev/block/bootdevice/by-name/userdata /data ext4 rw,seclabel,nosuid,nodev,noatime,noauto_da_alloc,data=ordered 0 0
# Note that access to /proc/ is restricted in later versions of Android. Will break the app.
RAW_MOUNT_PARSER = re.compile(
    r"^(?P<device>\S+) (?P<path>\S+) (?P<filesystem>\S+)", flags=re.MULTILINE
)


FILESYSTEM_BLACKLIST = set(
    [
        "anon_inodefs",
        "bdev",
        "binfmt_misc",
        "cgroup",
        "cpuset",
        "debugfs",
        "devpts",
        "devtmpfs",
        "ecryptfs",
        "fuse",
        "fuse.gvfsd-fuse",
        "fusectl",
        "hugetlbfs",
        "mqueue",
        "nfs",
        "nfs4",
        "nfsd",
        "pipefs",
        "proc",
        "pstore",
        "ramfs",
        "rootfs",
        "rpc_pipefs",
        "securityfs",
        "sockfs",
        "sysfs",
        "tmpfs",
        "cgmfs",
    ]
)

# These paths can be mounted as separate drives/partitions,
# so they should not be shown in the list of import/export drives.
PATH_PREFIX_BLACKLIST = ["/proc", "/sys", "/tmp", "/var", "/boot", "/dev"]


def get_drive_list():
    """
    Gets a list of drives and metadata by parsing the output of `mount`, and adding additional info from various commands.
    Disk size/usage comes from shutil.disk_usage or os.statvfs, and name/type info from dbus (Linux) or diskutil (OSX).
    """

    if sys.platform == "darwin":
        MOUNT_PARSER = OSX_MOUNT_PARSER
    else:
        MOUNT_PARSER = LINUX_MOUNT_PARSER

    try:
        drivelist = subprocess.Popen("mount", shell=True, stdout=subprocess.PIPE)
        drivelisto, err = drivelist.communicate()
        # Some Android devices at least now use the LINUX_MOUNT_PARSER format.
        # Try it and revert to RAW_MOUNT_PARSER if we can't find any matches with it.
        if on_android() and not MOUNT_PARSER.match(drivelisto.decode()):
            MOUNT_PARSER = RAW_MOUNT_PARSER
    except OSError:  # couldn't run `mount`, let's try reading the /etc/mounts listing directly
        with open("/proc/mounts") as f:
            drivelisto = f.read()
        MOUNT_PARSER = RAW_MOUNT_PARSER

    drives = []

    for drivematch in MOUNT_PARSER.finditer(drivelisto.decode()):

        drive = drivematch.groupdict()
{'fraction_non_alphanumeric': 0.1251838535708449, 'fraction_numerical': 0.003922209511358065, 'mean_word_length': 2.8881829733163915, 'pattern_counts': {'":': 2, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3010942', 'n_tokens_mistral': 2155, 'n_tokens_neox': 2125, 'n_words': 569}