<commit_before>import platform
import subprocess


class Package(object):
    def __init__(self, name, version, files=[], packfiles=True):
        self.name = name
        self.version = version
        self.files = list(files)

    def add_file(self, filename):
        self.files.append(filename)


magic_dirs = ('/dev', '/proc', '/sys')
system_dirs = ('/bin', '/etc', '/lib', '/sbin', '/usr', '/var')


class DpkgManager(object):
    def __init__(self):
        self.unknown_files = []
        self.packages = {}
        self.package_files = {}

    def search_for_file(self, f):
        # Special files
        if any(f.path.startswith(c) for c in magic_dirs):
            return

        # If it's not in a system directory, no need to look for it
        if (f.path.startswith('/usr/local') or
                not any(f.path.startswith(c) for c in system_dirs)):
            self.unknown_files.append(f)
            return

        # Looks in our cache
        if f.path in self.package_files:
            pkgname = self.package_files[f.path]
        else:
            pkgname = self._get_package_for_file(f.path)
            self.package_files[f.path] = pkgname

        # Stores the file
        if pkgname is None:
            self.unknown_files.append(f)
        else:
            if pkgname in self.packages:
                self.packages[pkgname].add_file(f)
            else:
                self._create_package(pkgname, [f])

    def _get_package_for_file(self, filename):
        p = subprocess.Popen(['dpkg', '-S', filename], stdout=subprocess.PIPE)
        try:
            for l in p.stdout:
                pkgname, f = l.split(': ', 1)
                f = f.strip()
                self.package_files[f] = pkgname
                if f == filename:
                    return pkgname
        finally:
            p.wait()
        assert p.returncode == 0

    def _create_package(self, pkgname, files):
        p = subprocess.Popen(['dpkg', '-l', pkgname], stdout=subprocess.PIPE)
        try:
            version = None
            for l in p.stdout:
                if not l.startswith('ii'):
                    continue
                fields = l.split()
                if fields[1] == pkgname:
                    version = fields[2]
                    break
        finally:
            p.wait()
        assert p.returncode == 0
        pkg = Package(pkgname, version, files)
        self.packages[pkgname] = pkg
        return pkg


def identify_packages(files):
    """Organizes the files, using the distribution's package manager.
    """
    distribution = platform.linux_distribution()[0].lower()
    if distribution == 'ubuntu':
        manager = DpkgManager()
    elif distribution == 'debian':
        manager = DpkgManager()
    else:
        return files, []

    for f in files:
        manager.search_for_file(f)

    return manager.unknown_files, manager.packages.values()
<commit_msg>Restores Python 3 compatibility<commit_after>import platform
import subprocess


class Package(object):
    def __init__(self, name, version, files=[], packfiles=True):
        self.name = name
        self.version = version
        self.files = list(files)

    def add_file(self, filename):
        self.files.append(filename)


magic_dirs = ('/dev', '/proc', '/sys')
system_dirs = ('/bin', '/etc', '/lib', '/sbin', '/usr', '/var')


class DpkgManager(object):
    def __init__(self):
        self.unknown_files = []
        self.packages = {}
        self.package_files = {}

    def search_for_file(self, f):
        # Special files
        if any(f.path.startswith(c) for c in magic_dirs):
            return

        # If it's not in a system directory, no need to look for it
        if (f.path.startswith('/usr/local') or
                not any(f.path.startswith(c) for c in system_dirs)):
            self.unknown_files.append(f)
            return

        # Looks in our cache
        if f.path in self.package_files:
            pkgname = self.package_files[f.path]
        else:
            pkgname = self._get_package_for_file(f.path)
            self.package_files[f.path] = pkgname

        # Stores the file
        if pkgname is None:
            self.unknown_files.append(f)
        else:
            if pkgname in self.packages:
                self.packages[pkgname].add_file(f)
            else:
                self._create_package(pkgname, [f])

    def _get_package_for_file(self, filename):
        p = subprocess.Popen(['dpkg', '-S', filename], stdout=subprocess.PIPE)
        try:
            for l in p.stdout:
                pkgname, f = l.split(b': ', 1)
                pkgname, f = pkgname.decode('ascii'), f.strip().decode('ascii')
                self.package_files[f] = pkgname
                if f == filename:
                    return pkgname
        finally:
            p.wait()
        return None

    def _create_package(self, pkgname, files):
        p = subprocess.Popen(['dpkg', '-l', pkgname], stdout=subprocess.PIPE)
        try:
            version = None
            for l in p.stdout:
                if not l.startswith(b'ii'):
                    continue
                fields = l.split()
                if fields[1].decode('ascii') == pkgname:
                    version = str(fields[2])
                    break
        finally:
            p.wait()
        assert p.returncode == 0
        pkg = Package(pkgname, version, files)
        self.packages[pkgname] = pkg
        return pkg


def identify_packages(files):
    """Organizes the files, using the distribution's package manager.
    """
    distribution = platform.linux_distribution()[0].lower()
    if distribution == 'ubuntu':
        manager = DpkgManager()
    elif distribution == 'debian':
        manager = DpkgManager()
    else:
        return files, []

    for f in files:
        manager.search_for_file(f)

    return manager.unknown_files, manager.packages.values()

{'fraction_non_alphanumeric': 0.11365935919055649, 'fraction_numerical': 0.002023608768971332, 'mean_word_length': 1.8002832861189801, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '774630', 'n_tokens_mistral': 1788, 'n_tokens_neox': 1682, 'n_words': 430}