<commit_before>import os

from toydist.core.pkg_objects import \
        Extension
from toydist.core.meta import \
        _set_metadata, _METADATA_FIELDS
from toydist.core.utils import \
        find_package, expand_glob
from toydist.core.descr_parser import \
        parse

class PackageDescription:
    @classmethod
    def __from_data(cls, data, user_flags):
        if not user_flags:
            user_flags = {}
        d = parse(data, user_flags)

        kw = {}
        for k in ['name', 'version', 'summary', 'url', 'author',
                'maintainer', 'maintainer_email', 'license',
                'author_email', 'description', 'platforms',
                'install_requires', 'build_requires', 'download_url',
                'classifiers']:
            # FIXME: consolidate this naming mess
            data_key = k.replace('_', '')
            if not d.has_key(data_key):
                kw[k] = None
            else:
                kw[k] = d[data_key]

        if d.has_key('extrasourcefiles'):
            kw['extra_source_files'] = d['extrasourcefiles']
        else:
            kw['extra_source_files'] = []

        if d.has_key('datafiles'):
            kw['data_files'] = d['datafiles']
        else:
            kw['data_files'] = []

        if d.has_key('library'):
            library = d['library'][""]

            if library.has_key('packages'):
                kw['packages'] = library['packages']
            else:
                kw['packages'] = []

            if library.has_key('modules'):
                kw['py_modules'] = library['modules']
            else:
                kw['py_modules'] = []

            if library.has_key('extension'):
                kw['extensions'] = library['extension']
            else:
                kw['extensions'] = []

            if library.has_key('installdepends'):
                kw['install_requires'] = library['installdepends']
            else:
                kw['install_requires'] = []

        if d.has_key('executables'):
            executables = {}
            for name, value in d["executables"].items():
                executables[name] = value
            kw["executables"] = executables
        else:
            kw["executables"] = {}

        return cls(**kw)

    @classmethod
    def from_string(cls, s, user_flags=None):
        """Create a PackageDescription from a string containing the package
        description."""
        return cls.__from_data(s.splitlines(), user_flags)

    @classmethod
    def from_file(cls, filename, user_flags=None):
        """Create a PackageDescription from a toysetup.info file."""
        info_file = open(filename, 'r')
        try:
            data = info_file.readlines()
            return cls.__from_data(data, user_flags)
        finally:
            info_file.close()

    def __init__(self, name, version=None, summary=None, url=None,
            author=None, author_email=None, maintainer=None,
            maintainer_email=None, license=None, description=None,
            platforms=None, packages=None, py_modules=None, extensions=None,
            install_requires=None, build_requires=None,
            download_url=None, extra_source_files=None, data_files=None,
            classifiers=None, provides=None, obsoletes=None, executables=None):
        # XXX: should we check that we have sequences when required
        # (py_modules, etc...) ?

        # Package metadata
        _args = locals()
        kw = dict([(k, _args[k]) for k in _METADATA_FIELDS if k in _args])
        _set_metadata(self, **kw)

        # Package content
        if not packages:
            self.packages = []
        else:
            self.packages = packages

        if not py_modules:
            self.py_modules = []
        else:
            self.py_modules = py_modules

        if not extensions:
            self.extensions = []
        else:
            self.extensions = extensions

        if not extra_source_files:
            self.extra_source_files = []
        else:
            self.extra_source_files = extra_source_files

        if not data_files:
            self.data_files = {}
        else:
            self.data_files = data_files

        if not executables:
            self.executables = {}
        else:
            self.executables = executables

    @property
    def top_levels(self):
        pkgs = list(set([k.split(".", 1)[0] for k in self.packages]))
        return pkgs

    def to_dict(self):
        """Return a distutils.core.setup compatible dict."""
        d = {'name': self.name,
            'version': self.version,
            'description': self.summary,
            'url': self.url,
            'author': self.author,
            'author_email': self.author_email,
            'maintainer': self.maintainer,
            'maintainer_email': self.maintainer_email,
            'license': self.license,
            'long_description': self.description,
            'platforms': self.platforms,
            'py_modules': self.py_modules,
            'ext_modules': self.extensions,
            'packages': self.packages,
            'install_requires': self.install_requires}

        return d

def file_list(pkg, root_src=""):
    # FIXME: root_src
    files = []
    for entry in pkg.extra_source_files:
        files.extend(expand_glob(entry))

    for p in pkg.packages:
        files.extend(find_package(p, root_src))

    for m in pkg.py_modules:
        files.append(os.path.join(root_src, '%s.py' % m))

    for section in pkg.data_files.values():
        for entry in section.files:
            files.extend([os.path.join(section.srcdir, f) for f in expand_glob(entry)])

    return files<commit_msg>BUG: fix handling of root_src in file_list.<commit_after>import os

from toydist.core.pkg_objects import \
        Extension
from toydist.core.meta import \
        _set_metadata, _METADATA_FIELDS
from toydist.core.utils import \
        find_package, expand_glob
from toydist.core.descr_parser import \
        parse

class PackageDescription:
    @classmethod
    def __from_data(cls, data, user_flags):
        if not user_flags:
            user_flags = {}
        d = parse(data, user_flags)

        kw = {}
        for k in ['name', 'version', 'summary', 'url', 'author',
                'maintainer', 'maintainer_email', 'license',
                'author_email', 'description', 'platforms',
                'install_requires', 'build_requires', 'download_url',
                'classifiers']:
            # FIXME: consolidate this naming mess
            data_key = k.replace('_', '')
            if not d.has_key(data_key):
                kw[k] = None
            else:
                kw[k] = d[data_key]

        if d.has_key('extrasourcefiles'):
            kw['extra_source_files'] = d['extrasourcefiles']
        else:
            kw['extra_source_files'] = []

        if d.has_key('datafiles'):
            kw['data_files'] = d['datafiles']
        else:
            kw['data_files'] = []

        if d.has_key('library'):
            library = d['library'][""]

            if library.has_key('packages'):
                kw['packages'] = library['packages']
            else:
                kw['packages'] = []

            if library.has_key('modules'):
                kw['py_modules'] = library['modules']
            else:
                kw['py_modules'] = []

            if library.has_key('extension'):
                kw['extensions'] = library['extension']
            else:
                kw['extensions'] = []

            if library.has_key('installdepends'):
                kw['install_requires'] = library['installdepends']
            else:
                kw['install_requires'] = []

        if d.has_key('executables'):
            executables = {}
            for name, value in d["executables"].items():
                executables[name] = value
            kw["executables"] = executables
        else:
            kw["executables"] = {}

        return cls(**kw)

    @classmethod
    def from_string(cls, s, user_flags=None):
        """Create a PackageDescription from a string containing the package
        description."""
        return cls.__from_data(s.splitlines(), user_flags)

    @classmethod
    def from_file(cls, filename, user_flags=None):
        """Create a PackageDescription from a toysetup.info file."""
        info_file = open(filename, 'r')
        try:
            data = info_file.readlines()
            return cls.__from_data(data, user_flags)
        finally:
            info_file.close()

    def __init__(self, name, version=None, summary=None, url=None,
            author=None, author_email=None, maintainer=None,
            maintainer_email=None, license=None, description=None,
            platforms=None, packages=None, py_modules=None, extensions=None,
            install_requires=None, build_requires=None,
            download_url=None, extra_source_files=None, data_files=None,
            classifiers=None, provides=None, obsoletes=None, executables=None):
        # XXX: should we check that we have sequences when required
        # (py_modules, etc...) ?

        # Package metadata
        _args = locals()
        kw = dict([(k, _args[k]) for k in _METADATA_FIELDS if k in _args])
        _set_metadata(self, **kw)

        # Package content
        if not packages:
            self.packages = []
        else:
            self.packages = packages

        if not py_modules:
            self.py_modules = []
        else:
            self.py_modules = py_modules

        if not extensions:
            self.extensions = []
        else:
            self.extensions = extensions

        if not extra_source_files:
            self.extra_source_files = []
        else:
            self.extra_source_files = extra_source_files

        if not data_files:
            self.data_files = {}
        else:
            self.data_files = data_files

        if not executables:
            self.executables = {}
        else:
            self.executables = executables

    @property
    def top_levels(self):
        pkgs = list(set([k.split(".", 1)[0] for k in self.packages]))
        return pkgs

def file_list(pkg, root_src=""):
    # FIXME: root_src
    files = []
    for entry in pkg.extra_source_files:
        files.extend(expand_glob(entry, root_src))

    for p in pkg.packages:
        files.extend(find_package(p, root_src))

    for m in pkg.py_modules:
        files.append(os.path.join(root_src, '%s.py' % m))

    for section in pkg.data_files.values():
        for entry in section.files:
            files.extend([os.path.join(root_src, section.srcdir, f) 
                          for f in expand_glob(entry, root_src)])

    return files
{'fraction_non_alphanumeric': 0.11317944899478778, 'fraction_numerical': 0.00037230081906180194, 'mean_word_length': 1.9174585935378767, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4085907', 'n_tokens_mistral': 3096, 'n_tokens_neox': 2992, 'n_words': 678}