<commit_before>"""Source files.

All source files must be contained in groups.  Groups do not nest.
"""
from __future__ import with_statement
import os
import gen.path
import re

_VALID_NAME = re.compile('^[A-Za-z0-9](?:[ -_A-Za-z0-9]*[A-Za-z0-9])$')
def valid_name(x):
    return _VALID_NAME.match(x) and '  ' not in x

_NONSIMPLE_CHAR = re.compile('[^-A-Za-z0-9]+')
def simple_name(x):
    return _NONSIMPLE_CHAR.sub('_', x).strip('-').lower()

_IS_ATOM = re.compile('^[_A-Za-z][_A-Za-z0-9]*$')
def is_atom(x):
    return bool(_IS_ATOM.match(x))

class Source(object):
    """A source file.

    Has a path, which is relative to the group.  Each source file also
    has a set of 'atoms', which are strings that describe how the
    source file is to be compiled.

    You should not construct a Source object directly, instead call
    the 'add' method on a Group object.
    """
    __slots__ = ['_path', '_atoms', '_group', '_sourcetype']

    def __init__(self, path, atoms, group):
        if not isinstance(path, gen.path.Path):
            raise TypeError('path must be Path instance')
        if not isinstance(atoms, tuple):
            raise TypeError('atoms must be tuple')
        if not isinstance(group, Group):
            raise TypeError('group must be Group instance')
        ext = path.ext
        if not ext:
            raise ValueError(
                'source file has no extension, cannot determine type: %s' %
                (path.posix,))
        try:
            stype = gen.path.EXTS[ext]
        except KeyError:
            raise ValueError(
                'source file %s has unknown extension, cannot determine type' %
                (path.posix,))
        self._path = path
        self._atoms = atoms
        self._group = group
        self._sourcetype = stype

    def __repr__(self):
        if self._atoms:
            return '<source %s %r atoms=%s>' % \
                (self._group.name, self._path.posix,
                 ', '.join(self._atoms))
        else:
            return '<source %s %r>' % \
                (self._group.name, self._path.posix)

    @property
    def group(self):
        """The group which this file belongs to."""
        return self._group

    @property
    def grouppath(self):
        """The path of the file, relative to the group."""
        return self._path

    @property
    def relpath(self):
        """The path of the file, relative to the top level."""
        return self._group.relpath / self._path

    @property
    def atoms(self):
        """The atoms which apply to this file, a tuple."""
        return self._atoms

    @property
    def sourcetype(self):
        """The type of this source file."""
        return self._sourcetype

class Group(object):
    """A group of source files.

    Each group has a root path, a name, and a list of sources.
    The name has two variants, the full version (which can have
    spaces) and the simple version (which can't).
    """
    __slots__ = ['_name', '_sname', '_path', '_sources', '_paths']

    def __init__(self, name, path):
        sname = simple_name(name)
        if not valid_name(name) or not sname:
            raise ValueError('invalid group name: %r' % (name,))
        if not isinstance(path, gen.path.Path):
            raise TypeError('group path must be Path')
        self._name = name
        self._sname = sname
        self._path = path
        self._sources = []
        self._paths = set()

    def add(self, path, atoms):
        """Add a source file to the group."""
        if path in self._paths:
            raise ValueError('duplicate path in group %s: %s' %
                             (self._name, path.posixpath))
        self._paths.add(path)<commit_msg>build: make sources sortable<commit_after>"""Source files.

All source files must be contained in groups.  Groups do not nest.
"""
from __future__ import with_statement
import os
import gen.path
import re

_VALID_NAME = re.compile('^[A-Za-z0-9](?:[ -_A-Za-z0-9]*[A-Za-z0-9])$')
def valid_name(x):
    return _VALID_NAME.match(x) and '  ' not in x

_NONSIMPLE_CHAR = re.compile('[^-A-Za-z0-9]+')
def simple_name(x):
    return _NONSIMPLE_CHAR.sub('_', x).strip('-').lower()

_IS_ATOM = re.compile('^[_A-Za-z][_A-Za-z0-9]*$')
def is_atom(x):
    return bool(_IS_ATOM.match(x))

class Source(object):
    """A source file.

    Has a path, which is relative to the group.  Each source file also
    has a set of 'atoms', which are strings that describe how the
    source file is to be compiled.

    You should not construct a Source object directly, instead call
    the 'add' method on a Group object.
    """
    __slots__ = ['_path', '_atoms', '_group', '_sourcetype']

    def __init__(self, path, atoms, group):
        if not isinstance(path, gen.path.Path):
            raise TypeError('path must be Path instance')
        if not isinstance(atoms, tuple):
            raise TypeError('atoms must be tuple')
        if not isinstance(group, Group):
            raise TypeError('group must be Group instance')
        ext = path.ext
        if not ext:
            raise ValueError(
                'source file has no extension, cannot determine type: %s' %
                (path.posix,))
        try:
            stype = gen.path.EXTS[ext]
        except KeyError:
            raise ValueError(
                'source file %s has unknown extension, cannot determine type' %
                (path.posix,))
        self._path = path
        self._atoms = atoms
        self._group = group
        self._sourcetype = stype

    def __repr__(self):
        if self._atoms:
            return '<source %s %r atoms=%s>' % \
                (self._group.name, self._path.posix,
                 ', '.join(self._atoms))
        else:
            return '<source %s %r>' % \
                (self._group.name, self._path.posix)

    @property
    def group(self):
        """The group which this file belongs to."""
        return self._group

    @property
    def grouppath(self):
        """The path of the file, relative to the group."""
        return self._path

    @property
    def relpath(self):
        """The path of the file, relative to the top level."""
        return self._group.relpath / self._path

    @property
    def atoms(self):
        """The atoms which apply to this file, a tuple."""
        return self._atoms

    @property
    def sourcetype(self):
        """The type of this source file."""
        return self._sourcetype

    def __cmp__(self, other):
        if not isinstance(other, Source):
            return NotImplemented
        c = cmp(self._group._name, other._group._name)
        if c:
            return c
        return cmp(self._path, other._path)

class Group(object):
    """A group of source files.

    Each group has a root path, a name, and a list of sources.
    The name has two variants, the full version (which can have
    spaces) and the simple version (which can't).
    """
    __slots__ = ['_name', '_sname', '_path', '_sources', '_paths']

    def __init__(self, name, path):
        sname = simple_name(name)
        if not valid_name(name) or not sname:
            raise ValueError('invalid group name: %r' % (name,))
        if not isinstance(path, gen.path.Path):
            raise TypeError('group path must be Path')
        self._name = name
        self._sname = sname
        self._path = path
        self._sources = []
        self._paths = set()

    def add(self, path, atoms):
        """Add a source file to the group."""
        if path in self._paths:
            raise ValueError('duplicate path in group %s: %s' %
                             (self._name, path.posixpath))
        self._paths.add(path)
{'fraction_non_alphanumeric': 0.11702954898911352, 'fraction_numerical': 0.002592016588906169, 'mean_word_length': 2.1331709297604546, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3910273', 'n_tokens_mistral': 2335, 'n_tokens_neox': 2272, 'n_words': 773}