<commit_before>from collections import namedtuple
import re


# This module raises this exception when it cannot parse an input sequence.
class ParseError(Exception): pass

# A singleton value used internally to indicate a parse failure.
ParseFailure = object()

# A tuple of (object, int). The object is the parse tree, and the int value
# is the index of the last item consumed by the parser, plus one. (So it's
# the index of the next item that the parser should consume.)
ParseResult = namedtuple('ParseResult', 'value, pos')

# Convenience function.
Regex = re.compile

# Special return value used to control the parser.
ParseStep = namedtuple('ParseStep', 'term, pos')


class ParsingOperand(object):
    '''
    This mixin-style class adds support for parsing operators:
        a & b evaluates to And(a, b).
        a | b evaluates to Or(a, b).
        a / b evaluates to Alt(a, b, allow_trailer=True).
        a // b evaluates to Alt(a, b, allow_trailer=False).
        ~a evaluates to Opt(a).
    '''
    def __and__(self, other): return And(self, other)
    def __rand__(self, other): return And(other, self)
    def __or__(self, other): return Or(self, other)
    def __ror__(self, other): return Or(other, self)
    def __div__(self, other): return Alt(self, other, allow_trailer=True)
    def __rdiv__(self, other): return Alt(other, self, allow_trailer=True)
    def __truediv__(self, other): return Alt(self, other, allow_trailer=True)
    def __rtruediv__(self, other): return Alt(other, self, allow_trailer=True)
    def __floordiv__(self, other): return Alt(self, other, allow_trailer=False)
    def __rfloordiv__(self, other): return Alt(other, self, allow_trailer=False)
    def __invert__(self): return Opt(self)


# These classes add the "&" and "|" parsing operators.
class TermMetaClass(type, ParsingOperand): pass
class Term(ParsingOperand): __metaclass__ = TermMetaClass


class SimpleTerm(Term):
    '''Abstract base class for terms that consist of a single subterm.'''
    def __init__(self, term):
        self.term = term


class Backtrack(Term):
    '''
    Moves the current position back by some number of spaces. If the new
    position would be less than zero, then it fails and has no other effect.
    '''
    def __init__(self, count=1):
        self.count = count

    def parse(self, source, pos):
        dst = pos - self.count
        yield ParseFailure if dst < 0 else ParseResult(None, dst)


def Alt(term, separator, allow_trailer=True):
    '''
    Parses a list of terms separated by a separator. Returns the elements
    in a normal list, and drops the separators.
    '''
    rest = List(Right(separator, term))
    tail = Opt(separator) if allow_trailer else None
    triple = (term, rest, tail)
    return Transform(Opt(triple), lambda t: [t[0]] + t[1] if t else [])


def And(left, right):
    return Left(left, Expect(right))


class Any(Term):
    '''
    Returns the next element of the input. Fails if the remaining input is
    empty. This class can be used as a term directly, or it can be
    instantiated.
    '''
    @staticmethod
    def parse(source, pos):
        yield (ParseFailure if pos >= len(source)
            else ParseResult(source[pos], pos + 1))


def AnyInst(*cls):
    return Where(lambda x: isinstance(x, cls))


class Bind(Term):
    def __init__(self, term, continuation):
        self.term = term
        self.continuation = continuation

    def parse(self, source, pos):
        arg = yield ParseStep(self.term, pos)
        if arg is ParseFailure:<commit_msg>Start moving classes together, to the top of the module.<commit_after>from collections import namedtuple
import re


# This module raises this exception when it cannot parse an input sequence.
class ParseError(Exception): pass

# A singleton value used internally to indicate a parse failure.
ParseFailure = object()

# A tuple of (object, int). The object is the parse tree, and the int value
# is the index of the last item consumed by the parser, plus one. (So it's
# the index of the next item that the parser should consume.)
ParseResult = namedtuple('ParseResult', 'value, pos')

# Convenience function.
Regex = re.compile

# Special return value used to control the parser.
ParseStep = namedtuple('ParseStep', 'term, pos')


class ParsingOperand(object):
    '''
    This mixin-style class adds support for parsing operators:
        a & b evaluates to And(a, b).
        a | b evaluates to Or(a, b).
        a / b evaluates to Alt(a, b, allow_trailer=True).
        a // b evaluates to Alt(a, b, allow_trailer=False).
        ~a evaluates to Opt(a).
    '''
    def __and__(self, other): return And(self, other)
    def __rand__(self, other): return And(other, self)
    def __or__(self, other): return Or(self, other)
    def __ror__(self, other): return Or(other, self)
    def __div__(self, other): return Alt(self, other, allow_trailer=True)
    def __rdiv__(self, other): return Alt(other, self, allow_trailer=True)
    def __truediv__(self, other): return Alt(self, other, allow_trailer=True)
    def __rtruediv__(self, other): return Alt(other, self, allow_trailer=True)
    def __floordiv__(self, other): return Alt(self, other, allow_trailer=False)
    def __rfloordiv__(self, other): return Alt(other, self, allow_trailer=False)
    def __invert__(self): return Opt(self)


# These classes add the "&" and "|" parsing operators.
class TermMetaClass(type, ParsingOperand): pass
class Term(ParsingOperand): __metaclass__ = TermMetaClass


class SimpleTerm(Term):
    '''Abstract base class for terms that consist of a single subterm.'''
    def __init__(self, term):
        self.term = term


class Any(Term):
    '''
    Returns the next element of the input. Fails if the remaining input is
    empty. This class can be used as a term directly, or it can be
    instantiated.
    '''
    @staticmethod
    def parse(source, pos):
        yield (ParseFailure if pos >= len(source)
            else ParseResult(source[pos], pos + 1))


class Backtrack(Term):
    '''
    Moves the current position back by some number of spaces. If the new
    position would be less than zero, then it fails and has no other effect.
    '''
    def __init__(self, count=1):
        self.count = count

    def parse(self, source, pos):
        dst = pos - self.count
        yield ParseFailure if dst < 0 else ParseResult(None, dst)


def Alt(term, separator, allow_trailer=True):
    '''
    Parses a list of terms separated by a separator. Returns the elements
    in a normal list, and drops the separators.
    '''
    rest = List(Right(separator, term))
    tail = Opt(separator) if allow_trailer else None
    triple = (term, rest, tail)
    return Transform(Opt(triple), lambda t: [t[0]] + t[1] if t else [])


def And(left, right):
    return Left(left, Expect(right))


def AnyInst(*cls):
    return Where(lambda x: isinstance(x, cls))


class Bind(Term):
    def __init__(self, term, continuation):
        self.term = term
        self.continuation = continuation

    def parse(self, source, pos):
        arg = yield ParseStep(self.term, pos)
        if arg is ParseFailure:
{'fraction_non_alphanumeric': 0.10037852236085798, 'fraction_numerical': 0.0014019346698443853, 'mean_word_length': 3.4393279402613564, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '459863', 'n_tokens_mistral': 2157, 'n_tokens_neox': 2082, 'n_words': 830}