<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Copyright (c) 2012 <NAME>
# Licensed under MIT License

from Parse import *
from cor import is_iterable, Err, integers
from Common import *

inf = const('inf')

class Forward(object):
    def __init__(self, name):
        self.name = ''.join(['>', name])

    def parse(self, src):
        return self.fn.parse(src)

def mk_rule(src):
    if isinstance(src, Rule):
        return src
    if is_str(src):
        return CharRule(src)
    raise Err("Can't make rule from {}", src)

def mk_rule_seq(r1, r2, name):
    r1 = mk_rule(r1)
    r2 = mk_rule(r2)
    return SeqRule((r1, r2), name)

def mk_rule_lookahead(r):
    r = mk_rule(r)
    return LookaheadRule(r, r.name)

def mk_rule_seq_lookahead(r1, r2, name):
    r1 = mk_rule(r1)
    r2 = mk_rule_lookahead(r2)
    return SeqRule((r1, r2), name)

def mk_rule_choice(r1, r2, name):
    r1 = mk_rule(r1)
    r2 = mk_rule(r2)
    return ChoiceRule((r1, r2), name)

def _mk_parser(name, generator, action, options):
    rule = generator()
    rule.name = name
    parser = rule(options)
    return parser

class Rule(object):

    def __init__(self, data, name, action = ignore):
        self.parser = None
        self.data = data
        self.name = name
        self.action = action
        self.__integers = integers()

    def __repr__(self):
        return ''.join([self.__class__.__name__,'(', repr(self.name), ')'])

    @property
    def _next_child_name(self):
        return '_'.join([self.name, str(self.__integers.next())])

    def __add__(self, other):
        return mk_rule_seq(self, other, self._next_child_name)

    def __radd__(self, other):
        if not is_str(other) and is_iterable(other):
            return other.__add__((self,))
        return mk_rule_seq(other, self, self._next_child_name)

    def __or__(self, other):
        return mk_rule_choice(self, other, self._next_child_name)

    def __ror__(self, other):
        return mk_rule_choice(other, self, self._next_child_name)

    def __and__(self, other):
        return mk_rule_seq_lookahead(self, other, self._next_child_name)

    def __rand__(self, other):
        return mk_rule_seq_lookahead(other, self, self._next_child_name)

    def __getitem__(self, k):
        if not isinstance(k, slice):
            return RangeRule(self, (k, k), self._next_child_name)

        if not (k.step is None or k.step == 1):
            raise cor.Err("Can't handle step != 1")
        start, stop = k.start, k.stop
        r = (0 if start is None else start,
             inf if stop is None else stop)
        return RangeRule(self, r, self._next_child_name)

    def __gt__(self, action):
        self.action = action
        return self

    def __neg__(self):
        return LookaheadRule(self, self.name)

    def __invert__(self):
        return NotRule(self, ''.join(['~', self.name]))

    def __call__(self, options = default_options):
        if self.parser:
            return self.parser

        self.parser = Forward(self.name)

        data = self.data
        if isinstance(data, Rule):
            data = data(options)
        elif not is_str(data) and is_iterable(data):
            data = [x(options) for x in data]
        parser = self.fn(self.name, data, self.action, options)
        self.parser.fn = parser
        self.parser = parser
        return parser

class TopRule(Rule):
    def __init__(self, fn, action = ignore):
        super(TopRule, self).__init__(fn, fn.__name__, action)
        self.fn = _mk_parser

class SeqRule(Rule):
    def __init__(self, rules, name, action = value):
        super(SeqRule, self).__init__(rules, name, action)
        self.fn = match_seq

    def __add__(self, other):
        return SeqRule(self.data + (mk_rule(other),), self.name)

    def __radd__(self, other):
        return SeqRule((mk_rule(other),) + self.data, self.name)

    def __and__(self, other):
        other = mk_rule_lookahead(other)
        return SeqRule(self.data + (other,), self.name)

    def __rand__(self, other):
        other = mk_rule_lookahead(other)
        return SeqRule((other,) + self.data, self.name)

class ChoiceRule(Rule):
    def __init__(self, rules, name, action = value):
        super(ChoiceRule, self).__init__(rules, name, action)
        self.fn = match_any

    def __or__(self, other):
        return ChoiceRule(self.data + (mk_rule(other),), self.name)

    def __ror__(self, other):<commit_msg>action is combined from default_action and action<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Copyright (c) 2012 <NAME>
# Licensed under MIT License

from Parse import *
from cor import is_iterable, Err, integers
from Common import *

inf = const('inf')

class Forward(object):
    def __init__(self, name):
        self.name = ''.join(['>', name])

    def parse(self, src):
        return self.fn.parse(src)

def mk_rule(src):
    if isinstance(src, Rule):
        return src
    if is_str(src):
        return CharRule(src)
    raise Err("Can't make rule from {}", src)

def mk_rule_seq(r1, r2, name):
    r1 = mk_rule(r1)
    r2 = mk_rule(r2)
    return SeqRule((r1, r2), name)

def mk_rule_lookahead(r):
    r = mk_rule(r)
    return LookaheadRule(r, r.name)

def mk_rule_seq_lookahead(r1, r2, name):
    r1 = mk_rule(r1)
    r2 = mk_rule_lookahead(r2)
    return SeqRule((r1, r2), name)

def mk_rule_choice(r1, r2, name):
    r1 = mk_rule(r1)
    r2 = mk_rule(r2)
    return ChoiceRule((r1, r2), name)

def _mk_parser(name, generator, action, options):
    rule = generator()
    rule.name = name
    parser = rule(options)
    return parser

class Rule(object):

    def __init__(self, data, name, action = ignore):
        self.parser = None
        self.data = data
        self.name = name
        self.default_action = action
        self._action = None
        self.__integers = integers()

    def __repr__(self):
        return ''.join([self.__class__.__name__,'(', repr(self.name), ')'])

    @property
    def _next_child_name(self):
        return '_'.join([self.name, str(self.__integers.next())])

    def __add__(self, other):
        return mk_rule_seq(self, other, self._next_child_name)

    def __radd__(self, other):
        if not is_str(other) and is_iterable(other):
            return other.__add__((self,))
        return mk_rule_seq(other, self, self._next_child_name)

    def __or__(self, other):
        return mk_rule_choice(self, other, self._next_child_name)

    def __ror__(self, other):
        return mk_rule_choice(other, self, self._next_child_name)

    def __and__(self, other):
        return mk_rule_seq_lookahead(self, other, self._next_child_name)

    def __rand__(self, other):
        return mk_rule_seq_lookahead(other, self, self._next_child_name)

    def __getitem__(self, k):
        if not isinstance(k, slice):
            return RangeRule(self, (k, k), self._next_child_name)

        if not (k.step is None or k.step == 1):
            raise cor.Err("Can't handle step != 1")
        start, stop = k.start, k.stop
        r = (0 if start is None else start,
             inf if stop is None else stop)
        return RangeRule(self, r, self._next_child_name)

    def __gt__(self, action):
        self._action = action
        return self

    @property
    def action(self):
        return self._action if self._action else self.default_action

    def __neg__(self):
        return LookaheadRule(self, self.name)

    def __invert__(self):
        return NotRule(self, ''.join(['~', self.name]))

    def __call__(self, options = default_options):
        if self.parser:
            return self.parser

        self.parser = Forward(self.name)

        data = self.data
        if isinstance(data, Rule):
            data = data(options)
        elif not is_str(data) and is_iterable(data):
            data = [x(options) for x in data]
        parser = self.fn(self.name, data, self.action, options)
        self.parser.fn = parser
        self.parser = parser
        return parser

class TopRule(Rule):
    def __init__(self, fn, action = ignore):
        super(TopRule, self).__init__(fn, fn.__name__, action)
        self.fn = _mk_parser

class SeqRule(Rule):
    def __init__(self, rules, name, action = value):
        super(SeqRule, self).__init__(rules, name, action)
        self.fn = match_seq

    def __add__(self, other):
        return SeqRule(self.data + (mk_rule(other),), self.name)

    def __radd__(self, other):
        return SeqRule((mk_rule(other),) + self.data, self.name)

    def __and__(self, other):
        other = mk_rule_lookahead(other)
        return SeqRule(self.data + (other,), self.name)

    def __rand__(self, other):
        other = mk_rule_lookahead(other)
        return SeqRule((other,) + self.data, self.name)

class ChoiceRule(Rule):
    def __init__(self, rules, name, action = value):
        for r in rules:
            r.default_action = value
        super(ChoiceRule, self).__init__(rules, name, action)
        self.fn = match_any

    def __or__(self, other):
        return ChoiceRule(self.data + (mk_rule(other),), self.name)

    def __ror__(self, other):
{'fraction_non_alphanumeric': 0.12119886239334937, 'fraction_numerical': 0.007000656311529206, 'mean_word_length': 2.713647441104793, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2431480', 'n_tokens_mistral': 3066, 'n_tokens_neox': 2974, 'n_words': 776}