<commit_before>#/usr/bin/python
# -*- coding: utf-8 -*-

# Copyright (c) 2012 <NAME>
# Licensed under MIT License

import string

import Rule
import Parse
from Common import *

def rule(fn): return Rule.TopRule(fn)

def char(c): return Rule.CharRule(c)
def text(s): return Rule.StringRule(s)

def source(src, begin = 0, end = None):
    return Parse.InfiniteInput(src, begin, end)

def cache_clean(rules_dict):
    '''rules_dict is ordinary result of grammar module globals()
    call'''
    for x in rules_dict.values():
        if isinstance(x, Rule.Rule):
            x.parser = None


@rule
def vspace(): return char('\n\r') > ignore
@rule
def hspace(): return char(' \t') > ignore
@rule
def crlf(): return text('\r\n') > ignore
@rule
def eol(): return eof | vspace > ignore
@rule
def eof(): return char(empty) > ignore
@rule
def space(): return char(' \n\r\t') > ignore
@rule
def spaces(): return space[0:] > ignore
@rule
def vspaces(): return vspace[0:] > ignore
@rule
def hspaces(): return hspace[0:] > ignore
@rule
def any_char(): return ~eof > value
@rule
def digit_dec() : return char('0123456789') > value
@rule
def digit_hex() : return char('0123456789ABCDEFabcdef') > value

@rule
def ascii():
    def __is_ascii(s): return s in string.ascii_letters
    return char(__is_ascii) > value
@rule
def ascii_digit(): return ascii | digit_dec

def within(begin, end):
    def check(c):
        i = ord(c)
        return i >= begin and i <= end
    @rule
    def fn():
        return char(check) > value
    return fn

@rule
def non_ascii():
    return within(0x80,0xff) > value

from datetime import datetime

@rule
def iso_date():
    return digit_dec[4]  + '-' + digit_dec[2] + '-' + digit_dec[2] + 'T' \
        + digit_dec[2] + ':' + digit_dec[2] + ':' + digit_dec[2] \
        + char('Z')[:1] \
        > (lambda x: datetime(*[int(list2str(i)) for i in x]))

@rule
def num_decimal(): return digit_dec[1:] & space \
    > (lambda x: int(list2str(x[0])))<commit_msg>within parser got name<commit_after>#/usr/bin/python
# -*- coding: utf-8 -*-

# Copyright (c) 2012 <NAME>
# Licensed under MIT License

import string

import Rule
import Parse
from Common import *

def rule(fn): return Rule.TopRule(fn)

def char(c): return Rule.CharRule(c)
def text(s): return Rule.StringRule(s)

def source(src, begin = 0, end = None):
    return Parse.InfiniteInput(src, begin, end)

def cache_clean(rules_dict):
    '''rules_dict is ordinary result of grammar module globals()
    call'''
    for x in rules_dict.values():
        if isinstance(x, Rule.Rule):
            x.parser = None


@rule
def vspace(): return char('\n\r') > ignore
@rule
def hspace(): return char(' \t') > ignore
@rule
def crlf(): return text('\r\n') > ignore
@rule
def eol(): return eof | vspace > ignore
@rule
def eof(): return char(empty) > ignore
@rule
def space(): return char(' \n\r\t') > ignore
@rule
def spaces(): return space[0:] > ignore
@rule
def vspaces(): return vspace[0:] > ignore
@rule
def hspaces(): return hspace[0:] > ignore
@rule
def any_char(): return ~eof > value
@rule
def digit_dec() : return char('0123456789') > value
@rule
def digit_hex() : return char('0123456789ABCDEFabcdef') > value

@rule
def ascii():
    def __is_ascii(s): return s in string.ascii_letters
    return char(__is_ascii) > value
@rule
def ascii_digit(): return ascii | digit_dec

def within(begin, end):
    def check(c):
        i = ord(c)
        return i >= begin and i <= end
    @rule
    def fn():
        return char(check) > value
    fn.name = '?[{}, {}]'.format(begin, end)
    return fn

@rule
def non_ascii():
    return within(0x80,0xff) > value

from datetime import datetime

@rule
def iso_date():
    return digit_dec[4]  + '-' + digit_dec[2] + '-' + digit_dec[2] + 'T' \
        + digit_dec[2] + ':' + digit_dec[2] + ':' + digit_dec[2] \
        + char('Z')[:1] \
        > (lambda x: datetime(*[int(list2str(i)) for i in x]))

@rule
def num_decimal(): return digit_dec[1:] & space \
    > (lambda x: int(list2str(x[0])))
{'fraction_non_alphanumeric': 0.15784215784215785, 'fraction_numerical': 0.02197802197802198, 'mean_word_length': 3.5563139931740615, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4509294', 'n_tokens_mistral': 1470, 'n_tokens_neox': 1392, 'n_words': 433}