<commit_before># -*- coding: utf-8 -*-
'''
Created : 2015-08-24

@author: <NAME>
'''

from textops import TextOp, NoAttr
import textops
from slugify import slugify
import re
import pprint
pp = pprint.PrettyPrinter(indent=4)
import copy
from datetime import datetime

class parsek(TextOp):
    ignore_case = False
    @classmethod
    def op(cls,text, pattern, key_name = 'key', key_update = None, *args,**kwargs):
        if isinstance(pattern,basestring):
            pattern = re.compile(pattern, re.I if cls.ignore_case else 0)
        out = []
        for line in cls._tolist(text):
            m = pattern.match(line)
            if m:
                dct = m.groupdict()
                key = dct.get(key_name)
                if key:
                    if key_update:
                        key = key_update(key)
                    out.append(key)
        return out

class parseki(parsek): ignore_case = True

class parsekv(TextOp):
    ignore_case = False
    @classmethod
    def op(cls,text, pattern, key_name = 'key', key_update = None, *args,**kwargs):
        if isinstance(pattern,basestring):
            pattern = re.compile(pattern, re.I if cls.ignore_case else 0)
        out = {}
        for line in cls._tolist(text):
            m = pattern.match(line)
            if m:
                dct = m.groupdict()
                key = dct.get(key_name)
                if key:
                    if key_update is None:
                        key_norm = re.sub(r'\s+','_',key.strip())
                    else:
                        key_norm = key_update(key)
                    out.update({ key_norm : dct })
        return out

class parsekvi(parsekv): ignore_case = True

class find_patterns(TextOp):
    stop_when_found = False
    ignore_case = False

    @classmethod
    def op(cls,text, patterns_dict):
        out = {}
        text = cls._tostr(text)
        for attr,pattern in patterns_dict.items():
            if isinstance(pattern,basestring):
                pattern = re.compile(pattern, re.M | (re.I if cls.ignore_case else 0))
            if pattern:
                m = pattern.search(text)
                if m :
                    tmp_groupdict = m.groupdict() or dict([('group%s' % k,v) for k,v in enumerate(m.groups())])
                    groupdict = {}
                    for grp, val in tmp_groupdict.items():
                        if grp[:3] == 'INT':
                            try:
                                groupdict[grp[3:]] = int(val)
                            except ValueError:
                                groupdict[grp[3:]] = 0
                        else:
                            groupdict[grp] = val
                    groupdict = cls.pre_store(groupdict)
                    if len(groupdict) == 1:
                        out.update({ attr : groupdict.popitem()[1] })
                    else:
                        out.update({ attr : groupdict })
                    if cls.stop_when_found:
                        break
        return out

    @classmethod
    def pre_store(self,groupdict):
        return groupdict

class find_patternsi(find_patterns): ignore_case=True


class find_first_pattern(find_patterns):
    stop_when_found = True

    @classmethod
    def op(cls,text, patterns):
        data = find_patterns.op(text, dict([('pat%s' % k,v) for k,v in enumerate(patterns)]))
        if not data:
            return NoAttr
        return data.popitem()[1]

class find_first_patterni(find_patterns): ignore_case=True

class state_machine(TextOp):
    @classmethod
    def op(cls,text, state_machine_def,*args,**kwargs):
        parser = StateMachineParser()
        if kwargs.get('debug'):
            parser.set_debug(True)
        parser.set_params(state_machine_def)
        out = parser.parse(text)
        return out

def index_normalize(index_val):
    return slugify(index_val)

STATE_MACHINE_LINE_REPARSE_MAX = 5

re_multiple_tag=re.compile('^_(\d+)(.*)$')
month_name_to_id = {
'jan' : 1,
'janv' : 1,<commit_msg>start coding simple_state_machine<commit_after># -*- coding: utf-8 -*-
'''
Created : 2015-08-24

@author: <NAME>
'''

from textops import TextOp, NoAttr
import textops
from slugify import slugify
import re
import pprint
pp = pprint.PrettyPrinter(indent=4)
import copy
from datetime import datetime

class parseg(TextOp):
    ignore_case = False
    @classmethod
    def op(cls,text, pattern, key_update = None, *args,**kwargs):
        if isinstance(pattern,basestring):
            pattern = re.compile(pattern, re.I if cls.ignore_case else 0)
        out = []
        for line in cls._tolist(text):
            m = pattern.match(line)
            if m:
                out.append(m.groupdict())
        return out

class parsegi(parseg): ignore_case = True

class parsek(TextOp):
    ignore_case = False
    @classmethod
    def op(cls,text, pattern, key_name = 'key', key_update = None, *args,**kwargs):
        if isinstance(pattern,basestring):
            pattern = re.compile(pattern, re.I if cls.ignore_case else 0)
        out = []
        for line in cls._tolist(text):
            m = pattern.match(line)
            if m:
                dct = m.groupdict()
                key = dct.get(key_name)
                if key:
                    if key_update:
                        key = key_update(key)
                    out.append(key)
        return out

class parseki(parsek): ignore_case = True

class parsekv(TextOp):
    ignore_case = False
    @classmethod
    def op(cls,text, pattern, key_name = 'key', key_update = None, *args,**kwargs):
        if isinstance(pattern,basestring):
            pattern = re.compile(pattern, re.I if cls.ignore_case else 0)
        out = {}
        for line in cls._tolist(text):
            m = pattern.match(line)
            if m:
                dct = m.groupdict()
                key = dct.get(key_name)
                if key:
                    if key_update is None:
                        key_norm = re.sub(r'\s+','_',key.strip())
                    else:
                        key_norm = key_update(key)
                    out.update({ key_norm : dct })
        return out

class parsekvi(parsekv): ignore_case = True

class find_patterns(TextOp):
    stop_when_found = False
    ignore_case = False

    @classmethod
    def op(cls,text, patterns_dict):
        out = {}
        text = cls._tostr(text)
        for attr,pattern in patterns_dict.items():
            if isinstance(pattern,basestring):
                pattern = re.compile(pattern, re.M | (re.I if cls.ignore_case else 0))
            if pattern:
                m = pattern.search(text)
                if m :
                    tmp_groupdict = m.groupdict() or dict([('group%s' % k,v) for k,v in enumerate(m.groups())])
                    groupdict = {}
                    for grp, val in tmp_groupdict.items():
                        if grp[:3] == 'INT':
                            try:
                                groupdict[grp[3:]] = int(val)
                            except ValueError:
                                groupdict[grp[3:]] = 0
                        else:
                            groupdict[grp] = val
                    groupdict = cls.pre_store(groupdict)
                    if len(groupdict) == 1:
                        out.update({ attr : groupdict.popitem()[1] })
                    else:
                        out.update({ attr : groupdict })
                    if cls.stop_when_found:
                        break
        return out

    @classmethod
    def pre_store(self,groupdict):
        return groupdict

class find_patternsi(find_patterns): ignore_case=True


class find_first_pattern(find_patterns):
    stop_when_found = True

    @classmethod
    def op(cls,text, patterns):
        data = find_patterns.op(text, dict([('pat%s' % k,v) for k,v in enumerate(patterns)]))
        if not data:
            return NoAttr
        return data.popitem()[1]

class find_first_patterni(find_patterns): ignore_case=True

class simple_state_machine(TextOp):
    pass







class state_machine(TextOp):
    @classmethod
    def op(cls,text, state_machine_def,*args,**kwargs):
        parser = StateMachineParser()
        if kwargs.get('debug'):
            parser.set_debug(True)
        parser.set_params(state_machine_def)
        out = parser.parse(text)
        return out

def index_normalize(index_val):
    return slugify(index_val)

STATE_MACHINE_LINE_REPARSE_MAX = 5

re_multiple_tag=re.compile('^_(\d+)(.*)$')
month_name_to_id = {
'jan' : 1,
'janv' : 1,
{'fraction_non_alphanumeric': 0.10439752712002799, 'fraction_numerical': 0.005482328239822699, 'mean_word_length': 1.7279669105949729, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2185207', 'n_tokens_mistral': 2596, 'n_tokens_neox': 2498, 'n_words': 586}