<commit_before># -*- coding: utf8 -*-
import re
import itertools
from collections import defaultdict
from operator import methodcaller

from pygments.lexer import RegexLexer, bygroups
from pygments.token import *

import trie
from utils import CachedAttr

_translate_table = dict(zip(map(ord, ' .,:[]()'), itertools.repeat(None)))


def squish(u, tr_table=_translate_table):
    'Stupid name for a stupid function.'
    return u.translate(tr_table).lower()


class LexerType(type):

    _classes = set()

    def __new__(meta, name, bases, attrs):
        cls = type.__new__(meta, name, bases, attrs)
        meta._classes.add(cls)
        return cls

    @classmethod
    def classes(cls):
        return itertools.ifilterfalse(methodcaller('isabstract'), cls._classes)


class Scanner(object):

    @CachedAttr
    def _slug_regex(self):
        'A prematurely optimized prefix tree regex for locating cite slugs.'
        return trie.build_regex(cls.slug for cls in LexerType.classes())

    @CachedAttr
    def _slug_lexer_mapping(self):
        'Mapping of matched citation slugs to lexer classes.'
        return dict((squish(cls.slug), cls) for cls in LexerType.classes())

    def __call__(self, string):
        for matchobj in re.finditer(self._slug_regex, string, re.I):
            lexer = self._slug_lexer_mapping[squish(matchobj.group())]
            yield lexer().lex(matchobj, string)


class _SplitLexerBase(RegexLexer):
    'This modified RegexLexer bails after encountering an error token.'

    def get_tokens_unprocessed(self, text, Error=Token.Error):
        for pos, token, string in RegexLexer.get_tokens_unprocessed(self, text):
            if token == Error:
                return
            else:
                yield pos, token, string


class LexerBase(object):
    __metaclass__ = LexerType

    @classmethod
    def isabstract(cls):
        return not hasattr(cls, 'slug')

    @CachedAttr
    def reverse_lexer(self):
        class ReverseLexer(_SplitLexerBase):
            flags = re.U | re.M
            tokens = self.reverse_tokens
        return ReverseLexer()

    @CachedAttr
    def forward_lexer(self):
        class ForwardLexer(_SplitLexerBase):
            flags = re.U | re.M
            tokens = self.forward_tokens
        return ForwardLexer()

    def lex(self, matchobj, string, Junk=Token.Junk):
        rev_string = string[::-1][-matchobj.start():]
        fwd_string = string[matchobj.end():]

        rev_lexer = self.reverse_lexer
        fwd_lexer = self.forward_lexer

        rev_tokenized = list(rev_lexer.get_tokens_unprocessed(rev_string))
        fwd_tokenized = list(fwd_lexer.get_tokens_unprocessed(fwd_string))

        result = defaultdict(list)
        for pos, token, text in rev_tokenized:
            if token != Junk:
                result[token].append(text[::-1])

        for pos, token, text in fwd_tokenized:
            if token != Junk:
                result[token].append(text)
        return dict(result)


# Custom Tokens
Statute = Token.Statute
Junk = Token.Junk


class Statute(LexerBase):

    reverse_tokens = {
        'root': [
            (ur'(\s*)(\d+)', bygroups(Junk, Statute.Title)),
            ]
        }

    forward_tokens = {
        'root': [
            (ur'(\s{,3})(\xa7{,2})(\s{,3})(\d[\w\-.]+)',
                bygroups(Junk, Statute.SecSymbol, Junk, Statute.Section)),
            ]
        }<commit_msg>try to get split lexer thing working<commit_after># -*- coding: utf8 -*-
import re
import itertools
from collections import defaultdict
from operator import methodcaller

from pygments.lexer import RegexLexer, bygroups
from pygments.token import *

import trie
from utils import CachedAttr

_translate_table = dict(zip(map(ord, ' .,:[]()'), itertools.repeat(None)))


def squish(u, tr_table=_translate_table):
    'Stupid name for a stupid function.'
    return u.translate(tr_table).lower()


class LexerType(type):

    _classes = set()

    def __new__(meta, name, bases, attrs):
        cls = type.__new__(meta, name, bases, attrs)
        meta._classes.add(cls)
        return cls

    @classmethod
    def classes(cls):
        return itertools.ifilterfalse(methodcaller('isabstract'), cls._classes)


class Scanner(object):

    @CachedAttr
    def _slug_regex(self):
        'A prematurely optimized prefix tree regex for locating cite slugs.'
        return trie.build_regex(cls.slug for cls in LexerType.classes())

    @CachedAttr
    def _slug_lexer_mapping(self):
        'Mapping of matched citation slugs to lexer classes.'
        return dict((squish(cls.slug), cls) for cls in LexerType.classes())

    def __call__(self, string):
        for matchobj in re.finditer(self._slug_regex, string, re.I):
            lexer = self._slug_lexer_mapping[squish(matchobj.group())]
            yield lexer().lex(matchobj, string)


class _SplitLexerBase(RegexLexer):
    'This modified RegexLexer bails after encountering an error token.'

    def get_tokens_unprocessed(self, text, Error=Token.Error):
        for pos, token, string in RegexLexer.get_tokens_unprocessed(self, text):
            if token == Error:
                return
            else:
                yield pos, token, string


class LexerBase(object):
    __metaclass__ = LexerType

    @classmethod
    def isabstract(cls):
        return not hasattr(cls, 'slug')

    @CachedAttr
    def reverse_lexer(self):
        class ReverseLexer(_SplitLexerBase):
            flags = re.U | re.M
            tokens = self.reverse_tokens
        return ReverseLexer()

    @CachedAttr
    def forward_lexer(self):
        class ForwardLexer(_SplitLexerBase):
            flags = re.U | re.M
            tokens = self.forward_tokens
        return ForwardLexer()

    def lex(self, matchobj, string, Junk=Token.Junk):

        result = defaultdict(list)

        rev_string = string[::-1][-matchobj.start():]
        rev_lexer = self.reverse_lexer
        rev_tokenized = list(rev_lexer.get_tokens_unprocessed(rev_string))
        for pos, token, text in rev_tokenized:
            if token != Junk:
                result[token].append(text[::-1])

        import pdb;pdb.set_trace()
        fwd_string = string[matchobj.end():]
        fwd_lexer = self.forward_lexer
        fwd_tokenized = list(fwd_lexer.get_tokens_unprocessed(fwd_string))
        for pos, token, text in fwd_tokenized:
            if token != Junk:
                result[token].append(text)
        return dict(result)


# Custom Tokens
Statute = Token.Statute
Junk = Token.Junk

# Regexes
rgxs = {
    'statute_section': '''
    (?x)                   # Compile verbose
    ,?\s{,2}               # Whitespace
    (\d[\w\-.]+)           # 123-2.1
    (?:\s{,2})             # Optional whitespace
    (\([\w\-.]+\))+        # Enumerations like "(1)" or "(12.1-a)"
    '''
    }


class Statute(LexerBase):

    reverse_tokens = {
        'root': [
            (ur'(\s*)(\d+)', bygroups(Junk, Statute.Title)),
            ]
        }

    forward_tokens = {
        'root': [
            (ur'(:?\s{,3})(:?\xa7{,2})*(?:\s{,3})%s' % rgxs['statute_section'],
                bygroups(Junk, Statute.SecSymbol, Statute.Section)),
            #(',?\s{,2}(\d[\w\-.]+)', bygroups(Statute.Section)),
            (ur'\s{,2}\((\d{4})(?:\)|(?: ed(?:\.?|ition),?))(?:\s{,2}Supp.? (\w+))?',
                bygroups(Statute.Edition, Statute.Supp)),
            (ur'(?:\s{,2}\([\w\-.]+\))+', Statute.Subdivisions)
            ],

        # 'subdivisions': [
        #     (ur'(?:\s{,2}\([\w\-.]+\))+', Statute.Subdivisions)
        #     ]
        }
{'fraction_non_alphanumeric': 0.1283436552905521, 'fraction_numerical': 0.004084859665305047, 'mean_word_length': 2.45, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1573178', 'n_tokens_mistral': 2517, 'n_tokens_neox': 2408, 'n_words': 531}