<commit_before>"""
Lexical Analysis Utilities

Exceptions:
    LexError
    LexCompileError
    LexRunError

Interface Classes:
    RegexLexer
"""

import collections
import re

class LexError(Exception):
    """
    Base class for all lexical analysis errors
    """

class LexCompileError(LexError):
    """
    Errors raised during compilation of lexical analysis rules.
    """

class LexRunError(LexError):
    """
    Errors raised during lexical analysis of the source text.
    """
    def __init__(self, msg, lineno, offset, text):
        super(LexRunError, self).__init__(msg, (lineno, offset, text))
        self.lineno = lineno
        self.offset = offset
        self.text = text


class RegexLexer(object):
    """
    Lexer based on regular expressions.
    """

    RULES = {
        'root': []
    }

    def __init__(self, string):
        self.string = string

        self.pos = None
        self.lineno = None
        self.offset = None

        self.states = list()
        self.tokens = collections.deque()

        self.gtoks = None

        self._compile_rules()

    def __iter__(self):
        self.pos = 0
        self.lineno = 1
        self.offset = 1

        self.states.clear()
        self.states.append('root')
        self.tokens.clear()

        self.gtoks = self._iter_tokens()

        return self

    def __next__(self):
        return next(self.gtoks)

    def _compile_rules(self):
        """Compile the rules into the internal lexer state."""
        self._rules = dict()
        for state, table in self.RULES.items():
            patterns = list()
            actions = list()
            nextstates = list()
            for i, row in enumerate(table):
                if len(row) == 2:
                    pattern, action = row
                    nextstate = None
                elif len(row) == 3:
                    pattern, action, nextstate = row
                else:
                    fstr = "invalid RULES: state {}, row {}"
                    raise LexCompileError(fstr.format(state, i))
                patterns.append(pattern)
                actions.append(action)
                nextstates.append(nextstate)
            reobj = re.compile('|'.join("(" + p + ")" for p in patterns))
            self._rules[state] = (reobj, actions, nextstates)
<commit_msg>Fix problem with list.clear in Python 3.2<commit_after>"""
Lexical Analysis Utilities

Exceptions:
    LexError
    LexCompileError
    LexRunError

Interface Classes:
    RegexLexer
"""

import collections
import re

class LexError(Exception):
    """
    Base class for all lexical analysis errors
    """

class LexCompileError(LexError):
    """
    Errors raised during compilation of lexical analysis rules.
    """

class LexRunError(LexError):
    """
    Errors raised during lexical analysis of the source text.
    """
    def __init__(self, msg, lineno, offset, text):
        super(LexRunError, self).__init__(msg, (lineno, offset, text))
        self.lineno = lineno
        self.offset = offset
        self.text = text


class RegexLexer(object):
    """
    Lexer based on regular expressions.
    """

    RULES = {
        'root': []
    }

    def __init__(self, string):
        self.string = string

        self.pos = None
        self.lineno = None
        self.offset = None

        self.states = list()
        self.tokens = collections.deque()

        self.gtoks = None

        self._compile_rules()

    def __iter__(self):
        self.pos = 0
        self.lineno = 1
        self.offset = 1

        self.states = ['root']
        self.tokens.clear()

        self.gtoks = self._iter_tokens()

        return self

    def __next__(self):
        return next(self.gtoks)

    def _compile_rules(self):
        """Compile the rules into the internal lexer state."""
        self._rules = dict()
        for state, table in self.RULES.items():
            patterns = list()
            actions = list()
            nextstates = list()
            for i, row in enumerate(table):
                if len(row) == 2:
                    pattern, action = row
                    nextstate = None
                elif len(row) == 3:
                    pattern, action, nextstate = row
                else:
                    fstr = "invalid RULES: state {}, row {}"
                    raise LexCompileError(fstr.format(state, i))
                patterns.append(pattern)
                actions.append(action)
                nextstates.append(nextstate)
            reobj = re.compile('|'.join("(" + p + ")" for p in patterns))
            self._rules[state] = (reobj, actions, nextstates)

{'fraction_non_alphanumeric': 0.1037451571244081, 'fraction_numerical': 0.002582866982350409, 'mean_word_length': 1.7776449491930664, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3701545', 'n_tokens_mistral': 1357, 'n_tokens_neox': 1259, 'n_words': 354}