<commit_before>from collections import namedtuple
import string
from enum import Enum
import logging


class EndOfFile(Exception):
    pass

logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)

cat_codes = [
    'escape',  # 0
    'begin_group',  # 1
    'end_group',  # 2
    'math_shift',  # 3
    'align_tab',  # 4
    'end_of_line',  # 5
    'parameter',  # 6
    'superscript',  # 7
    'subscript',  # 8
    'ignored',  # 9
    'space',  # 10
    'letter',  # 11
    'other',  # 12
    'active',  # 13
    'comment',  # 14
    'invalid',  # 15
]

CatCode = Enum('CatCode', {symbol: i for i, symbol in enumerate(cat_codes)})


weird_char_codes = {
    'null': 0,
    'line_feed': 10,
    'carriage_return': 13,
    'delete': 127,
}
weird_chars = {
    k: chr(v) for k, v in weird_char_codes.items()
}
WeirdChar = Enum('WeirdChar', weird_chars)


tokenise_cats = [
    CatCode.begin_group,
    CatCode.end_group,
    CatCode.math_shift,
    CatCode.align_tab,
    CatCode.parameter,
    CatCode.superscript,  # Assuming not part of hex trio.
    CatCode.subscript,
    CatCode.letter,
    CatCode.other,
    CatCode.active,
]


math_classes = [
    'ordinary',  # 0
    'large_operator',  # 1
    'binary_relation',  # 2
    'relation',  # 3
    'opening',  # 4
    'closing',  # 5
    'punctuation',  # 6
    'variable_family',  # 7
    'special_active',  # 8 (weird special case)
]

MathClass = Enum('MathClass', {symbol: i for i, symbol in enumerate(math_classes)})

MathCode = namedtuple('MathCode', ('math_class', 'family', 'position'))


class ReadingState(Enum):
    line_begin = 'N'
    line_middle = 'M'
    skipping_blanks = 'S'


class Mode(Enum):
    # Building the main vertical list.
    vertical_mode = 'V'
    # Building a vertical list for a vbox.
    internal_vertical_mode = 'IV'
    # Building a horizontal list for a paragraph.
    horizontal_mode = 'H'
    # Building a horizontal list for an hbox.
    restricted_horizontal_mode = 'RH'
    # Building a formula to be placed in a horizontal list.
    math_mode = 'M'
    # Building a formula to be placed on a line by itself,
    # interrupting the current paragraph.
    display_math_mode = 'DM'


class State(object):

    def __init__(self, chars):
        self.i = -1
        self.chars = chars

        self.reading_state = ReadingState.line_begin
        # At the beginning, TeX is in vertical mode, ready to construct pages.
        self.mode = Mode.vertical_mode
        self.initialize_char_cats()
        self.initialize_char_math_codes()
        self.initialize_case_codes()
        self.initialize_control_sequences()
        self.expanding_tokens = True

    def disable_expansion(self):
        self.expanding_tokens = False

    def enable_expansion(self):
        self.expanding_tokens = True

    def initialize_control_sequences(self):
        self.control_sequences = {}
        # TODO: should these control sequences actually return
        # (char, cat) pairs? Rather than just a plain character?
        self.control_sequences.update({c: [c] for c in self.char_to_cat})

    def initialize_char_cats(self):
        self.char_to_cat = {
            chr(i): CatCode.other for i in range(128)
        }
        self.char_to_cat.update({let: CatCode.letter
                                 for let in string.ascii_letters})

        self.char_to_cat['\\'] = CatCode.escape
        self.char_to_cat[' '] = CatCode.space
        self.char_to_cat['%'] = CatCode.comment
        self.char_to_cat[WeirdChar.null.value] = CatCode.ignored
        # NON-STANDARD
        self.char_to_cat[WeirdChar.line_feed.value] = CatCode.end_of_line
        self.char_to_cat[WeirdChar.carriage_return.value] = CatCode.end_of_line
        self.char_to_cat[WeirdChar.delete.value] = CatCode.invalid

    def initialize_char_math_codes(self):
        self.char_to_math_code = {}
        for i in range(256):
            if chr(i) in string.ascii_letters:
                family = 1
            else:
                family = 0
            if chr(i) in (string.ascii_letters + string.digits):
                math_class = MathClass.variable_family
            else:
                math_class = MathClass.ordinary
            self.char_to_math_code[i] = MathCode(math_class=math_class,
                                                 family=family,
                                                 position=i)
            # TODO: handle special "8000 value, page 155 of The TeXbook.

    def initialize_case_codes(self):
        self.lower_case_code, self.upper_case_code = [{chr(i): chr(0)
                                                       for i in range(128)}
                                                      for _ in range(2)]
        for lower, upper in zip(string.ascii_lowercase, string.ascii_uppercase):
            self.lower_case_code[lower] = lower
            self.upper_case_code[upper] = upper
            self.lower_case_code[upper] = lower
            self.upper_case_code[lower] = upper

    def peek_ahead(self, n=1):
        try:
            char = self.chars[self.i + n]
        except IndexError:
            raise EndOfFile
        cat = self.char_to_cat[char]
        return char, cat

    @property<commit_msg>Clean up dealing with ASCII characters in processor<commit_after>from collections import namedtuple
import string
from string import ascii_letters, ascii_lowercase, ascii_uppercase
from enum import Enum
import logging


ascii_characters = ''.join(chr(i) for i in range(128))


class EndOfFile(Exception):
    pass

logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)

cat_codes = [
    'escape',  # 0
    'begin_group',  # 1
    'end_group',  # 2
    'math_shift',  # 3
    'align_tab',  # 4
    'end_of_line',  # 5
    'parameter',  # 6
    'superscript',  # 7
    'subscript',  # 8
    'ignored',  # 9
    'space',  # 10
    'letter',  # 11
    'other',  # 12
    'active',  # 13
    'comment',  # 14
    'invalid',  # 15
]

CatCode = Enum('CatCode', {symbol: i for i, symbol in enumerate(cat_codes)})


weird_char_codes = {
    'null': 0,
    'line_feed': 10,
    'carriage_return': 13,
    'delete': 127,
}
weird_chars = {
    k: chr(v) for k, v in weird_char_codes.items()
}
WeirdChar = Enum('WeirdChar', weird_chars)


tokenise_cats = [
    CatCode.begin_group,
    CatCode.end_group,
    CatCode.math_shift,
    CatCode.align_tab,
    CatCode.parameter,
    CatCode.superscript,  # Assuming not part of hex trio.
    CatCode.subscript,
    CatCode.letter,
    CatCode.other,
    CatCode.active,
]


math_classes = [
    'ordinary',  # 0
    'large_operator',  # 1
    'binary_relation',  # 2
    'relation',  # 3
    'opening',  # 4
    'closing',  # 5
    'punctuation',  # 6
    'variable_family',  # 7
    'special_active',  # 8 (weird special case)
]

MathClass = Enum('MathClass', {symbol: i for i, symbol in enumerate(math_classes)})

MathCode = namedtuple('MathCode', ('math_class', 'family', 'position'))


class ReadingState(Enum):
    line_begin = 'N'
    line_middle = 'M'
    skipping_blanks = 'S'


class Mode(Enum):
    # Building the main vertical list.
    vertical_mode = 'V'
    # Building a vertical list for a vbox.
    internal_vertical_mode = 'IV'
    # Building a horizontal list for a paragraph.
    horizontal_mode = 'H'
    # Building a horizontal list for an hbox.
    restricted_horizontal_mode = 'RH'
    # Building a formula to be placed in a horizontal list.
    math_mode = 'M'
    # Building a formula to be placed on a line by itself,
    # interrupting the current paragraph.
    display_math_mode = 'DM'


class State(object):

    def __init__(self, chars):
        self.i = -1
        self.chars = chars

        self.reading_state = ReadingState.line_begin
        # At the beginning, TeX is in vertical mode, ready to construct pages.
        self.mode = Mode.vertical_mode
        self.initialize_char_cats()
        self.initialize_char_math_codes()
        self.initialize_case_codes()
        self.initialize_control_sequences()
        self.expanding_tokens = True

    def disable_expansion(self):
        self.expanding_tokens = False

    def enable_expansion(self):
        self.expanding_tokens = True

    def initialize_control_sequences(self):
        self.control_sequences = {}
        # TODO: should these control sequences actually return
        # (char, cat) pairs? Rather than just a plain character?
        self.control_sequences.update({c: [c] for c in self.char_to_cat})

    def initialize_char_cats(self):
        self.char_to_cat = {
            c: CatCode.other for c in ascii_characters
        }
        self.char_to_cat.update({let: CatCode.letter
                                 for let in ascii_letters})

        self.char_to_cat['\\'] = CatCode.escape
        self.char_to_cat[' '] = CatCode.space
        self.char_to_cat['%'] = CatCode.comment
        self.char_to_cat[WeirdChar.null.value] = CatCode.ignored
        # NON-STANDARD
        self.char_to_cat[WeirdChar.line_feed.value] = CatCode.end_of_line
        self.char_to_cat[WeirdChar.carriage_return.value] = CatCode.end_of_line
        self.char_to_cat[WeirdChar.delete.value] = CatCode.invalid

    def initialize_char_math_codes(self):
        self.char_to_math_code = {}
        for i, c in enumerate(ascii_characters):
            if c in ascii_letters:
                family = 1
            else:
                family = 0
            if c in (ascii_letters + string.digits):
                math_class = MathClass.variable_family
            else:
                math_class = MathClass.ordinary
            self.char_to_math_code[i] = MathCode(math_class=math_class,
                                                 family=family,
                                                 position=i)
            # TODO: handle special "8000 value, page 155 of The TeXbook.

    def initialize_case_codes(self):
        self.lower_case_code, self.upper_case_code = [
            {c: chr(0) for c in ascii_characters}
            for _ in range(2)
        ]
        for lower, upper in zip(ascii_lowercase, ascii_uppercase):
            self.lower_case_code[lower] = lower
            self.upper_case_code[upper] = upper
            self.lower_case_code[upper] = lower
            self.upper_case_code[lower] = upper

    def peek_ahead(self, n=1):
        try:
            char = self.chars[self.i + n]
        except IndexError:
            raise EndOfFile
        cat = self.char_to_cat[char]
        return char, cat

    @property
{'fraction_non_alphanumeric': 0.09922215898311516, 'fraction_numerical': 0.011003604629102636, 'mean_word_length': 2.2752407579993785, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2670986', 'n_tokens_mistral': 3469, 'n_tokens_neox': 3248, 'n_words': 787}