<commit_before>from enum import Enum
from collections import namedtuple
import re
import logging

log = logging.getLogger(__name__)


class SequenceType(Enum):
    CURSOR_UP = "A"
    CURSOR_DOWN = "B"
    CURSOR_FORWARD = "C"
    CURSOR_BACK = "D"
    CURSOR_NEXT_LINE = "E"
    CURSOR_PREVIOUS_LINE = "F"
    CURSOR_HORIZONTAL_ABSOLUTE = "G"
    CURSOR_POSITION = "H"
    CURSOR_VERTICAL_ABSOLUTE = "d"
    ERASE_IN_DISPLAY = "J"
    ERASE_IN_LINE = "K"
    ERASE_CHARACTER = "X"
    SCROLL_UP = "S"
    SCROLL_DOWN = "T"
    HORIZONTAL_VERTICAL_POSITION = "f"
    SELECT_GRAPHIC_RENDITION = "m"
    SAVE_CURSOR_POSITION = "s"
    RESTORE_CURSOR_POSITION = "u"
    ENABLE_SETTING = "h"
    DISABLE_SETTING = "l"
    SET_WINDOW = "r"
    UNKNOWN = "ZZZ"


class ParseException(Exception):
    pass


class EscapeSequence():
    # list of escape sequences that use default values of 0
    # instead of the standard of 1
    _default0 = [SequenceType.ERASE_IN_DISPLAY, SequenceType.ERASE_IN_LINE,
                 SequenceType.SELECT_GRAPHIC_RENDITION]
    # list of escape sequences that include an extra '?' in the string
    _hasQMark = [SequenceType.ENABLE_SETTING, SequenceType.DISABLE_SETTING]

    def __init__(self, data, endingLetter, unknownSeq=None):
        self.data = data
        # save a copy in case it's unknown
        self.unknownSequence = unknownSeq
        try:
            self.sequenceType = SequenceType(endingLetter)
        except BaseException:
            # technically, this should have been created using UNKNOWN type
            # assume it can be recreated 'normally'
            self.sequenceType = SequenceType.UNKNOWN
            self.unknownSequence = "\x1b[" + ";".join(
                str(x) for x in self.data) + endingLetter

    def __str__(self):
        # just return the string represnting this sequence
        if self.unknownSequence:
            return self.unknownSequence
        qMark = ""
        if self.sequenceType in EscapeSequence._hasQMark:
            qMark = "?"
        return "\x1b[" + qMark + ";".join(
            str(x) for x in self.data) + self.sequenceType.value

    def __repr__(self):
        return repr(str(self))

    def get_data(self, index):
        # this is just a wrapper to handle default values
        if not isinstance(index, int):
            raise TypeException("index must be an integer")
        if index < len(self.data):
            return self.data[index]
        else:
            # but there are some special cases,
            # because who needs consistency anyways?
            if self.sequenceType in EscapeSequence._default0:
                return 0
            else:
                return 1


class TerminalBuffer():
    _parser = re.compile(r"^\x1b\[\??([\d;]*)(\w)")
    # secondary parser for other sequences that are sufficiently different
    # these commonly don't have data, or don't use nums exclusively for data
    _unknownParser = re.compile(r"^\x1b[\(\)\=\>\%]?([\w\d])?")

    class Position:
        def __init__(self, x, y):
            self.x = x
            self.y = y

    class Character:
        def __init__(self, value, color):
            self.value = value
            self.color = color

        def __str__(self):
            if self.value:
                return self.value
            return " "

        def reset(self):
            self.value = ""
            self.color = None

    def __init__(self, width=80, height=24, ignoreUnsupported=True):
        self.width = width
        self.height = height
        self.windowTop = 0
        self.windowBottom = height - 1
        # if we find an escape sequence that's not supported
        # this dictates whether the terminal errors, or silently ignores it
        self.ignoreUnsupported = ignoreUnsupported

        # this is 'backwards' from normal
        # (i.e. to get the char at x,y you do self.terminal[y][x])
        # the purpose of this is to make pushing full lines up easier
        self.terminal = [
            [self.Character("", None) for _ in range(self.width)]
            for __ in range(self.height)]
        self.cursorPosition = self.Position(0, 0)
        self.savedCursorPosition = self.Position(-1, -1)

        # this represents default coloring,
        # so set it as the current color from the start
        self.currentColor = EscapeSequence([0], "m")

    def __str__(self):
        return self.get_text()

    def parse_sequence(self, string):
        match = TerminalBuffer._parser.match(string)
        if not match:
            match = TerminalBuffer._unknownParser.match(string)

            if not match:
                raise ParseException("Couldn't parse sequence:" + repr(string))

            # just make an unknown sequence
            return EscapeSequence(
                [],
                SequenceType.UNKNOWN.value,
                string[match.start():match.end()]
            ), string[match.end():]

        data, char = match.groups()

        if not data:
            data = []
        else:
            data = [int(x) for x in data.split(';')]

        result = EscapeSequence(data, char), string[match.end():]
        return result

    def get_next_sequence(self, string):
        while string:
            # if the string is an escape sequence,
            # we need to construct the sequence based on the following chars
            if string[0] == '\x1b':
                # parse_sequence handles truncating the string for iteration
                seq, string = self.parse_sequence(string)
                yield seq

            # if the string is a normal string, just yield
            else:
                yield string[0]
                # slice the string for the next iteration
                string = string[1:]

    def input(self, string):
        log_dict = {}
        for val in self.get_next_sequence(string):
            if isinstance(val, EscapeSequence):
                self.apply_sequence(val, log_dict)
            else:
                # handle special case characters
                if val == '\r':
                    self.cursorPosition.x = 0
                elif val == '\b':
                    self.move_cursor(-1, 0, True)
                elif val == '\n':
                    self.move_cursor(0, 1, True)
                elif val == '\x0f' or val == '\x00':
                    pass
                # standard characters
                else:
                    self.terminal[self.cursorPosition.y][self.cursorPosition.x] \
                        = self.Character(val, self.currentColor)
                    self.move_cursor(1, 0, True)

        if len(log_dict):
            log.warning("Ignored sequence counts: " + str(log_dict))

    def get_text(self, x=0, y=0, w=0, h=0, color=False):<commit_msg>pep8 for realz<commit_after>from enum import Enum
from collections import namedtuple
import re
import logging

log = logging.getLogger(__name__)


class SequenceType(Enum):
    CURSOR_UP = "A"
    CURSOR_DOWN = "B"
    CURSOR_FORWARD = "C"
    CURSOR_BACK = "D"
    CURSOR_NEXT_LINE = "E"
    CURSOR_PREVIOUS_LINE = "F"
    CURSOR_HORIZONTAL_ABSOLUTE = "G"
    CURSOR_POSITION = "H"
    CURSOR_VERTICAL_ABSOLUTE = "d"
    ERASE_IN_DISPLAY = "J"
    ERASE_IN_LINE = "K"
    ERASE_CHARACTER = "X"
    SCROLL_UP = "S"
    SCROLL_DOWN = "T"
    HORIZONTAL_VERTICAL_POSITION = "f"
    SELECT_GRAPHIC_RENDITION = "m"
    SAVE_CURSOR_POSITION = "s"
    RESTORE_CURSOR_POSITION = "u"
    ENABLE_SETTING = "h"
    DISABLE_SETTING = "l"
    SET_WINDOW = "r"
    UNKNOWN = "ZZZ"


class ParseException(Exception):
    pass


class EscapeSequence():
    # list of escape sequences that use default values of 0
    # instead of the standard of 1
    _default0 = [SequenceType.ERASE_IN_DISPLAY, SequenceType.ERASE_IN_LINE,
                 SequenceType.SELECT_GRAPHIC_RENDITION]
    # list of escape sequences that include an extra '?' in the string
    _hasQMark = [SequenceType.ENABLE_SETTING, SequenceType.DISABLE_SETTING]

    def __init__(self, data, endingLetter, unknownSeq=None):
        self.data = data
        # save a copy in case it's unknown
        self.unknownSequence = unknownSeq
        try:
            self.sequenceType = SequenceType(endingLetter)
        except BaseException:
            # technically, this should have been created using UNKNOWN type
            # assume it can be recreated 'normally'
            self.sequenceType = SequenceType.UNKNOWN
            self.unknownSequence = "\x1b[" + ";".join(
                str(x) for x in self.data) + endingLetter

    def __str__(self):
        # just return the string represnting this sequence
        if self.unknownSequence:
            return self.unknownSequence
        qMark = ""
        if self.sequenceType in EscapeSequence._hasQMark:
            qMark = "?"
        return "\x1b[" + qMark + ";".join(
            str(x) for x in self.data) + self.sequenceType.value

    def __repr__(self):
        return repr(str(self))

    def get_data(self, index):
        # this is just a wrapper to handle default values
        if not isinstance(index, int):
            raise TypeException("index must be an integer")
        if index < len(self.data):
            return self.data[index]
        else:
            # but there are some special cases,
            # because who needs consistency anyways?
            if self.sequenceType in EscapeSequence._default0:
                return 0
            else:
                return 1


class TerminalBuffer():
    _parser = re.compile(r"^\x1b\[\??([\d;]*)(\w)")
    # secondary parser for other sequences that are sufficiently different
    # these commonly don't have data, or don't use nums exclusively for data
    _unknownParser = re.compile(r"^\x1b[\(\)\=\>\%]?([\w\d])?")

    class Position:
        def __init__(self, x, y):
            self.x = x
            self.y = y

    class Character:
        def __init__(self, value, color):
            self.value = value
            self.color = color

        def __str__(self):
            if self.value:
                return self.value
            return " "

        def reset(self):
            self.value = ""
            self.color = None

    def __init__(self, width=80, height=24, ignoreUnsupported=True):
        self.width = width
        self.height = height
        self.windowTop = 0
        self.windowBottom = height - 1
        # if we find an escape sequence that's not supported
        # this dictates whether the terminal errors, or silently ignores it
        self.ignoreUnsupported = ignoreUnsupported

        # this is 'backwards' from normal
        # (i.e. to get the char at x,y you do self.terminal[y][x])
        # the purpose of this is to make pushing full lines up easier
        self.terminal = [
            [self.Character("", None) for _ in range(self.width)]
            for __ in range(self.height)]
        self.cursorPosition = self.Position(0, 0)
        self.savedCursorPosition = self.Position(-1, -1)

        # this represents default coloring,
        # so set it as the current color from the start
        self.currentColor = EscapeSequence([0], "m")

    def __str__(self):
        return self.get_text()

    def parse_sequence(self, string):
        match = TerminalBuffer._parser.match(string)
        if not match:
            match = TerminalBuffer._unknownParser.match(string)

            if not match:
                raise ParseException("Couldn't parse sequence:" + repr(string))

            # just make an unknown sequence
            return EscapeSequence(
                [],
                SequenceType.UNKNOWN.value,
                string[match.start():match.end()]
            ), string[match.end():]

        data, char = match.groups()

        if not data:
            data = []
        else:
            data = [int(x) for x in data.split(';')]

        result = EscapeSequence(data, char), string[match.end():]
        return result

    def get_next_sequence(self, string):
        while string:
            # if the string is an escape sequence,
            # we need to construct the sequence based on the following chars
            if string[0] == '\x1b':
                # parse_sequence handles truncating the string for iteration
                seq, string = self.parse_sequence(string)
                yield seq

            # if the string is a normal string, just yield
            else:
                yield string[0]
                # slice the string for the next iteration
                string = string[1:]

    def input(self, string):
        log_dict = {}
        for val in self.get_next_sequence(string):
            if isinstance(val, EscapeSequence):
                self.apply_sequence(val, log_dict)
            else:
                # handle special case characters
                if val == '\r':
                    self.cursorPosition.x = 0
                elif val == '\b':
                    self.move_cursor(-1, 0, True)
                elif val == '\n':
                    self.move_cursor(0, 1, True)
                elif val == '\x0f' or val == '\x00':
                    pass
                # standard characters
                else:
                    self.terminal[self.cursorPosition.y] \
                        [self.cursorPosition.x] \
                        = self.Character(val, self.currentColor)
                    self.move_cursor(1, 0, True)

        if len(log_dict):
            log.warning("Ignored sequence counts: " + str(log_dict))

    def get_text(self, x=0, y=0, w=0, h=0, color=False):
{'fraction_non_alphanumeric': 0.09268256772673734, 'fraction_numerical': 0.005815665488810365, 'mean_word_length': 1.8793980500211955, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5180947', 'n_tokens_mistral': 3985, 'n_tokens_neox': 3765, 'n_words': 1112}