<commit_before>import logging

from terminal import Terminal
from term import TextAttribute, TextMode, reserve, clone_attr
from term_char_width import char_width
class TerminalGUI(Terminal):
    def __init__(self, cfg):
        Terminal.__init__(self, cfg)

        self.term_widget = None
        self.session = None

        self.lines = []
        self.line_options = []
        self.col = 0
        self.row = 0
        self.remain_buffer = []

        self.last_line_option_row = -1
        self.last_line_option_col = -1
        self.cur_line_option = TextAttribute(None, None, None)
        self.saved_lines, self.saved_line_options, self.saved_cursor = [], [], (0, 0)
        self.bold_mode = False
        self.scroll_region = None

        self.view_history_begin = None
        self.history_lines = []
        self.history_line_options = []

    def get_line(self, row):
        reserve(self.lines, row + 1, [])

        self.get_line_option(row)

        return self.lines[row]

    def get_cur_line(self):
        line = self.get_line(self.row)

        reserve(line, self.col + 1, ' ')

        return line

    def wrap_line(self, c, insert):
        self.col = 0
        self.cursor_down(None)
        self._save_buffer(c, insert)

    def save_buffer(self, c, insert = False):
        line = self.get_cur_line()

        self.get_cur_option()
        line_option = self.get_cur_line_option()

        #take care utf_8
        self.remain_buffer.append(c)

        c = ''.join(self.remain_buffer).decode('utf_8', errors='ignore')

        if len(c) == 0:
            logging.getLogger('term_gui').debug('remain_buffer found:{}'.format(map(ord, self.remain_buffer)))
            return

        self.remain_buffer = []

        w = char_width(c)

        if w == 0 or w == -1:
            logging.getLogger('term_gui').warning(u'save buffer get a invalid width char: w= {}, c={}'.format(w, c))

        if len(c.encode('utf_8')) > 1 and w > 1:
            c += '\000'

        if self.cfg.debug_more:
            logging.getLogger('term_gui').debug(u'save buffer width:{},{},{},len={}'.format(self.col, self.row, w, len(c)))

        if insert:
            if len(line) + len(c) > self.get_cols():
                wrap_c = line[self.get_cols() - len(line) - len(c):]

                if wrap_c[0] == '\000':
                    wrap_c = line[self.get_cols() - len(c) - 1]

                two_bytes = len(wrap_c)
                wrap_c = wrap_c.replace('\000', '')

                line = line[:self.get_cols() - two_bytes]
                self._save_buffer(c, insert)
                self.wrap_line(wrap_c, insert)
            else:
                self._save_buffer(c, insert)
        else:
            if self.col + len(c) > self.get_cols():
                #wrap
                self.wrap_line(c, insert)
            else:
                self._save_buffer(c, insert)


    def _save_buffer(self, c, insert):
        line = self.get_cur_line()

        self.get_cur_option()
        line_option = self.get_cur_line_option()

        if self.cfg.debug_more:
            logging.getLogger('term_gui').debug(u'save buffer:{},{},{},len={}'.format(self.col, self.row, c, len(c)))

        if insert:
            line.insert(self.col, c[0])
            #update line option
            line_option.insert(self.col, clone_attr(self.cur_line_option))

            if len(c) > 1:
                line.insert(self.col + 1, c[1])
                #update line option
                line_option.insert(self.col + 1, clone_attr(self.cur_line_option))
        else:
            reserve(line, self.col + len(c), ' ')
            reserve(line_option, self.col + len(c), None)
            line[self.col] = c[0]
            line_option[self.col] = clone_attr(self.cur_line_option)
            self.col += 1
            if len(c) > 1:
                line[self.col] = c[1]
                line_option[self.col] = clone_attr(self.cur_line_option)
                self.col += 1

    def get_rows(self):
        return self.term_widget.visible_rows

    def get_cols(self):
        cols = self.term_widget.visible_cols

        return cols

    def get_history_text(self):
        return self.history_lines + self.lines, self.history_line_options + self.line_options

    def get_text(self):
        if self.view_history_begin is not None:
            l, o = self.get_history_text()
            lines = l[self.view_history_begin: self.view_history_begin + self.get_rows()]
            line_options = o[self.view_history_begin: self.view_history_begin + self.get_rows()]
            return lines, line_options

        if len(self.lines) <= self.get_rows():
            return self.lines + [self.create_new_line()] * (self.get_rows() - len(self.lines)), self.line_options + [self.create_new_line_option()] * (self.get_rows() - len(self.lines))
        else:
            logging.getLogger('get_text').debug('{}={}'.format(len(self.lines), self.get_rows))
            lines = self.lines[len(self.lines) - self.get_rows():]
            line_options = self.line_options[len(self.lines) - self.get_rows():]
            return lines, line_options

    def output_normal_data(self, c, insert = False):
        if c == '\x1b':
            logging.getLogger('term_gui').error('normal data has escape char')
            sys.exit(1)

        self.save_buffer(c, insert)

    def output_status_line_data(self, c):
        if c == '\x1b':
            logging.getLogger('term_gui').error('status line data has escape char')
            sys.exit(1)
        pass

    def get_cursor(self):
        if len(self.lines) <= self.get_rows():
            return (self.col, self.row)
        else:
            return (self.col, self.row - len(self.lines) + self.get_rows())

    def set_cursor(self, col, row):
        self.col = col
        if len(self.lines) <= self.get_rows():
            self.row = row
        else:
            self.row = row + len(self.lines) - self.get_rows()

        logging.getLogger('term_gui').debug('termianl cursor:{}, {}'.format(self.col, self.row));

    def cursor_right(self, context):
        logging.getLogger('term_gui').debug('cursor right:{}, {}'.format(self.col, self.row));<commit_msg>when insert char to line, col,row should not move even wrap, now fixed<commit_after>import logging

from terminal import Terminal
from term import TextAttribute, TextMode, reserve, clone_attr
from term_char_width import char_width
class TerminalGUI(Terminal):
    def __init__(self, cfg):
        Terminal.__init__(self, cfg)

        self.term_widget = None
        self.session = None

        self.lines = []
        self.line_options = []
        self.col = 0
        self.row = 0
        self.remain_buffer = []

        self.last_line_option_row = -1
        self.last_line_option_col = -1
        self.cur_line_option = TextAttribute(None, None, None)
        self.saved_lines, self.saved_line_options, self.saved_cursor = [], [], (0, 0)
        self.bold_mode = False
        self.scroll_region = None

        self.view_history_begin = None
        self.history_lines = []
        self.history_line_options = []

    def get_line(self, row):
        reserve(self.lines, row + 1, [])

        self.get_line_option(row)

        return self.lines[row]

    def get_cur_line(self):
        line = self.get_line(self.row)

        reserve(line, self.col + 1, ' ')

        return line

    def wrap_line(self, c, insert):
        save_col, save_row = self.col, self.row
        
        self.col = 0
        self.cursor_down(None)
        for cc in c:
            if cc == '\000':
                continue
            self._save_buffer(c, insert)

        if insert:
            self.col, self.row = save_col, save_row

    def save_buffer(self, c, insert = False):
        line = self.get_cur_line()

        self.get_cur_option()
        line_option = self.get_cur_line_option()

        #take care utf_8
        self.remain_buffer.append(c)

        c = ''.join(self.remain_buffer).decode('utf_8', errors='ignore')

        if len(c) == 0:
            logging.getLogger('term_gui').debug('remain_buffer found:{}'.format(map(ord, self.remain_buffer)))
            return

        self.remain_buffer = []

        w = char_width(c)

        if w == 0 or w == -1:
            logging.getLogger('term_gui').warning(u'save buffer get a invalid width char: w= {}, c={}'.format(w, c))

        if len(c.encode('utf_8')) > 1 and w > 1:
            c += '\000'

        if self.cfg.debug_more:
            logging.getLogger('term_gui').debug(u'save buffer width:{},{},{},len={}, line_len={}, cols={}'.format(self.col, self.row, w, len(c), len(line), self.get_cols()))

        if insert:
            if len(line) + len(c) > self.get_cols():
                wrap_c = line[self.get_cols() - len(line) - len(c):]

                if wrap_c[0] == '\000':
                    wrap_c = line[self.get_cols() - len(line) - len(c) - 1:]

                two_bytes = len(wrap_c)

                line = line[:self.get_cols() - two_bytes]
                
                if self.cfg.debug_more:
                    logging.getLogger('term_gui').debug(u'save buffer wrap:c=[{}], wrap=[{}]'.format(c, wrap_c))
                    
                self._save_buffer(c, insert)
                self.wrap_line(wrap_c, insert)
            else:
                self._save_buffer(c, insert)
        else:
            if self.col + len(c) > self.get_cols():
                #wrap
                self.wrap_line(c, insert)
            else:
                self._save_buffer(c, insert)


    def _save_buffer(self, c, insert):
        line = self.get_cur_line()

        self.get_cur_option()
        line_option = self.get_cur_line_option()

        if self.cfg.debug_more:
            logging.getLogger('term_gui').debug(u'save buffer:{},{},{},len={}'.format(self.col, self.row, c, len(c)))

        if insert:
            line.insert(self.col, c[0])
            #update line option
            line_option.insert(self.col, clone_attr(self.cur_line_option))

            if len(c) > 1:
                line.insert(self.col + 1, c[1])
                #update line option
                line_option.insert(self.col + 1, clone_attr(self.cur_line_option))
        else:
            reserve(line, self.col + len(c), ' ')
            reserve(line_option, self.col + len(c), None)
            line[self.col] = c[0]
            line_option[self.col] = clone_attr(self.cur_line_option)
            self.col += 1
            if len(c) > 1:
                line[self.col] = c[1]
                line_option[self.col] = clone_attr(self.cur_line_option)
                self.col += 1

    def get_rows(self):
        return self.term_widget.visible_rows

    def get_cols(self):
        cols = self.term_widget.visible_cols

        return cols

    def get_history_text(self):
        return self.history_lines + self.lines, self.history_line_options + self.line_options

    def get_text(self):
        if self.view_history_begin is not None:
            l, o = self.get_history_text()
            lines = l[self.view_history_begin: self.view_history_begin + self.get_rows()]
            line_options = o[self.view_history_begin: self.view_history_begin + self.get_rows()]
            return lines, line_options

        if len(self.lines) <= self.get_rows():
            return self.lines + [self.create_new_line()] * (self.get_rows() - len(self.lines)), self.line_options + [self.create_new_line_option()] * (self.get_rows() - len(self.lines))
        else:
            logging.getLogger('get_text').debug('{}={}'.format(len(self.lines), self.get_rows))
            lines = self.lines[len(self.lines) - self.get_rows():]
            line_options = self.line_options[len(self.lines) - self.get_rows():]
            return lines, line_options

    def output_normal_data(self, c, insert = False):
        if c == '\x1b':
            logging.getLogger('term_gui').error('normal data has escape char')
            sys.exit(1)

        try:
            for cc in c:
                self.save_buffer(cc, insert)
        except:
            logging.getLogger('term_gui').exception('save buffer failed')

    def output_status_line_data(self, c):
        if c == '\x1b':
            logging.getLogger('term_gui').error('status line data has escape char')
            sys.exit(1)
        pass

    def get_cursor(self):
        if len(self.lines) <= self.get_rows():
            return (self.col, self.row)
        else:
            return (self.col, self.row - len(self.lines) + self.get_rows())

    def set_cursor(self, col, row):
        self.col = col
        if len(self.lines) <= self.get_rows():
            self.row = row
        else:
            self.row = row + len(self.lines) - self.get_rows()

        logging.getLogger('term_gui').debug('termianl cursor:{}, {}'.format(self.col, self.row));

    def cursor_right(self, context):
        logging.getLogger('term_gui').debug('cursor right:{}, {}'.format(self.col, self.row));
{'fraction_non_alphanumeric': 0.13226699477083975, 'fraction_numerical': 0.0064595509074131035, 'mean_word_length': 2.0357142857142856, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4200285', 'n_tokens_mistral': 4212, 'n_tokens_neox': 4046, 'n_words': 802}