<commit_before>"""
Welcome to the encoding dance!

In a nutshell, text columns are actually a proxy class for byte columns,
which just encode/decodes contents.
"""

from mitmproxy.tools.console import signals
from mitmproxy.tools.console.grideditor import col_bytes


class Column(col_bytes.Column):
    def __init__(self, heading, encoding="utf8", errors="surrogateescape"):
        super().__init__(heading)
        self.encoding_args = encoding, errors

    def Display(self, data):
        return TDisplay(data, self.encoding_args)

    def Edit(self, data):
        return TEdit(data, self.encoding_args)

    def blank(self):
        return ""


# This is the same for both edit and display.
class EncodingMixin:
    def __init__(self, data, encoding_args):
        self.encoding_args = encoding_args
        super().__init__(data.encode(*self.encoding_args))

    def get_data(self):
        data = super().get_data()
        try:
            return data.decode(*self.encoding_args)
        except ValueError:
            signals.status_message.send(
                self,
                message="Invalid encoding.",
                expire=1000
            )
            raise


# urwid forces a different name for a subclass.
class TDisplay(EncodingMixin, col_bytes.Display):
    pass
<commit_msg>Fixed a bug with bytes encoding error<commit_after>"""
Welcome to the encoding dance!

In a nutshell, text columns are actually a proxy class for byte columns,
which just encode/decodes contents.
"""

from mitmproxy.tools.console import signals
from mitmproxy.tools.console.grideditor import col_bytes


class Column(col_bytes.Column):
    def __init__(self, heading, encoding="utf8", errors="surrogateescape"):
        super().__init__(heading)
        self.encoding_args = encoding, errors

    def Display(self, data):
        return TDisplay(data, self.encoding_args)

    def Edit(self, data):
        return TEdit(data, self.encoding_args)

    def blank(self):
        return ""


# This is the same for both edit and display.
class EncodingMixin:
    def __init__(self, data, encoding_args):
        self.encoding_args = encoding_args
        super().__init__(data.__str__().encode(*self.encoding_args))

    def get_data(self):
        data = super().get_data()
        try:
            return data.decode(*self.encoding_args)
        except ValueError:
            signals.status_message.send(
                self,
                message="Invalid encoding.",
                expire=1000
            )
            raise


# urwid forces a different name for a subclass.
class TDisplay(EncodingMixin, col_bytes.Display):
    pass

{'fraction_non_alphanumeric': 0.09032501889644746, 'fraction_numerical': 0.003779289493575208, 'mean_word_length': 2.728169014084507, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3425709', 'n_tokens_mistral': 744, 'n_tokens_neox': 717, 'n_words': 210}