<commit_before>import os
import base64
import binascii
from collections import namedtuple

import hexdump
import intervaltree

from PyQt5 import uic
from PyQt5.QtGui import QIcon
from PyQt5.QtGui import QBrush
from PyQt5.QtGui import QPixmap
from PyQt5.QtGui import QMouseEvent
from PyQt5.QtGui import QKeySequence
from PyQt5.QtGui import QFontDatabase
import PyQt5.QtCore as QtCore
from PyQt5.QtCore import Qt
from PyQt5.QtCore import QSize
from PyQt5.QtCore import QMimeData
from PyQt5.QtCore import pyqtSignal
from PyQt5.QtCore import QModelIndex
from PyQt5.QtCore import QItemSelection
from PyQt5.QtCore import QItemSelectionModel
from PyQt5.QtCore import QAbstractTableModel
from PyQt5.QtWidgets import QMenu
from PyQt5.QtWidgets import QAction
from PyQt5.QtWidgets import QTableView
from PyQt5.QtWidgets import QSizePolicy
from PyQt5.QtWidgets import QApplication
from PyQt5.QtWidgets import QInputDialog
from PyQt5.QtWidgets import QItemDelegate
from PyQt5.QtWidgets import QAbstractItemView

from common import h
from common import LoggingObject
from tablecellstylemodels import row_start_index
from tablecellstylemodels import row_end_index
from tablecellstylemodels import row_number
from tablecellstylemodels import ROLE_BORDER
from tablecellstylemodels import ColorModel
from tablecellstylemodels import BorderModel

NamedColor = namedtuple("NamedColor", ["name", "qcolor"])
QT_COLORS = (
        NamedColor("red", Qt.red),
        NamedColor("green", Qt.green),
        NamedColor("blue", Qt.blue),
        NamedColor("black", Qt.black),
        NamedColor("dark red", Qt.darkRed),
        NamedColor("dark green", Qt.darkGreen),
        NamedColor("dark blue", Qt.darkBlue),
        NamedColor("cyan", Qt.cyan),
        NamedColor("magenta", Qt.magenta),
        NamedColor("yellow", Qt.yellow),
        NamedColor("gray", Qt.gray),
        NamedColor("dark cyan", Qt.darkCyan),
        NamedColor("dark magenta", Qt.darkMagenta),
        NamedColor("dark yellow", Qt.darkYellow),
        NamedColor("dark gray", Qt.darkGray),
        NamedColor("light gray", Qt.lightGray),
)


def make_color_icon(color):
        pixmap = QPixmap(10, 10)
        pixmap.fill(color)
        return QIcon(pixmap)


class HexItemDelegate(QItemDelegate):
    def __init__(self, model, parent, *args):
        super(HexItemDelegate, self).__init__(parent)
        self._model = model

    def paint(self, qpainter, option, qindex):
        super(HexItemDelegate, self).paint(qpainter, option, qindex)
        border = self._model.data(qindex, ROLE_BORDER)

        if border is None:
            return

        qpainter.setPen(border.theme.color)
        r = option.rect
        if border.top:
            qpainter.drawLine(r.topLeft(), r.topRight())

        if border.bottom:
            qpainter.drawLine(r.bottomLeft(), r.bottomRight())

        if border.left:
            qpainter.drawLine(r.topLeft(), r.bottomLeft())

        if border.right:
            qpainter.drawLine(r.topRight(), r.bottomRight())


class HexTableModel(QAbstractTableModel):
    FILTER = ''.join([(len(repr(chr(x)))==3) and chr(x) or '.' for x in range(256)])
<commit_msg>hexview: char pane now understands backslashes<commit_after>import os
import base64
import binascii
from collections import namedtuple

import hexdump
import intervaltree

from PyQt5 import uic
from PyQt5.QtGui import QIcon
from PyQt5.QtGui import QBrush
from PyQt5.QtGui import QPixmap
from PyQt5.QtGui import QMouseEvent
from PyQt5.QtGui import QKeySequence
from PyQt5.QtGui import QFontDatabase
import PyQt5.QtCore as QtCore
from PyQt5.QtCore import Qt
from PyQt5.QtCore import QSize
from PyQt5.QtCore import QMimeData
from PyQt5.QtCore import pyqtSignal
from PyQt5.QtCore import QModelIndex
from PyQt5.QtCore import QItemSelection
from PyQt5.QtCore import QItemSelectionModel
from PyQt5.QtCore import QAbstractTableModel
from PyQt5.QtWidgets import QMenu
from PyQt5.QtWidgets import QAction
from PyQt5.QtWidgets import QTableView
from PyQt5.QtWidgets import QSizePolicy
from PyQt5.QtWidgets import QApplication
from PyQt5.QtWidgets import QInputDialog
from PyQt5.QtWidgets import QItemDelegate
from PyQt5.QtWidgets import QAbstractItemView

from common import h
from common import LoggingObject
from tablecellstylemodels import row_start_index
from tablecellstylemodels import row_end_index
from tablecellstylemodels import row_number
from tablecellstylemodels import ROLE_BORDER
from tablecellstylemodels import ColorModel
from tablecellstylemodels import BorderModel

NamedColor = namedtuple("NamedColor", ["name", "qcolor"])
QT_COLORS = (
        NamedColor("red", Qt.red),
        NamedColor("green", Qt.green),
        NamedColor("blue", Qt.blue),
        NamedColor("black", Qt.black),
        NamedColor("dark red", Qt.darkRed),
        NamedColor("dark green", Qt.darkGreen),
        NamedColor("dark blue", Qt.darkBlue),
        NamedColor("cyan", Qt.cyan),
        NamedColor("magenta", Qt.magenta),
        NamedColor("yellow", Qt.yellow),
        NamedColor("gray", Qt.gray),
        NamedColor("dark cyan", Qt.darkCyan),
        NamedColor("dark magenta", Qt.darkMagenta),
        NamedColor("dark yellow", Qt.darkYellow),
        NamedColor("dark gray", Qt.darkGray),
        NamedColor("light gray", Qt.lightGray),
)


def make_color_icon(color):
        pixmap = QPixmap(10, 10)
        pixmap.fill(color)
        return QIcon(pixmap)


class HexItemDelegate(QItemDelegate):
    def __init__(self, model, parent, *args):
        super(HexItemDelegate, self).__init__(parent)
        self._model = model

    def paint(self, qpainter, option, qindex):
        super(HexItemDelegate, self).paint(qpainter, option, qindex)
        border = self._model.data(qindex, ROLE_BORDER)

        if border is None:
            return

        qpainter.setPen(border.theme.color)
        r = option.rect
        if border.top:
            qpainter.drawLine(r.topLeft(), r.topRight())

        if border.bottom:
            qpainter.drawLine(r.bottomLeft(), r.bottomRight())

        if border.left:
            qpainter.drawLine(r.topLeft(), r.bottomLeft())

        if border.right:
            qpainter.drawLine(r.topRight(), r.bottomRight())


class HexTableModel(QAbstractTableModel):
    FILTER = ''.join([(len(repr(chr(x)))==3 or chr(x) == "\\") and chr(x) or '.' for x in range(256)])

{'fraction_non_alphanumeric': 0.09216952891129668, 'fraction_numerical': 0.010713723018749016, 'mean_word_length': 3.7801204819277108, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4694907', 'n_tokens_mistral': 2045, 'n_tokens_neox': 1938, 'n_words': 513}