<commit_before>import hexdump

from PyQt5 import uic
from PyQt5.QtGui import QBrush
from PyQt5.QtGui import QPalette
from PyQt5.QtGui import QFontDatabase
from PyQt5.QtGui import QTextDocument
from PyQt5.QtCore import Qt
from PyQt5.QtCore import pyqtSignal
from PyQt5.QtCore import QAbstractTableModel
from PyQt5.QtCore import QIdentityProxyModel
from PyQt5.QtCore import QItemSelection
from PyQt5.QtCore import QModelIndex
from PyQt5.QtCore import QItemSelectionModel
from PyQt5.QtWidgets import QWidget
from PyQt5.QtWidgets import QTextEdit
from PyQt5.QtWidgets import QGridLayout
from PyQt5.QtWidgets import QApplication

import os.path, sys
sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir))
from common import h
from common import LoggingObject


class HexTableModel(QAbstractTableModel):
    FILTER = ''.join([(len(repr(chr(x)))==3) and chr(x) or '.' for x in range(256)])
    colorChanged = pyqtSignal()
    def __init__(self, buf, parent=None, *args):
        super(HexTableModel, self).__init__(parent, *args)
        self._buf = buf
        self._colorStart = None
        self._colorEnd = None

    def rowCount(self, parent):
        if len(self._buf) % 0x10 != 0:
            return (len(self._buf) // 0x10) + 1
        else:
            return (len(self._buf) // 0x10)

    def columnCount(self, parent):
        return 0x21

    def data(self, index, role):
        if not index.isValid():
            return None
        elif role == Qt.BackgroundRole:
            if self._colorStart is None or self._colorEnd is None:
                return None
            elif self._colorStart <= self.qindex2index(index) < self._colorEnd:
                # BUG: need to highlight char entries, too
                color = QApplication.palette().color(QPalette.Highlight)
                return QBrush(color)
            else:
                return None
        elif role == Qt.DisplayRole:
            if index.column() == 0x10:
                return ""
            if index.column() > 0x10:
                c = self._buf[self.qindex2index(index)]
                return chr(ord(c)).translate(HexTableModel.FILTER)
            else:
                c = ord(self._buf[self.qindex2index(index)])
                return "%02x" % (c)

    def headerData(self, section, orientation, role):
        if role != Qt.DisplayRole:
            return None
        elif orientation == Qt.Horizontal:
            if section < 0x10:
                return "%01X" % (section)
            else:
                return ""
        elif orientation == Qt.Vertical:
            return "%04X" % (section * 0x10)
        else:
            return None

    def colorRange(self, start, end):
        self.clearColor()
        self._colorStart = start
        self._colorEnd = end
        for i in xrange(start, end):
            qib = self.index2qindexb(i)
            qic = self.index2qindexb(i)
            self.dataChanged.emit(qib, qib)
            self.dataChanged.emit(qic, qic)
        self.colorChanged.emit()

    def clearColor(self):
        oldstart = self._colorStart
        oldend = self._colorEnd
        self._colorStart = None
        self._colorEnd = None
        if oldstart is not None and oldend is not None:
            for i in xrange(oldstart, oldend):
                qib = self.index2qindexb(i)
                qic = self.index2qindexb(i)
                self.dataChanged.emit(qib, qib)
                self.dataChanged.emit(qic, qic)
        self.colorChanged.emit()

    def qindex2index(self, index):
        if index.row() > 0x10:
            return (0x10 * index.row() - 0x11) + index.column()
        else:
            return (0x10 * index.row()) + index.column()

    def index2qindexb(self, index):
        """ for the byte side """
        r = index // 0x10
        c = index % 0x10
        return self.index(r, c)

    def index2qindexc(self, index):
        """ for the char side """
        r = (index // 0x10)
        c = index % 0x10 + 0x11
        return self.index(r, c)


class HexItemSelectionModel(QItemSelectionModel):
    def isInBytesSide(self, qindex):
        return qindex.column() < 0x10

    def isInCharsSide(self, qindex):
        return qindex.column() > 0x10

    # TODO: currently ignoring selectionFlags...<commit_msg>hexview: fix most coloring artifacts, was due to incorrect index computation<commit_after>import hexdump

from PyQt5 import uic
from PyQt5.QtGui import QBrush
from PyQt5.QtGui import QPalette
from PyQt5.QtGui import QFontDatabase
from PyQt5.QtGui import QTextDocument
from PyQt5.QtCore import Qt
from PyQt5.QtCore import pyqtSignal
from PyQt5.QtCore import QAbstractTableModel
from PyQt5.QtCore import QIdentityProxyModel
from PyQt5.QtCore import QItemSelection
from PyQt5.QtCore import QModelIndex
from PyQt5.QtCore import QItemSelectionModel
from PyQt5.QtWidgets import QWidget
from PyQt5.QtWidgets import QTextEdit
from PyQt5.QtWidgets import QGridLayout
from PyQt5.QtWidgets import QApplication

import os.path, sys
sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir))
from common import h
from common import LoggingObject


class HexTableModel(QAbstractTableModel):
    FILTER = ''.join([(len(repr(chr(x)))==3) and chr(x) or '.' for x in range(256)])
    colorChanged = pyqtSignal()
    def __init__(self, buf, parent=None, *args):
        super(HexTableModel, self).__init__(parent, *args)
        self._buf = buf
        self._colorStart = None
        self._colorEnd = None

    def rowCount(self, parent):
        if len(self._buf) % 0x10 != 0:
            return (len(self._buf) // 0x10) + 1
        else:
            return (len(self._buf) // 0x10)

    def columnCount(self, parent):
        return 0x21

    def data(self, index, role):
        if not index.isValid():
            return None
        elif self.qindex2index(index) >= len(self._buf):
            return None
        elif role == Qt.BackgroundRole:
            if self._colorStart is None or self._colorEnd is None:
                return None
            elif self._colorStart <= self.qindex2index(index) < self._colorEnd:
                color = QApplication.palette().color(QPalette.Highlight)
                return QBrush(color)
            else:
                return None
        elif role == Qt.DisplayRole:
            if index.column() == 0x10:
                return ""
            if index.column() > 0x10:
                c = self._buf[self.qindex2index(index)]
                return chr(ord(c)).translate(HexTableModel.FILTER)
            else:
                c = ord(self._buf[self.qindex2index(index)])
                return "%02x" % (c)

    def headerData(self, section, orientation, role):
        if role != Qt.DisplayRole:
            return None
        elif orientation == Qt.Horizontal:
            if section < 0x10:
                return "%01X" % (section)
            else:
                return ""
        elif orientation == Qt.Vertical:
            return "%04X" % (section * 0x10)
        else:
            return None

    def colorRange(self, start, end):
        self.clearColor()
        self._colorStart = start
        self._colorEnd = end
        for i in xrange(start, end):
            # mark data changed to encourage re-rendering of cell
            qib = self.index2qindexb(i)
            qic = self.index2qindexc(i)
            self.dataChanged.emit(qib, qib)
            self.dataChanged.emit(qic, qic)
        self.colorChanged.emit()

    def clearColor(self):
        oldstart = self._colorStart
        oldend = self._colorEnd
        self._colorStart = None
        self._colorEnd = None
        if oldstart is not None and oldend is not None:
            # mark data changed to encourage re-rendering of cell
            for i in xrange(oldstart, oldend):
                qib = self.index2qindexb(i)
                qic = self.index2qindexc(i)
                self.dataChanged.emit(qib, qib)
                self.dataChanged.emit(qic, qic)
        self.colorChanged.emit()

    def qindex2index(self, index):
        if index.column() > 0x10:
            return (0x10 * index.row()) + index.column() - 0x11
        else:
            return (0x10 * index.row()) + index.column()

    def index2qindexb(self, index):
        """ for the byte side """
        r = index // 0x10
        c = index % 0x10
        return self.index(r, c)

    def index2qindexc(self, index):
        """ for the char side """
        r = (index // 0x10)
        c = index % 0x10 + 0x11
        return self.index(r, c)


class HexItemSelectionModel(QItemSelectionModel):
    def isInBytesSide(self, qindex):
        return qindex.column() < 0x10

    def isInCharsSide(self, qindex):
        return qindex.column() > 0x10

    # TODO: currently ignoring selectionFlags...
{'fraction_non_alphanumeric': 0.09625425652667423, 'fraction_numerical': 0.021679909194097617, 'mean_word_length': 2.1167315175097277, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2302512', 'n_tokens_mistral': 2776, 'n_tokens_neox': 2612, 'n_words': 705}