<commit_before>from PyQt4.QtCore import pyqtSignal, QObject, Qt, QPointF, QRectF, QPoint, QSizeF, QEvent, QTimer, QLineF
from PyQt4.QtGui import QColor, QFont, QFontMetricsF, QPolygonF, QWidget, QScrollBar, QVBoxLayout, QHBoxLayout, \
                        QPainter, QBrush, QPalette, QPen, QApplication, QRegion, QLineEdit, QValidator, \
                        QTextEdit, QTextOption, QSizePolicy, QStyle, QStyleOptionFrameV2, QTextCursor, QTextDocument, \
                        QTextBlockFormat, QPlainTextDocumentLayout, QAbstractTextDocumentLayout, QTextCharFormat, \
                        QTextTableFormat, QRawFont
import math
import html
from hex.valuecodecs import IntegerCodec
from hex.formatters import IntegerFormatter
from hex.editor import DataSpan
from hex.proxystyle import ProxyStyle
import hex.encodings as encodings
import hex.utils as utils


# Why we need to make different model/view classes? Why not to use existing ones?
# These classes are specialized and optimized for our needs. But main cause is that standard model classes will not
# work with very big amount of data (especially on 32-bit machines) where number of rows exceeds 32-bit int limit.
# Another difference is that custom model class represents irregular array instead of tree-like structure Qt does.


class ModelIndex(object):
    def __init__(self, row=-1, column=-1, model=None, internal_data=None):
        self.__row = row
        self.__column = column
        self.__model = model
        self.__data = internal_data

    @property
    def row(self):
        """This value is always positive int for valid indexes, -1 otherwise"""
        return self.__row if self.valid else -1

    @property
    def column(self):
        """This value is always positive int for valid indexes, -1 otherwise"""
        return self.__column if self.valid else -1

    @property
    def model(self):
        """Returns model this index belongs to, None for invalid indexes"""
        return self.__model if self.valid else None

    @property
    def virtual(self):
        # virtual indexes does not represent real data, but used to represent empty cells in regular models.
        # Regular model has finite set of not-virtual indexes and can have infinite set of virtual ones
        return self.flags & ColumnModel.FlagVirtual

    @property
    def valid(self):
        """Return True if index is valid"""
        return self.__model is not None and self.__row >= 0 and self.__column >= 0

    @property
    def internalData(self):
        return self.__data if self.valid else None

    def data(self, role=Qt.DisplayRole):
        """Data for index"""
        return self.__model.indexData(self, role) if self.valid else None

    @property
    def flags(self):
        return self.__model.indexFlags(self) if self.valid else 0

    def __eq__(self, other):
        """Invalid index is not equal to any index"""
        if not isinstance(other, ModelIndex):
            return NotImplemented
        return (self.valid and other.valid and self.row == other.row and self.column == other.column
                and self.model is other.model)

    def __ne__(self, other):
        if not isinstance(other, ModelIndex):
            return NotImplemented
        return not self.__eq__(other)

    def __bool__(self):
        return self.valid

    def __isCompatible(self, other):
        return isinstance(other, ModelIndex) and self.valid and other.valid

    def __lt__(self, other):
        if not self.__isCompatible(other):
            return NotImplemented
        return self.row < other.row or (self.row == other.row and self.column < other.column)

    def __gt__(self, other):
        if not self.__isCompatible(other):
            return NotImplemented
        return self.row > other.row or (self.row == other.row and self.column > other.column)

    def __le__(self, other):
        if not self.__isCompatible(other):
            return NotImplemented
        return self.__lt__(other) or self.__eq__(other)

    def __ge__(self, other):
        if not self.__isCompatible(other):
            return NotImplemented
        return self.__gt__(other) or self.__eq__(other)

    @property
    def next(self):
        if not self.valid:
            return ModelIndex()

        next_index = self.model.index(self.row, self.column + 1)
        if not next_index:
            row = self.row + 1
            while not next_index and (self.model.rowCount() < 0 or row < self.model.rowCount()):
                next_index = self.model.index(row, 0)
                row += 1

        return next_index

    @property
    def previous(self):
        if not self.valid:
            return ModelIndex()

        prev_index = self.model.index(self.row, self.column - 1)
        if not prev_index:
            row = self.row - 1
            while not prev_index and row >= 0:
                prev_index = self.model.lastRowIndex(row)
                row -= 1

        return prev_index


class ColumnModel(QObject):
    EditorDataRole = Qt.UserRole + 1
    EditorPositionRole = Qt.UserRole + 2
    DataSizeRole = Qt.UserRole + 3

    FlagVirtual = 1
    FlagEditable = 2
    FlagModified = 4

    dataChanged = pyqtSignal(ModelIndex, ModelIndex)
    modelReset = pyqtSignal()

    def __init__(self, editor=None):
        QObject.__init__(self)

        if editor is not None:
            with editor.lock:
                self.__editor = editor
                self.__editor.dataChanged.connect(self.onEditorDataChanged)
                self.__editor.resized.connect(self.onEditorDataResized)
        else:
            self.__editor = None

    @property
    def editor(self):
        return self.__editor

    def rowCount(self):
        """Should return -1 if model has infinite number of rows."""
        raise NotImplementedError()

    def realRowCount(self):
        """Should return positive integer, infinite number of real rows is not allowed"""
        return self.rowCount()

    def columnCount(self, row):
        """Should return -1 if no row exists"""
        raise NotImplementedError()

    def realColumnCount(self, row):
        return self.columnCount(row)

    def index(self, row, column):
        """Create index for row and column. Can return virtual index"""
        return ModelIndex(row, column, self) if self.hasIndex(row, column) else ModelIndex()

    def lastRowIndex(self, row):
        """Return last index on given row."""
        return self.index(row, self.columnCount(row) - 1)

    def lastRealRowIndex(self, row):
        return self.index(row, self.realColumnCount(row) - 1)

    def indexFlags(self, index):
        return 0

    def indexData(self, index, role=Qt.DisplayRole):
        return None

    def hasIndex(self, row, column):
        """Return True if there is index at row and column in this model"""
        if self.rowCount() < 0 or row < self.rowCount():
            return column < self.columnCount(row)
        return False

    def hasRealIndex(self, row, column):
        return row < self.realRowCount() and column < self.realColumnCount(row)

    def hasRow(self, row):
        return row >= 0 and (self.rowCount() < 0 or row + 1 < self.rowCount())

    @property
    def firstIndex(self):
        return self.index(0, 0)

    @property
    def lastIndex(self):
        """Return last index in the model"""
        if self.rowCount() <= 0:
            return ModelIndex()

        row = self.rowCount() - 1
        index = ModelIndex()
        while not index and row >= 0:
            index = self.lastRowIndex(row)
            row += 1
        return index

    @property
    def lastRealIndex(self):
        row = self.realRowCount() - 1
        index = ModelIndex()
        while not index and row >= 0:
            index = self.lastRealRowIndex(row)
            row -= 1
        return index

    def setIndexData(self, index, value, role=Qt.EditRole):
        return False

    def onEditorDataChanged(self, start, length):
        pass

    def onEditorDataResized(self, new_size):
        pass

    @property
    def preferSpaced(self):
        """Whether view should display indexes with spaces between them by default"""
        return True

    def indexFromPosition(self, position):
        """Return index matching given editor position. Can return virtual index"""
        raise NotImplementedError()

    @property
    def regular(self):
        """Return True if this model is regular. Regular models have same number of columns in each row and same
        text length for cell."""
        return False

    def createEditWidget(self, parent, index, style_options):
        return None


def index_range(start_index, end_index, include_last):
    """Iterator allows traversing through model indexes."""
    if not start_index or not end_index or end_index < start_index:
        return
    current_index = start_index
    while current_index:
        if include_last and not (current_index <= end_index):
            break
        elif not include_last and not (current_index < end_index):
            break
        yield current_index
        current_index = current_index.next


class EmptyColumnModel(ColumnModel):
    def rowCount(self):
        return 0

    def columnCount(self, row):
        return 0

    def indexFromPosition(self, position):
        return ModelIndex()


class HexColumnModel(ColumnModel):
    """Standart column for hex-editors. Displays data as numbers. This model is regular (has equal number of
    columns in each row, except last one) and infinite (supports virtual indexes).
    """

    def __init__(self, editor, valuecodec, formatter, columns_on_row=16):
        ColumnModel.__init__(self, editor)
        self.valuecodec = valuecodec
        self.formatter = formatter
        self._rowCount = 0
        self._bytesOnRow = columns_on_row * self.valuecodec.dataSize
        self._columnsOnRow = columns_on_row
        min_value, max_value = self.valuecodec.minimal, self.valuecodec.maximal
        self._cellTextSize = max(len(self.formatter.format(min_value)), len(self.formatter.format(max_value)))
        self._updateRowCount()

    def rowCount(self):
        return -1

    def columnCount(self, row):
        return self._columnsOnRow

    def realRowCount(self):
        return self._rowCount

    def realColumnCount(self, row):
        if row + 1 == self._rowCount:
            count = (len(self.editor) % self.bytesOnRow) // self.valuecodec.dataSize
            return count or self._columnsOnRow
        elif row >= self._rowCount:
            return 0
        return self._columnsOnRow

    def indexFromPosition(self, editor_position):
        return self.index(int(editor_position // self.bytesOnRow),
                          int(editor_position % self.bytesOnRow) // self.valuecodec.dataSize)

    def indexData(self, index, role=Qt.DisplayRole):
        if not index or self.editor is None:
            return None
        editor_position = self.bytesOnRow * index.row + self.valuecodec.dataSize * index.column

        if role == Qt.DisplayRole or role == Qt.EditRole:
            if editor_position >= len(self.editor):
                return '.' * self._cellTextSize
            else:
                editor_data = self.editor.read(editor_position, self.valuecodec.dataSize)
                return self.formatter.format(self.valuecodec.decode(editor_data))
        elif role == self.EditorDataRole:
            if editor_position >= len(self.editor):
                return bytes()
            else:
                return self.editor.read(editor_position, self.valuecodec.dataSize)
        elif role == self.DataSizeRole:
            return self.valuecodec.dataSize
        elif role == self.EditorPositionRole:
            return editor_position
        return None

    def setIndexData(self, index, value, role):
        if index and index.model is self:
            if role == self.EditorDataRole:
                position = self.indexData(index, ColumnModel.EditorPositionRole)
                if position is not None and position >= 0:
                    if value != index.data(self.EditorDataRole):
                        try:
                            self.editor.writeSpan(position, DataSpan(self.editor, value))
                            return True
                        except IOError:
                            pass
        return False

    def indexFlags(self, index):
        flags = ColumnModel.FlagEditable
        if index.row >= self._rowCount or (index.row == self._rowCount - 1 and index > self.lastRealIndex):
            flags |= ColumnModel.FlagVirtual
        elif self.editor is not None and self.editor.isRangeModified(index.data(self.EditorPositionRole),
                                                                   index.data(self.DataSizeRole)):
            flags |= ColumnModel.FlagModified
        return flags

    @property
    def bytesOnRow(self):
        return self._bytesOnRow

    def onEditorDataChanged(self, start, length):
        length = length if length >= 0 else len(self.editor) - start
        self.dataChanged.emit(self.indexFromPosition(start), self.indexFromPosition(start + length - 1))

    def onEditorDataResized(self, new_size):
        old_last_index = self.lastRealIndex
        self._updateRowCount()
        self.dataChanged.emit(old_last_index, self.lastRealIndex)

    @property
    def preferSpaced(self):
        return True

    def _updateRowCount(self):
        self._rowCount = len(self.editor) // self.bytesOnRow + bool(len(self.editor) % self.bytesOnRow)

    def createEditWidget(self, parent, index):
        return HexColumnEditWidget(parent, index)


class CharColumnModel(ColumnModel):
    """This column displays data as characters in one of possible encodings. It depends on encoding how many bytes
    will occupy each cell. Since even cells of one row can have different number of bytes, we can either have constant
    number of bytes for row or constant number of columns in it. In first case this leads to awful (unaligned) look of widget in
    case of multi-byte encodings (utf-8, for example). Second case leads to awful behaviour - CharColumn and HexColumn
    cannot be synchronized to display same bytes on same rows (this behaviour is common to other hex-editors).
    """

    ReplacementCharacter = '·'

    def __init__(self, editor, codec, render_font, bytes_on_row=16):
        ColumnModel.__init__(self, editor)
        self._rowCount = 0
        self._codec = codec

        # number of bytes on row should be multiplier of codec.unitSize
        if bytes_on_row % codec.unitSize:
            raise ValueError('number of bytes on row should be multiplier of encoding unit size')
        self._bytesOnRow = bytes_on_row

        self._renderFont = QRawFont.fromFont(render_font)

        self._updateRowCount()

    @property
    def codec(self):
        return self._codec

    @property
    def renderFont(self):
        return self._renderFont

    @renderFont.setter
    def renderFont(self, new_font):
        if isinstance(new_font, QFont):
            new_font = QRawFont.fromFont(new_font)
        self._renderFont = new_font
        self.modelReset.emit()

    def _updateRowCount(self):
        self._rowCount = len(self.editor) // self._bytesOnRow + bool(len(self.editor) % self._bytesOnRow)

    def rowCount(self):
        return -1

    def columnCount(self, row):
        return self._bytesOnRow // self._codec.unitSize

    def realRowCount(self):
        return self._rowCount

    def realColumnCount(self, row):
        position = row * self._bytesOnRow
        if position >= len(self.editor):
            return 0
        elif row + 1 == self._rowCount:
            bytes_left = position % self._bytesOnRow + 1
            return bytes_left // self._codec.unitSize + bool(bytes_left % self._codec.unitSize)
        else:
            return self._bytesOnRow // self._codec.unitSize

    def indexFromPosition(self, position):
        return self.index(position // self._bytesOnRow,
                         (position % self._bytesOnRow) // self._codec.unitSize)

    def indexData(self, index, role=Qt.DisplayRole):
        if not index or self.editor is None:
            return None

        position = index.row * self._bytesOnRow + index.column * self._codec.unitSize
        is_virtual = position >= len(self.editor)

        if role == self.EditorPositionRole:
            return position
        elif role == self.DataSizeRole:
            if is_virtual:
                return self._codec.unitSize
            try:
                return self._codec.getCharacterSize(self.editor, position)
            except encodings.EncodingError:
                return 1
        elif role == self.EditorDataRole:
            if is_virtual:
                return bytes()
            try:
                return self.editor.readAtEnd(self._codec.findCharacterStart(self.editor, position),
                                         self._codec.getCharacterSize(self.editor, position))
            except encodings.EncodingError:
                return self.editor.readAtEnd(position, 1)
        elif role in (Qt.DisplayRole, Qt.EditRole):
            if is_virtual:
                return '.'
            try:
                character_start = self._codec.findCharacterStart(self.editor, position)
                if character_start != position:
                    return ' '
                else:
                    decoded = self._codec.decodeCharacter(self.editor, position)
                    return self._translateToVisualCharacter(decoded)
            except encodings.EncodingError:
                return '∎'
        return None

    def indexFlags(self, index):
        flags = self.FlagEditable
        if index.row >= self._rowCount and index > self.lastRealIndex:
            flags |= self.FlagVirtual
        elif self.editor.isRangeModified(index.data(self.EditorPositionRole), index.data(self.DataSizeRole)):
            flags |= self.FlagModified
        return flags

    def setIndexData(self, index, value, role=Qt.DisplayRole):
        if not (index and self.editor is not None and role == Qt.EditorDataRole):
            return False

        position = self.indexData(index, self.EditorPositionRole)
        old_size = self._codec.getCharacterSize(self.editor, position)
        encoded = self._codec.encodeString(value)

        self.editor.beginComplexAction()
        try:
            self.editor.remove(position, old_size)
            self.editor.insertSpan(position, DataSpan(self.editor, encoded))
            return True
        finally:
            self.editor.endComplexAction()

    def _translateToVisualCharacter(self, text):
        import unicodedata

        result = ''
        for char in text:
            if unicodedata.category(char) in ('Cc', 'Cf', 'Cn', 'Co', 'Cs', 'Lm', 'Mc', 'Zl', 'Zp'):
                result += self.ReplacementCharacter
            elif not self._renderFont.supportsCharacter(char):
                result += self.ReplacementCharacter
            else:
                result += char
        return result

    @property
    def preferSpaced(self):
        return False

    @property
    def regular(self):
        return True


class AddressColumnModel(ColumnModel):
    def __init__(self, linked_model, formatter=None):
        ColumnModel.__init__(self, linked_model.editor)
        self._linkedModel = linked_model
        self._formatter = formatter or IntegerFormatter()
        self._baseAddress = 0
        self._updatePadding()

    @property
    def linkedModel(self):
        return self._linkedModel

    @linkedModel.setter
    def linkedModel(self, model):
        if self._linkedModel is not model:
            self._linkedModel = model
            self.modelReset.emit()

    @property
    def baseAddress(self):
        return self._baseAddress

    @baseAddress.setter
    def baseAddress(self, new_base):
        if self._baseAddress != new_base:
            self._baseAddress = new_base
            self.modelReset.emit()

    def rowCount(self):
        if self._linkedModel is not None:
            return self._linkedModel.rowCount()
        return 0

    def columnCount(self, row):
        return 1 if self._linkedModel is not None and self._linkedModel.hasRow(row) else 0

    def realRowCount(self):
        if self._linkedModel is not None:
            return self._linkedModel.realRowCount()
        return 0

    def realColumnCount(self, row):
        if self._linkedModel is not None:
            return int(bool(self._linkedModel.realColumnCount(row)))
        return 0

    def indexData(self, index, role=Qt.DisplayRole):
        if self._linkedModel is not None:
            model_index = self._linkedModel.index(index.row, 0)
            if role == Qt.DisplayRole and model_index:
                raw = model_index.data(self.EditorPositionRole) - self._baseAddress
                return self._formatter.format(raw) if self._formatter is not None else raw
            elif role == self.EditorPositionRole:
                return model_index.data(self.EditorPositionRole)
            elif role == self.DataSizeRole:
                return sum(index.data(self.DataSizeRole) for index in index_range(
                    model_index, self._linkedModel.lastRowIndex(index.row), include_last=True
                ))
        return None

    def indexFromPosition(self, position):
        if self._linkedModel is not None:
            model_index = self._linkedModel.indexFromPosition(position)
            return self.index(model_index.row, 0) if model_index else ModelIndex()
        return self._linkedModelone

    def indexFlags(self, index):
        if self._linkedModel is not None:
            model_index = self._linkedModel.index(index.row, 0)
            if model_index and model_index.flags & ColumnModel.FlagVirtual:
                return ColumnModel.FlagVirtual
        return 0

    def _maxLengthForSize(self, size):
        """Calculate maximal length of address text for given editor size"""
        sign1 = self._baseAddress > 0
        sign2 = self._baseAddress < len(self._linkedModel.editor)
        max_raw = max(abs(0 - self._baseAddress) + sign1,
                      abs(len(self._linkedModel.editor) - self._baseAddress) + sign2)
        return len(self._formatter.format(max_raw))

    def _updatePadding(self):
        if self._formatter is None:
            self._formatter = IntegerFormatter()
        self._formatter.padding = self._maxLengthForSize(len(self.editor))

    def onEditorDataResized(self, new_size):
        self._updatePadding()


class FrameProxyModel(ColumnModel):
    """This proxy model displays only number of rows starting after given first row. Proxy model does not check
    if there are indexes on these rows."""

    frameScrolled = pyqtSignal(int, int)  # first argument is new first frame row, second one is old first frame row
    frameResized = pyqtSignal(int, int)  # first argument is new frame size, second one is old frame size

    def __init__(self, source_model):
        ColumnModel.__init__(self, None)
        self.__firstRow = 0
        self.__rowCount = 0
        self.sourceModel = source_model or EmptyColumnModel()
        self.sourceModel.dataChanged.connect(self.__onDataChanged)
        self.sourceModel.modelReset.connect(self.modelReset)

    def setFrame(self, first_row, row_count):
        self.resizeFrame(row_count)
        self.scrollFrame(first_row)

    def scrollFrame(self, new_first_row):
        if self.__firstRow != new_first_row:
            old_first_row = self.__firstRow
            self.__firstRow = new_first_row
            self.frameScrolled.emit(new_first_row, old_first_row)

    def resizeFrame(self, new_frame_size):
        if self.__rowCount != new_frame_size:
            old_size = self.__rowCount
            self.__rowCount = new_frame_size
            self.frameResized.emit(new_frame_size, old_size)

    def rowCount(self):
        model_row_count = self.sourceModel.rowCount()
        if model_row_count >= 0:
            return min(self.__rowCount, model_row_count)
        else:
            return self.__rowCount

    def columnCount(self, row):
        return self.sourceModel.columnCount(row + self.__firstRow)

    def realRowCount(self):
        return min(self.__rowCount, self.sourceModel.rowCount())

    def realColumnCount(self, row):
        return self.sourceModel.realColumnCount(row + self.__firstRow)

    def indexFromPosition(self, position):
        return self.fromSourceIndex(self.sourceModel.indexFromPosition(position))

    def index(self, row, column=0):
        return self.fromSourceIndex(self.sourceModel.index(row + self.__firstRow, column))

    def hasIndex(self, row, column):
        return ColumnModel.hasIndex(self, row, column) and self.sourceModel.hasIndex(row + self.__firstRow, column)

    def hasRealIndex(self, row, column):
        return ColumnModel.hasIndex(self, row, column) and self.sourceModel.hasRealIndex(row + self.__firstRow, column)

    def indexData(self, index, role=Qt.DisplayRole):
        return self.sourceModel.indexData(self.toSourceIndex(index), role)

    def indexFlags(self, index):
        return self.sourceModel.indexFlags(self.toSourceIndex(index))

    def setIndexData(self, index, value, role=Qt.EditRole):
        return self.sourceModel.setIndexData(self.toSourceIndex(index), value, role)

    def toSourceIndex(self, index):
        if not index or index.model is self.sourceModel:
            return index
        elif self.hasIndex(index.row, index.column):
            return self.sourceModel.index(index.row + self.__firstRow, index.column)
        else:
            return ModelIndex()

    def fromSourceIndex(self, index):
        if not index or index.model is self:
            return index
        elif self.hasIndex(index.row - self.__firstRow, index.column):
            return ModelIndex(index.row - self.__firstRow, index.column, self)
        else:
            return ModelIndex()

    def __onDataChanged(self, first, last):
        # check if update area lays inside frame
        if last.row < self.__firstRow or first.row > self.toSourceIndex(self.lastIndex).row:
            return

        first = self.fromSourceIndex(first) or self.firstIndex
        last = self.fromSourceIndex(last) or self.lastIndex

        self.dataChanged.emit(first, last)

    @property
    def regular(self):
        return self.sourceModel.regular


DefaultFont = QFont('Ubuntu Mono', 13)
VisualSpace = 10

# default color theme
BackgroundColor = QColor(250, 250, 245)
TextColor = QColor(Qt.black)
BorderColor = QColor(Qt.black)
InactiveTextColor = QColor(Qt.darkGray)
CaretBackgroundColor = QColor(150, 250, 160, 100)
CaretBorderColor = QColor(0, 0, 0, 255)
SelectionBackgroundColor = QColor(220, 250, 245, 100)
SelectionBorderColor = QColor(20, 205, 195)
# EditCaretBackgroundColor = QColor(0, 0, 0, 150)
# EditCaretBorderColor = EditCaretBackgroundColor
EditCaretTextColor = QColor(Qt.white)
ModifiedTextColor = QColor(Qt.red)


class RowData(object):
    def __init__(self):
        self.text = ''
        self.html = ''
        self.items = []


class IndexData(object):
    def __init__(self, index):
        self._text = None
        self.index = index
        self.firstCharIndex = 0
        self.firstHtmlCharIndex = 0
        self.html = None
        self.color = None

    @property
    def text(self):
        return self.data()

    def data(self, role=Qt.DisplayRole):
        if role == Qt.DisplayRole:
            if self._text is None:
                self._text = self.index.data()
            return self._text
        if self.index:
            return self.index.data(role)
        return None


class ColumnDocumentBackend(QObject):
    documentUpdated = pyqtSignal()

    def __init__(self, column):
        QObject.__init__(self)
        self._document = None
        self._column = column
        
    @property
    def document(self):
        if self._document is None:
            self.generateDocument()
        return self._document

    @property
    def generated(self):
        return self._document is not None

    def generateDocument(self):
        raise NotImplementedError()

    def updateRow(self, row_index, cached_row):
        raise NotImplementedError()

    def removeRows(self, row_index, number_of_rows):
        raise NotImplementedError()

    def insertRow(self, row_index, number_of_rows):
        raise NotImplementedError()

    def rectForCell(self, row_index, cell_index):
        raise NotImplementedError()

    def cellForPoint(self, point):
        raise NotImplementedError()

    def clear(self):
        self._document = None


class PlainTextDocumentBackend(ColumnDocumentBackend):
    def __init__(self, column):
        ColumnDocumentBackend.__init__(self, column)

    def generateDocument(self):
        if self._document is None:
            self._document = self._column.createDocumentTemplate()

            cursor = QTextCursor(self._document)
            block_format = self._documentBlockFormat
            for row_data in self._column._cache:
                cursor.movePosition(QTextCursor.End)
                cursor.insertHtml(row_data.html)
                cursor.insertBlock()
                cursor.setBlockFormat(block_format)

            self.documentUpdated.emit()

    @property
    def _documentBlockFormat(self):
        block_format = QTextBlockFormat()
        block_format.setLineHeight(self._column._fontMetrics.height(), QTextBlockFormat.FixedHeight)
        return block_format

    def updateRow(self, row_index, row_data):
        if self._document is not None:
            block = self._document.findBlockByLineNumber(row_index)
            if block.isValid():
                cursor = QTextCursor(block)
                cursor.movePosition(QTextCursor.EndOfBlock, QTextCursor.KeepAnchor)
                cursor.insertHtml(row_data.html)

                self.documentUpdated.emit()

    def removeRows(self, row_index, number_of_rows):
        if self._document is not None:
            for x in range(number_of_rows):
                block = self._document.findBlockByLineNumber(row_index)
                if block.isValid():
                    cursor = QTextCursor(block)
                    cursor.movePosition(QTextCursor.EndOfBlock, QTextCursor.KeepAnchor)
                    cursor.removeSelectedText()
                    if row_index == 0:
                        cursor.deleteChar()

            self.documentUpdated.emit()

    def insertRows(self, row_index, number_of_rows):
        if self._document is not None:
            for x in range(number_of_rows):
                if row_index < 0:
                    cursor = QTextCursor(self._document)
                    cursor.movePosition(QTextCursor.End)
                else:
                    block = self._document.findBlockByLineNumber(row_index)
                    cursor = QTextCursor(block)

                cursor.insertBlock()
                cursor.setBlockFormat(self._documentBlockFormat)

            self.documentUpdated.emit()

    def rectForCell(self, row_index, cell_index):
        self.generateDocument()

        try:
            index_data = self._column._cache[row_index].items[cell_index]
        except IndexError:
            return QRectF()

        if index_data is not None:
            block = self._document.findBlockByLineNumber(row_index)
            if block.isValid():
                block_rect = self._document.documentLayout().blockBoundingRect(block)
                line = block.layout().lineAt(0)
                x = line.cursorToX(index_data.firstCharIndex)[0] + VisualSpace
                y = block_rect.y() + line.position().y()
                width = self._column._fontMetrics.width(index_data.text)
                return QRectF(x, y, width, self._column._fontMetrics.height())
        return QRectF()

    def cellForPoint(self, point):
        self.generateDocument()

        point = QPointF(point.x() - VisualSpace, point.y())
        char_position = self._document.documentLayout().hitTest(point, Qt.ExactHit)
        block = self._document.findBlock(char_position)
        if block.isValid():
            row_index = block.firstLineNumber()
            row_data = self._column._cache[row_index]
            line_char_index = char_position - block.position()
            for column in range(len(row_data.items)):
                index_data = row_data.items[column]
                if index_data.firstCharIndex + len(index_data.text) > line_char_index:
                    return row_index, column
        return -1, -1


class TableDocumentBackend(ColumnDocumentBackend):
    def __init__(self, column):
        ColumnDocumentBackend.__init__(self, column)

    def generateDocument(self):
        if self._document is None:
            self._document = self._column.createDocumentTemplate()

            cursor = QTextCursor(self._document)

            cursor.beginEditBlock()
            try:
                self._table = cursor.insertTable(len(self._column._cache), self._column.model.columnCount(0))
                self._table.setFormat(self._tableFormat)
                for row_index in range(len(self._column._cache)):
                    for column_index in range(self._table.columns()):
                        cell = self._table.cellAt(row_index, column_index)
                        cursor = cell.firstCursorPosition()
                        try:
                            cursor.insertHtml(self._column._cache[row_index].items[column_index].html)
                        except IndexError:
                            pass
            finally:
                cursor.endEditBlock()

            self.documentUpdated.emit()

    @property
    def _tableFormat(self):
        fmt = QTextTableFormat()
        fmt.setBorderStyle(QTextTableFormat.BorderStyle_None)
        fmt.setBorder(0)
        fmt.setCellPadding(0)
        fmt.setCellSpacing(0)
        return fmt

    def updateRow(self, row_index, row_data):
        if self._document is not None:
            cursor = self._table.cellAt(row_index, 0).firstCursorPosition()
            cursor.movePosition(QTextCursor.NextRow)
            cursor.removeSelectedText()
            cursor.beginEditBlock()  # we do not use undo/redo, but this thing increases perfomance, blocking updating
                                     # layout until endEditBlock is called.
            try:
                for column_index in range(self._table.columns()):
                    cursor = self._table.cellAt(row_index, column_index).firstCursorPosition()
                    cursor.insertHtml(row_data.items[column_index].html)
            finally:
                cursor.endEditBlock()

            self.documentUpdated.emit()

    def removeRows(self, row_index, number_of_rows):
        if self._document is not None:
            self._table.removeRows(row_index, number_of_rows)
            self.documentUpdated.emit()

    def insertRows(self, row_index, number_of_rows):
        if self._document is not None:
            if row_index < 0:
                self._table.appendRows(number_of_rows)
            else:
                self._table.insertRows(row_index, number_of_rows)

            self.documentUpdated.emit()

    def rectForCell(self, row_index, cell_index):
        self.generateDocument()

        cursor = self._table.cellAt(row_index, cell_index).firstCursorPosition()
        if not cursor.isNull():
            r = self._document.documentLayout().blockBoundingRect(cursor.block())
            r.translate(VisualSpace, 0)
            return r

        return QRectF()

    def cellForPoint(self, point):
        self.generateDocument()

        for row_index in range(len(self._column._cache)):
            for column_index in range(self._table.columns()):
                if self.rectForCell(row_index, column_index).contains(point):
                    return row_index, column_index
        return -1, -1



class Column(QObject):
    updateRequested = pyqtSignal()
    resizeRequested = pyqtSignal(QSizeF)

    def __init__(self, model):
        QObject.__init__(self)
        self.sourceModel = model
        self.model = FrameProxyModel(model)
        self.model.dataChanged.connect(self._onDataChanged)
        self.model.modelReset.connect(self._updateCache)
        self.model.frameScrolled.connect(self._onFrameScrolled)
        self.model.frameResized.connect(self._onFrameResized)

        self._geom = QRectF()
        self._font = DefaultFont
        self._fontMetrics = QFontMetricsF(self._font)

        self._fullVisibleRows = 0
        self._visibleRows = 0
        self._firstVisibleRow = 0

        self._spaced = self.sourceModel.preferSpaced
        self._cache = []
        # if self.sourceModel.regular:
        #     self._documentBackend = TableDocumentBackend(self)
        # else:
        self._documentBackend = PlainTextDocumentBackend(self)
        self._documentBackend.documentUpdated.connect(self._onDocumentUpdated)

    @property
    def geometry(self):
        return self._geom

    @geometry.setter
    def geometry(self, rect):
        self._geom = rect
        self._updateGeometry()
        self.updateRequested.emit()  # even if frame not changed, we should redraw widget

    @property
    def firstVisibleRow(self):
        return self._firstVisibleRow

    def scrollToFirstRow(self, source_row_index):
        if self._firstVisibleRow != source_row_index:
            self._firstVisibleRow = source_row_index
            self.model.scrollFrame(self._firstVisibleRow)

    @property
    def lastFullVisibleRow(self):
        if self._fullVisibleRows > 0:
            return self._firstVisibleRow + self._fullVisibleRows - 1
        return self._firstVisibleRow

    @property
    def lastVisibleRow(self):
        if self._visibleRows > 0:
            return self._firstVisibleRow + self._visibleRows - 1
        return self._firstVisibleRow

    @property
    def visibleRows(self):
        return self._visibleRows

    @property
    def fullVisibleRows(self):
        return self._fullVisibleRows

    @property
    def firstVisibleIndex(self):
        return self.model.index(0, 0)

    @property
    def lastVisibleIndex(self):
        return self.model.lastRowIndex(self._visibleRows - 1)

    @property
    def lastFullVisibleIndex(self):
        return self.model.lastRowIndex(self._fullVisibleRows - 1)

    @property
    def font(self):
        return self._font

    @font.setter
    def font(self, new_font):
        self._font = new_font
        self._fontMetrics = QFontMetricsF(new_font)
        if hasattr(self.sourceModel, 'renderFont'):  # well, this is hack until i invent better solution...
            self.sourceModel.renderFont = new_font
        self._documentBackend.clear()
        self._updateGeometry()

    @property
    def editor(self):
        return self.sourceModel.editor

    @property
    def spaced(self):
        return self._spaced

    @spaced.setter
    def spaced(self, new_spaced):
        if self._spaced != new_spaced:
            self._spaced = new_spaced
            self._updateCache()

    def getRowCachedData(self, visible_row_index):
        return self._cache[visible_row_index] if 0 <= visible_row_index < len(self._cache) else None

    def getIndexCachedData(self, index):
        index = self.model.fromSourceIndex(index)
        row_data = self.getRowCachedData(index.row)
        if row_data is not None and index.column < len(row_data.items):
            return row_data.items[index.column]
        return None

    def getRectForIndex(self, index):
        index = self.model.fromSourceIndex(index)
        return self._documentBackend.rectForCell(index.row, index.column)

    # def getRectForRow(self, visible_row_index):
    #     if 0 <= visible_row_index < self._visibleRows:
    #         block = self._document.findBlockByLineNumber(visible_row_index)
    #         if block.isValid():
    #             rect = block.layout().lineAt(0).rect()
    #             rect.translate(VisualSpace, 0)
    #             return rect
    #     return QRectF()

    def getPolygonsForRange(self, first_index, last_index):
        first_index = self.model.toSourceIndex(first_index)
        last_index = self.model.toSourceIndex(last_index)

        first_visible_source_index = self.model.toSourceIndex(self.firstVisibleIndex)
        last_visible_source_index = self.model.toSourceIndex(self.lastVisibleIndex)
        if not first_index or not last_index or (first_index > last_visible_source_index or
                                                         last_index < first_visible_source_index):
            return tuple()

        first_index = max(first_index, first_visible_source_index)
        last_index = min(last_index, last_visible_source_index)

        if first_index == last_index:
            return QPolygonF(self.getRectForIndex(first_index)),
        else:
            r1 = self.getRectForIndex(first_index)
            r2 = self.getRectForIndex(last_index)

            if first_index.row == last_index.row:
                return QPolygonF(QRectF(r1.topLeft(), r2.bottomRight())),
            elif first_index.row + 1 == last_index.row and r1.left() > r2.right():
                first_row_last_index = self.sourceModel.lastRowIndex(first_index.row)
                return (
                    QPolygonF(QRectF(r1.topLeft(), self.getRectForIndex(first_row_last_index).bottomRight())),
                    QPolygonF(QRectF(QPointF(VisualSpace, r2.top()), r2.bottomRight()))
                )
            else:
                range_polygon = QPolygonF()

                for row_index in range(first_index.row, last_index.row):
                    index = self.sourceModel.index(row_index, self.sourceModel.columnCount(row_index) - 1)
                    index_rect = self.getRectForIndex(index)
                    range_polygon.append(index_rect.topRight())
                    range_polygon.append(index_rect.bottomRight())

                range_polygon.append(r2.topRight())
                range_polygon.append(r2.bottomRight())
                range_polygon.append(self.getRectForIndex(self.sourceModel.index(last_index.row, 0)).bottomLeft())
                range_polygon.append(self.getRectForIndex(self.sourceModel.index(first_index.row, 0)).bottomLeft())
                range_polygon.append(r1.bottomLeft())
                range_polygon.append(r1.topLeft())

                return range_polygon,

    def indexFromPoint(self, point):
        row, column = self._documentBackend.cellForPoint(point)
        return self.model.toSourceIndex(self.model.index(row, column))

    def paint(self, paint_data, is_leading):
        painter = paint_data.painter
        painter.save()

        painter.setPen(TextColor if is_leading else InactiveTextColor)
        painter.translate(VisualSpace, 0)

        # little trick to quickly change default text color for document without re-generating it
        paint_context = QAbstractTextDocumentLayout.PaintContext()
        paint_context.palette.setColor(QPalette.Text, TextColor if is_leading else InactiveTextColor)
        # standard QTextDocument.draw also sets clip rect here, but we already have one
        self._documentBackend.document.documentLayout().draw(painter, paint_context)

        painter.restore()

    def paintCaret(self, paint_data, is_leading, caret_position):
        painter = paint_data.painter
        if caret_position >= 0:
            caret_index = self.model.fromSourceIndex(self.sourceModel.indexFromPosition(caret_position))
            if caret_index and self.isIndexVisible(caret_index, False):
                caret_rect = self.getRectForIndex(caret_index)
                painter.setBrush(QBrush(CaretBackgroundColor))
                painter.setPen(CaretBorderColor)
                painter.drawRect(caret_rect)

    def paintSelection(self, paint_data, is_leading, selection):
        if selection is not None and len(selection) > 0:
            painter = paint_data.painter
            for sel_polygon in self.getPolygonsForRange(self.sourceModel.indexFromPosition(selection.start),
                                        self.sourceModel.indexFromPosition(selection.start + len(selection) -1)):
                painter.setBrush(SelectionBackgroundColor)
                painter.setPen(QPen(QBrush(SelectionBorderColor), 2.0))
                painter.drawPolygon(sel_polygon)

    def _updateGeometry(self):
        self._fullVisibleRows = int(self._geom.height() // self._fontMetrics.height())
        self._visibleRows = self._fullVisibleRows + bool(int(self._geom.height()) % int(self._fontMetrics.height()))
        self.model.resizeFrame(self._visibleRows)

    def _updateCache(self):
        self._cache = [None for row in range(self._visibleRows)]
        self._documentBackend.clear()

        for row in range(self._visibleRows):
            self._updateCachedRow(row)

        self.updateRequested.emit()

    def _onDocumentUpdated(self):
        ideal_width = self._documentBackend._document.idealWidth() + VisualSpace * 2
        if ideal_width != self._geom.width():
            self.resizeRequested.emit(QSizeF(ideal_width, self._geom.height()))

    def _updateCachedRow(self, row_index):
        assert(0 <= row_index < len(self._cache))

        row_data = RowData()
        column_count = self.model.columnCount(row_index)
        for column_index in range(column_count):
            index = self.model.toSourceIndex(self.model.index(row_index, column_index))
            index_data = IndexData(index)
            index_data.firstCharIndex = len(row_data.text)
            index_data.firstHtmlCharIndex = len(row_data.html)
            index_text = index_data.data()

            if index_text is not None:
                cell_classes = []
                if index.flags & ColumnModel.FlagModified:
                    cell_classes.append('cell-mod')
                if index.flags & ColumnModel.FlagVirtual:
                    cell_classes.append('highlight')

                prepared_text = html.escape(index_text)
                prepared_text = prepared_text.replace(' ', '&nbsp;')
                if cell_classes:
                    # unfortunately, HTML subset supported by Qt does not include multiclasses
                    for css_class in cell_classes:
                        index_html = '<span class={0}>{1}</span>'.format(css_class, prepared_text)
                else:
                    index_html = '<span>{0}</span>'.format(prepared_text)

                row_data.html += index_html
                row_data.text += index_text
                index_data.html = index_html

            if self.spaced and column_index + 1 < column_count:
                row_data.text += ' '
                row_data.html += '<span> </span>'

            row_data.items.append(index_data)

        self._cache[row_index] = row_data
        self._documentBackend.updateRow(row_index, row_data)

    def _onFrameScrolled(self, new_first_row, old_first_row):<commit_msg>bug with virtual indexes in CharColumnModel is fixed; virtual indexes are no more highlighed<commit_after>from PyQt4.QtCore import pyqtSignal, QObject, Qt, QPointF, QRectF, QPoint, QSizeF, QEvent, QTimer, QLineF
from PyQt4.QtGui import QColor, QFont, QFontMetricsF, QPolygonF, QWidget, QScrollBar, QVBoxLayout, QHBoxLayout, \
                        QPainter, QBrush, QPalette, QPen, QApplication, QRegion, QLineEdit, QValidator, \
                        QTextEdit, QTextOption, QSizePolicy, QStyle, QStyleOptionFrameV2, QTextCursor, QTextDocument, \
                        QTextBlockFormat, QPlainTextDocumentLayout, QAbstractTextDocumentLayout, QTextCharFormat, \
                        QTextTableFormat, QRawFont
import math
import html
from hex.valuecodecs import IntegerCodec
from hex.formatters import IntegerFormatter
from hex.editor import DataSpan
from hex.proxystyle import ProxyStyle
import hex.encodings as encodings
import hex.utils as utils


# Why we need to make different model/view classes? Why not to use existing ones?
# These classes are specialized and optimized for our needs. But main cause is that standard model classes will not
# work with very big amount of data (especially on 32-bit machines) where number of rows exceeds 32-bit int limit.
# Another difference is that custom model class represents irregular array instead of tree-like structure Qt does.


class ModelIndex(object):
    def __init__(self, row=-1, column=-1, model=None, internal_data=None):
        self.__row = row
        self.__column = column
        self.__model = model
        self.__data = internal_data

    @property
    def row(self):
        """This value is always positive int for valid indexes, -1 otherwise"""
        return self.__row if self.valid else -1

    @property
    def column(self):
        """This value is always positive int for valid indexes, -1 otherwise"""
        return self.__column if self.valid else -1

    @property
    def model(self):
        """Returns model this index belongs to, None for invalid indexes"""
        return self.__model if self.valid else None

    @property
    def virtual(self):
        # virtual indexes does not represent real data, but used to represent empty cells in regular models.
        # Regular model has finite set of not-virtual indexes and can have infinite set of virtual ones
        return self.flags & ColumnModel.FlagVirtual

    @property
    def valid(self):
        """Return True if index is valid"""
        return self.__model is not None and self.__row >= 0 and self.__column >= 0

    @property
    def internalData(self):
        return self.__data if self.valid else None

    def data(self, role=Qt.DisplayRole):
        """Data for index"""
        return self.__model.indexData(self, role) if self.valid else None

    @property
    def flags(self):
        return self.__model.indexFlags(self) if self.valid else 0

    def __eq__(self, other):
        """Invalid index is not equal to any index"""
        if not isinstance(other, ModelIndex):
            return NotImplemented
        return (self.valid and other.valid and self.row == other.row and self.column == other.column
                and self.model is other.model)

    def __ne__(self, other):
        if not isinstance(other, ModelIndex):
            return NotImplemented
        return not self.__eq__(other)

    def __bool__(self):
        return self.valid

    def __isCompatible(self, other):
        return isinstance(other, ModelIndex) and self.valid and other.valid

    def __lt__(self, other):
        if not self.__isCompatible(other):
            return NotImplemented
        return self.row < other.row or (self.row == other.row and self.column < other.column)

    def __gt__(self, other):
        if not self.__isCompatible(other):
            return NotImplemented
        return self.row > other.row or (self.row == other.row and self.column > other.column)

    def __le__(self, other):
        if not self.__isCompatible(other):
            return NotImplemented
        return self.__lt__(other) or self.__eq__(other)

    def __ge__(self, other):
        if not self.__isCompatible(other):
            return NotImplemented
        return self.__gt__(other) or self.__eq__(other)

    @property
    def next(self):
        if not self.valid:
            return ModelIndex()

        next_index = self.model.index(self.row, self.column + 1)
        if not next_index:
            row = self.row + 1
            while not next_index and (self.model.rowCount() < 0 or row < self.model.rowCount()):
                next_index = self.model.index(row, 0)
                row += 1

        return next_index

    @property
    def previous(self):
        if not self.valid:
            return ModelIndex()

        prev_index = self.model.index(self.row, self.column - 1)
        if not prev_index:
            row = self.row - 1
            while not prev_index and row >= 0:
                prev_index = self.model.lastRowIndex(row)
                row -= 1

        return prev_index


class ColumnModel(QObject):
    EditorDataRole = Qt.UserRole + 1
    EditorPositionRole = Qt.UserRole + 2
    DataSizeRole = Qt.UserRole + 3

    FlagVirtual = 1
    FlagEditable = 2
    FlagModified = 4

    dataChanged = pyqtSignal(ModelIndex, ModelIndex)
    modelReset = pyqtSignal()

    def __init__(self, editor=None):
        QObject.__init__(self)

        if editor is not None:
            with editor.lock:
                self.__editor = editor
                self.__editor.dataChanged.connect(self.onEditorDataChanged)
                self.__editor.resized.connect(self.onEditorDataResized)
        else:
            self.__editor = None

    @property
    def editor(self):
        return self.__editor

    def rowCount(self):
        """Should return -1 if model has infinite number of rows."""
        raise NotImplementedError()

    def realRowCount(self):
        """Should return positive integer, infinite number of real rows is not allowed"""
        return self.rowCount()

    def columnCount(self, row):
        """Should return -1 if no row exists"""
        raise NotImplementedError()

    def realColumnCount(self, row):
        return self.columnCount(row)

    def index(self, row, column):
        """Create index for row and column. Can return virtual index"""
        return ModelIndex(row, column, self) if self.hasIndex(row, column) else ModelIndex()

    def lastRowIndex(self, row):
        """Return last index on given row."""
        return self.index(row, self.columnCount(row) - 1)

    def lastRealRowIndex(self, row):
        return self.index(row, self.realColumnCount(row) - 1)

    def indexFlags(self, index):
        return 0

    def indexData(self, index, role=Qt.DisplayRole):
        return None

    def hasIndex(self, row, column):
        """Return True if there is index at row and column in this model"""
        if self.rowCount() < 0 or row < self.rowCount():
            return column < self.columnCount(row)
        return False

    def hasRealIndex(self, row, column):
        return row < self.realRowCount() and column < self.realColumnCount(row)

    def hasRow(self, row):
        return row >= 0 and (self.rowCount() < 0 or row + 1 < self.rowCount())

    @property
    def firstIndex(self):
        return self.index(0, 0)

    @property
    def lastIndex(self):
        """Return last index in the model"""
        if self.rowCount() <= 0:
            return ModelIndex()

        row = self.rowCount() - 1
        index = ModelIndex()
        while not index and row >= 0:
            index = self.lastRowIndex(row)
            row += 1
        return index

    @property
    def lastRealIndex(self):
        row = self.realRowCount() - 1
        index = ModelIndex()
        while not index and row >= 0:
            index = self.lastRealRowIndex(row)
            row -= 1
        return index

    def setIndexData(self, index, value, role=Qt.EditRole):
        return False

    def onEditorDataChanged(self, start, length):
        pass

    def onEditorDataResized(self, new_size):
        pass

    @property
    def preferSpaced(self):
        """Whether view should display indexes with spaces between them by default"""
        return True

    def indexFromPosition(self, position):
        """Return index matching given editor position. Can return virtual index"""
        raise NotImplementedError()

    @property
    def regular(self):
        """Return True if this model is regular. Regular models have same number of columns in each row and same
        text length for cell."""
        return False

    def createEditWidget(self, parent, index, style_options):
        return None


def index_range(start_index, end_index, include_last):
    """Iterator allows traversing through model indexes."""
    if not start_index or not end_index or end_index < start_index:
        return
    current_index = start_index
    while current_index:
        if include_last and not (current_index <= end_index):
            break
        elif not include_last and not (current_index < end_index):
            break
        yield current_index
        current_index = current_index.next


class EmptyColumnModel(ColumnModel):
    def rowCount(self):
        return 0

    def columnCount(self, row):
        return 0

    def indexFromPosition(self, position):
        return ModelIndex()


class HexColumnModel(ColumnModel):
    """Standart column for hex-editors. Displays data as numbers. This model is regular (has equal number of
    columns in each row, except last one) and infinite (supports virtual indexes).
    """

    def __init__(self, editor, valuecodec, formatter, columns_on_row=16):
        ColumnModel.__init__(self, editor)
        self.valuecodec = valuecodec
        self.formatter = formatter
        self._rowCount = 0
        self._bytesOnRow = columns_on_row * self.valuecodec.dataSize
        self._columnsOnRow = columns_on_row
        min_value, max_value = self.valuecodec.minimal, self.valuecodec.maximal
        self._cellTextSize = max(len(self.formatter.format(min_value)), len(self.formatter.format(max_value)))
        self._updateRowCount()

    def rowCount(self):
        return -1

    def columnCount(self, row):
        return self._columnsOnRow

    def realRowCount(self):
        return self._rowCount

    def realColumnCount(self, row):
        if row + 1 == self._rowCount:
            count = (len(self.editor) % self.bytesOnRow) // self.valuecodec.dataSize
            return count or self._columnsOnRow
        elif row >= self._rowCount:
            return 0
        return self._columnsOnRow

    def indexFromPosition(self, editor_position):
        return self.index(int(editor_position // self.bytesOnRow),
                          int(editor_position % self.bytesOnRow) // self.valuecodec.dataSize)

    def indexData(self, index, role=Qt.DisplayRole):
        if not index or self.editor is None:
            return None
        editor_position = self.bytesOnRow * index.row + self.valuecodec.dataSize * index.column

        if role == Qt.DisplayRole or role == Qt.EditRole:
            if editor_position >= len(self.editor):
                return '.' * self._cellTextSize
            else:
                editor_data = self.editor.read(editor_position, self.valuecodec.dataSize)
                return self.formatter.format(self.valuecodec.decode(editor_data))
        elif role == self.EditorDataRole:
            if editor_position >= len(self.editor):
                return bytes()
            else:
                return self.editor.read(editor_position, self.valuecodec.dataSize)
        elif role == self.DataSizeRole:
            return self.valuecodec.dataSize
        elif role == self.EditorPositionRole:
            return editor_position
        return None

    def setIndexData(self, index, value, role):
        if index and index.model is self:
            if role == self.EditorDataRole:
                position = self.indexData(index, ColumnModel.EditorPositionRole)
                if position is not None and position >= 0:
                    if value != index.data(self.EditorDataRole):
                        try:
                            self.editor.writeSpan(position, DataSpan(self.editor, value))
                            return True
                        except IOError:
                            pass
        return False

    def indexFlags(self, index):
        flags = ColumnModel.FlagEditable
        if index.row >= self._rowCount or (index.row == self._rowCount - 1 and index > self.lastRealIndex):
            flags |= ColumnModel.FlagVirtual
        elif self.editor is not None and self.editor.isRangeModified(index.data(self.EditorPositionRole),
                                                                   index.data(self.DataSizeRole)):
            flags |= ColumnModel.FlagModified
        return flags

    @property
    def bytesOnRow(self):
        return self._bytesOnRow

    def onEditorDataChanged(self, start, length):
        length = length if length >= 0 else len(self.editor) - start
        self.dataChanged.emit(self.indexFromPosition(start), self.indexFromPosition(start + length - 1))

    def onEditorDataResized(self, new_size):
        old_last_index = self.lastRealIndex
        self._updateRowCount()
        self.dataChanged.emit(old_last_index, self.lastRealIndex)

    @property
    def preferSpaced(self):
        return True

    def _updateRowCount(self):
        self._rowCount = len(self.editor) // self.bytesOnRow + bool(len(self.editor) % self.bytesOnRow)

    def createEditWidget(self, parent, index):
        return HexColumnEditWidget(parent, index)


class CharColumnModel(ColumnModel):
    """This column displays data as characters in one of possible encodings. It depends on encoding how many bytes
    will occupy each cell. Since even cells of one row can have different number of bytes, we can either have constant
    number of bytes for row or constant number of columns in it. In first case this leads to awful (unaligned) look of widget in
    case of multi-byte encodings (utf-8, for example). Second case leads to awful behaviour - CharColumn and HexColumn
    cannot be synchronized to display same bytes on same rows (this behaviour is common to other hex-editors).
    """

    ReplacementCharacter = '·'

    def __init__(self, editor, codec, render_font, bytes_on_row=16):
        ColumnModel.__init__(self, editor)
        self._rowCount = 0
        self._codec = codec

        # number of bytes on row should be multiplier of codec.unitSize
        if bytes_on_row % codec.unitSize:
            raise ValueError('number of bytes on row should be multiplier of encoding unit size')
        self._bytesOnRow = bytes_on_row

        self._renderFont = QRawFont.fromFont(render_font)

        self._updateRowCount()

    @property
    def codec(self):
        return self._codec

    @property
    def renderFont(self):
        return self._renderFont

    @renderFont.setter
    def renderFont(self, new_font):
        if isinstance(new_font, QFont):
            new_font = QRawFont.fromFont(new_font)
        self._renderFont = new_font
        self.modelReset.emit()

    def _updateRowCount(self):
        self._rowCount = len(self.editor) // self._bytesOnRow + bool(len(self.editor) % self._bytesOnRow)

    def rowCount(self):
        return -1

    def columnCount(self, row):
        return self._bytesOnRow // self._codec.unitSize

    def realRowCount(self):
        return self._rowCount

    def realColumnCount(self, row):
        position = row * self._bytesOnRow
        if position >= len(self.editor):
            return 0
        elif row + 1 == self._rowCount:
            bytes_left = position % self._bytesOnRow + 1
            return bytes_left // self._codec.unitSize + bool(bytes_left % self._codec.unitSize)
        else:
            return self._bytesOnRow // self._codec.unitSize

    def indexFromPosition(self, position):
        return self.index(position // self._bytesOnRow,
                         (position % self._bytesOnRow) // self._codec.unitSize)

    def indexData(self, index, role=Qt.DisplayRole):
        if not index or self.editor is None:
            return None

        position = index.row * self._bytesOnRow + index.column * self._codec.unitSize
        is_virtual = position >= len(self.editor)

        if role == self.EditorPositionRole:
            return position
        elif role == self.DataSizeRole:
            if is_virtual:
                return self._codec.unitSize
            try:
                return self._codec.getCharacterSize(self.editor, position)
            except encodings.EncodingError:
                return 1
        elif role == self.EditorDataRole:
            if is_virtual:
                return bytes()
            try:
                return self.editor.readAtEnd(self._codec.findCharacterStart(self.editor, position),
                                         self._codec.getCharacterSize(self.editor, position))
            except encodings.EncodingError:
                return self.editor.readAtEnd(position, 1)
        elif role in (Qt.DisplayRole, Qt.EditRole):
            if is_virtual:
                return '.'
            try:
                character_start = self._codec.findCharacterStart(self.editor, position)
                if character_start != position:
                    return ' '
                else:
                    decoded = self._codec.decodeCharacter(self.editor, position)
                    return self._translateToVisualCharacter(decoded)
            except encodings.EncodingError:
                return '∎'
        return None

    def indexFlags(self, index):
        flags = self.FlagEditable
        if index.row + 1 >= self.realRowCount() and index > self.lastRealIndex:
            flags |= self.FlagVirtual
        elif self.editor.isRangeModified(index.data(self.EditorPositionRole), index.data(self.DataSizeRole)):
            flags |= self.FlagModified
        return flags

    def setIndexData(self, index, value, role=Qt.DisplayRole):
        if not (index and self.editor is not None and role == Qt.EditorDataRole):
            return False

        position = self.indexData(index, self.EditorPositionRole)
        old_size = self._codec.getCharacterSize(self.editor, position)
        encoded = self._codec.encodeString(value)

        self.editor.beginComplexAction()
        try:
            self.editor.remove(position, old_size)
            self.editor.insertSpan(position, DataSpan(self.editor, encoded))
            return True
        finally:
            self.editor.endComplexAction()

    def _translateToVisualCharacter(self, text):
        import unicodedata

        result = ''
        for char in text:
            if unicodedata.category(char) in ('Cc', 'Cf', 'Cn', 'Co', 'Cs', 'Lm', 'Mc', 'Zl', 'Zp'):
                result += self.ReplacementCharacter
            elif not self._renderFont.supportsCharacter(char):
                result += self.ReplacementCharacter
            else:
                result += char
        return result

    @property
    def preferSpaced(self):
        return False

    @property
    def regular(self):
        return True


class AddressColumnModel(ColumnModel):
    def __init__(self, linked_model, formatter=None):
        ColumnModel.__init__(self, linked_model.editor)
        self._linkedModel = linked_model
        self._formatter = formatter or IntegerFormatter()
        self._baseAddress = 0
        self._updatePadding()

    @property
    def linkedModel(self):
        return self._linkedModel

    @linkedModel.setter
    def linkedModel(self, model):
        if self._linkedModel is not model:
            self._linkedModel = model
            self.modelReset.emit()

    @property
    def baseAddress(self):
        return self._baseAddress

    @baseAddress.setter
    def baseAddress(self, new_base):
        if self._baseAddress != new_base:
            self._baseAddress = new_base
            self.modelReset.emit()

    def rowCount(self):
        if self._linkedModel is not None:
            return self._linkedModel.rowCount()
        return 0

    def columnCount(self, row):
        return 1 if self._linkedModel is not None and self._linkedModel.hasRow(row) else 0

    def realRowCount(self):
        if self._linkedModel is not None:
            return self._linkedModel.realRowCount()
        return 0

    def realColumnCount(self, row):
        if self._linkedModel is not None:
            return int(bool(self._linkedModel.realColumnCount(row)))
        return 0

    def indexData(self, index, role=Qt.DisplayRole):
        if self._linkedModel is not None:
            model_index = self._linkedModel.index(index.row, 0)
            if role == Qt.DisplayRole and model_index:
                raw = model_index.data(self.EditorPositionRole) - self._baseAddress
                return self._formatter.format(raw) if self._formatter is not None else raw
            elif role == self.EditorPositionRole:
                return model_index.data(self.EditorPositionRole)
            elif role == self.DataSizeRole:
                return sum(index.data(self.DataSizeRole) for index in index_range(
                    model_index, self._linkedModel.lastRowIndex(index.row), include_last=True
                ))
        return None

    def indexFromPosition(self, position):
        if self._linkedModel is not None:
            model_index = self._linkedModel.indexFromPosition(position)
            return self.index(model_index.row, 0) if model_index else ModelIndex()
        return self._linkedModelone

    def indexFlags(self, index):
        if self._linkedModel is not None:
            model_index = self._linkedModel.index(index.row, 0)
            if model_index and model_index.flags & ColumnModel.FlagVirtual:
                return ColumnModel.FlagVirtual
        return 0

    def _maxLengthForSize(self, size):
        """Calculate maximal length of address text for given editor size"""
        sign1 = self._baseAddress > 0
        sign2 = self._baseAddress < len(self._linkedModel.editor)
        max_raw = max(abs(0 - self._baseAddress) + sign1,
                      abs(len(self._linkedModel.editor) - self._baseAddress) + sign2)
        return len(self._formatter.format(max_raw))

    def _updatePadding(self):
        if self._formatter is None:
            self._formatter = IntegerFormatter()
        self._formatter.padding = self._maxLengthForSize(len(self.editor))

    def onEditorDataResized(self, new_size):
        self._updatePadding()


class FrameProxyModel(ColumnModel):
    """This proxy model displays only number of rows starting after given first row. Proxy model does not check
    if there are indexes on these rows."""

    frameScrolled = pyqtSignal(int, int)  # first argument is new first frame row, second one is old first frame row
    frameResized = pyqtSignal(int, int)  # first argument is new frame size, second one is old frame size

    def __init__(self, source_model):
        ColumnModel.__init__(self, None)
        self.__firstRow = 0
        self.__rowCount = 0
        self.sourceModel = source_model or EmptyColumnModel()
        self.sourceModel.dataChanged.connect(self.__onDataChanged)
        self.sourceModel.modelReset.connect(self.modelReset)

    def setFrame(self, first_row, row_count):
        self.resizeFrame(row_count)
        self.scrollFrame(first_row)

    def scrollFrame(self, new_first_row):
        if self.__firstRow != new_first_row:
            old_first_row = self.__firstRow
            self.__firstRow = new_first_row
            self.frameScrolled.emit(new_first_row, old_first_row)

    def resizeFrame(self, new_frame_size):
        if self.__rowCount != new_frame_size:
            old_size = self.__rowCount
            self.__rowCount = new_frame_size
            self.frameResized.emit(new_frame_size, old_size)

    def rowCount(self):
        model_row_count = self.sourceModel.rowCount()
        if model_row_count >= 0:
            return min(self.__rowCount, model_row_count)
        else:
            return self.__rowCount

    def columnCount(self, row):
        return self.sourceModel.columnCount(row + self.__firstRow)

    def realRowCount(self):
        return min(self.__rowCount, self.sourceModel.rowCount())

    def realColumnCount(self, row):
        return self.sourceModel.realColumnCount(row + self.__firstRow)

    def indexFromPosition(self, position):
        return self.fromSourceIndex(self.sourceModel.indexFromPosition(position))

    def index(self, row, column=0):
        return self.fromSourceIndex(self.sourceModel.index(row + self.__firstRow, column))

    def hasIndex(self, row, column):
        return ColumnModel.hasIndex(self, row, column) and self.sourceModel.hasIndex(row + self.__firstRow, column)

    def hasRealIndex(self, row, column):
        return ColumnModel.hasIndex(self, row, column) and self.sourceModel.hasRealIndex(row + self.__firstRow, column)

    def indexData(self, index, role=Qt.DisplayRole):
        return self.sourceModel.indexData(self.toSourceIndex(index), role)

    def indexFlags(self, index):
        return self.sourceModel.indexFlags(self.toSourceIndex(index))

    def setIndexData(self, index, value, role=Qt.EditRole):
        return self.sourceModel.setIndexData(self.toSourceIndex(index), value, role)

    def toSourceIndex(self, index):
        if not index or index.model is self.sourceModel:
            return index
        elif self.hasIndex(index.row, index.column):
            return self.sourceModel.index(index.row + self.__firstRow, index.column)
        else:
            return ModelIndex()

    def fromSourceIndex(self, index):
        if not index or index.model is self:
            return index
        elif self.hasIndex(index.row - self.__firstRow, index.column):
            return ModelIndex(index.row - self.__firstRow, index.column, self)
        else:
            return ModelIndex()

    def __onDataChanged(self, first, last):
        # check if update area lays inside frame
        if last.row < self.__firstRow or first.row > self.toSourceIndex(self.lastIndex).row:
            return

        first = self.fromSourceIndex(first) or self.firstIndex
        last = self.fromSourceIndex(last) or self.lastIndex

        self.dataChanged.emit(first, last)

    @property
    def regular(self):
        return self.sourceModel.regular


DefaultFont = QFont('Ubuntu Mono', 13)
VisualSpace = 10

# default color theme
BackgroundColor = QColor(250, 250, 245)
TextColor = QColor(Qt.black)
BorderColor = QColor(Qt.black)
InactiveTextColor = QColor(Qt.darkGray)
CaretBackgroundColor = QColor(150, 250, 160, 100)
CaretBorderColor = QColor(0, 0, 0, 255)
SelectionBackgroundColor = QColor(220, 250, 245, 100)
SelectionBorderColor = QColor(20, 205, 195)
# EditCaretBackgroundColor = QColor(0, 0, 0, 150)
# EditCaretBorderColor = EditCaretBackgroundColor
EditCaretTextColor = QColor(Qt.white)
ModifiedTextColor = QColor(Qt.red)


class RowData(object):
    def __init__(self):
        self.text = ''
        self.html = ''
        self.items = []


class IndexData(object):
    def __init__(self, index):
        self._text = None
        self.index = index
        self.firstCharIndex = 0
        self.firstHtmlCharIndex = 0
        self.html = None
        self.color = None

    @property
    def text(self):
        return self.data()

    def data(self, role=Qt.DisplayRole):
        if role == Qt.DisplayRole:
            if self._text is None:
                self._text = self.index.data()
            return self._text
        if self.index:
            return self.index.data(role)
        return None


class ColumnDocumentBackend(QObject):
    documentUpdated = pyqtSignal()

    def __init__(self, column):
        QObject.__init__(self)
        self._document = None
        self._column = column
        
    @property
    def document(self):
        if self._document is None:
            self.generateDocument()
        return self._document

    @property
    def generated(self):
        return self._document is not None

    def generateDocument(self):
        raise NotImplementedError()

    def updateRow(self, row_index, cached_row):
        raise NotImplementedError()

    def removeRows(self, row_index, number_of_rows):
        raise NotImplementedError()

    def insertRow(self, row_index, number_of_rows):
        raise NotImplementedError()

    def rectForCell(self, row_index, cell_index):
        raise NotImplementedError()

    def cellForPoint(self, point):
        raise NotImplementedError()

    def clear(self):
        self._document = None


class PlainTextDocumentBackend(ColumnDocumentBackend):
    def __init__(self, column):
        ColumnDocumentBackend.__init__(self, column)

    def generateDocument(self):
        if self._document is None:
            self._document = self._column.createDocumentTemplate()

            cursor = QTextCursor(self._document)
            block_format = self._documentBlockFormat
            for row_data in self._column._cache:
                cursor.movePosition(QTextCursor.End)
                cursor.insertHtml(row_data.html)
                cursor.insertBlock()
                cursor.setBlockFormat(block_format)

            self.documentUpdated.emit()

    @property
    def _documentBlockFormat(self):
        block_format = QTextBlockFormat()
        block_format.setLineHeight(self._column._fontMetrics.height(), QTextBlockFormat.FixedHeight)
        return block_format

    def updateRow(self, row_index, row_data):
        if self._document is not None:
            block = self._document.findBlockByLineNumber(row_index)
            if block.isValid():
                cursor = QTextCursor(block)
                cursor.movePosition(QTextCursor.EndOfBlock, QTextCursor.KeepAnchor)
                cursor.insertHtml(row_data.html)

                self.documentUpdated.emit()

    def removeRows(self, row_index, number_of_rows):
        if self._document is not None:
            for x in range(number_of_rows):
                block = self._document.findBlockByLineNumber(row_index)
                if block.isValid():
                    cursor = QTextCursor(block)
                    cursor.movePosition(QTextCursor.EndOfBlock, QTextCursor.KeepAnchor)
                    cursor.removeSelectedText()
                    if row_index == 0:
                        cursor.deleteChar()

            self.documentUpdated.emit()

    def insertRows(self, row_index, number_of_rows):
        if self._document is not None:
            for x in range(number_of_rows):
                if row_index < 0:
                    cursor = QTextCursor(self._document)
                    cursor.movePosition(QTextCursor.End)
                else:
                    block = self._document.findBlockByLineNumber(row_index)
                    cursor = QTextCursor(block)

                cursor.insertBlock()
                cursor.setBlockFormat(self._documentBlockFormat)

            self.documentUpdated.emit()

    def rectForCell(self, row_index, cell_index):
        self.generateDocument()

        try:
            index_data = self._column._cache[row_index].items[cell_index]
        except IndexError:
            return QRectF()

        if index_data is not None:
            block = self._document.findBlockByLineNumber(row_index)
            if block.isValid():
                block_rect = self._document.documentLayout().blockBoundingRect(block)
                line = block.layout().lineAt(0)
                x = line.cursorToX(index_data.firstCharIndex)[0] + VisualSpace
                y = block_rect.y() + line.position().y()
                width = self._column._fontMetrics.width(index_data.text)
                return QRectF(x, y, width, self._column._fontMetrics.height())
        return QRectF()

    def cellForPoint(self, point):
        self.generateDocument()

        point = QPointF(point.x() - VisualSpace, point.y())
        char_position = self._document.documentLayout().hitTest(point, Qt.ExactHit)
        block = self._document.findBlock(char_position)
        if block.isValid():
            row_index = block.firstLineNumber()
            row_data = self._column._cache[row_index]
            line_char_index = char_position - block.position()
            for column in range(len(row_data.items)):
                index_data = row_data.items[column]
                if index_data.firstCharIndex + len(index_data.text) > line_char_index:
                    return row_index, column
        return -1, -1


class TableDocumentBackend(ColumnDocumentBackend):
    def __init__(self, column):
        ColumnDocumentBackend.__init__(self, column)

    def generateDocument(self):
        if self._document is None:
            self._document = self._column.createDocumentTemplate()

            cursor = QTextCursor(self._document)

            cursor.beginEditBlock()
            try:
                self._table = cursor.insertTable(len(self._column._cache), self._column.model.columnCount(0))
                self._table.setFormat(self._tableFormat)
                for row_index in range(len(self._column._cache)):
                    for column_index in range(self._table.columns()):
                        cell = self._table.cellAt(row_index, column_index)
                        cursor = cell.firstCursorPosition()
                        try:
                            cursor.insertHtml(self._column._cache[row_index].items[column_index].html)
                        except IndexError:
                            pass
            finally:
                cursor.endEditBlock()

            self.documentUpdated.emit()

    @property
    def _tableFormat(self):
        fmt = QTextTableFormat()
        fmt.setBorderStyle(QTextTableFormat.BorderStyle_None)
        fmt.setBorder(0)
        fmt.setCellPadding(0)
        fmt.setCellSpacing(0)
        return fmt

    def updateRow(self, row_index, row_data):
        if self._document is not None:
            cursor = self._table.cellAt(row_index, 0).firstCursorPosition()
            cursor.movePosition(QTextCursor.NextRow)
            cursor.removeSelectedText()
            cursor.beginEditBlock()  # we do not use undo/redo, but this thing increases perfomance, blocking updating
                                     # layout until endEditBlock is called.
            try:
                for column_index in range(self._table.columns()):
                    cursor = self._table.cellAt(row_index, column_index).firstCursorPosition()
                    cursor.insertHtml(row_data.items[column_index].html)
            finally:
                cursor.endEditBlock()

            self.documentUpdated.emit()

    def removeRows(self, row_index, number_of_rows):
        if self._document is not None:
            self._table.removeRows(row_index, number_of_rows)
            self.documentUpdated.emit()

    def insertRows(self, row_index, number_of_rows):
        if self._document is not None:
            if row_index < 0:
                self._table.appendRows(number_of_rows)
            else:
                self._table.insertRows(row_index, number_of_rows)

            self.documentUpdated.emit()

    def rectForCell(self, row_index, cell_index):
        self.generateDocument()

        cursor = self._table.cellAt(row_index, cell_index).firstCursorPosition()
        if not cursor.isNull():
            r = self._document.documentLayout().blockBoundingRect(cursor.block())
            r.translate(VisualSpace, 0)
            return r

        return QRectF()

    def cellForPoint(self, point):
        self.generateDocument()

        for row_index in range(len(self._column._cache)):
            for column_index in range(self._table.columns()):
                if self.rectForCell(row_index, column_index).contains(point):
                    return row_index, column_index
        return -1, -1



class Column(QObject):
    updateRequested = pyqtSignal()
    resizeRequested = pyqtSignal(QSizeF)

    def __init__(self, model):
        QObject.__init__(self)
        self.sourceModel = model
        self.model = FrameProxyModel(model)
        self.model.dataChanged.connect(self._onDataChanged)
        self.model.modelReset.connect(self._updateCache)
        self.model.frameScrolled.connect(self._onFrameScrolled)
        self.model.frameResized.connect(self._onFrameResized)

        self._geom = QRectF()
        self._font = DefaultFont
        self._fontMetrics = QFontMetricsF(self._font)

        self._fullVisibleRows = 0
        self._visibleRows = 0
        self._firstVisibleRow = 0

        self._spaced = self.sourceModel.preferSpaced
        self._cache = []
        # if self.sourceModel.regular:
        #     self._documentBackend = TableDocumentBackend(self)
        # else:
        self._documentBackend = PlainTextDocumentBackend(self)
        self._documentBackend.documentUpdated.connect(self._onDocumentUpdated)

    @property
    def geometry(self):
        return self._geom

    @geometry.setter
    def geometry(self, rect):
        self._geom = rect
        self._updateGeometry()
        self.updateRequested.emit()  # even if frame not changed, we should redraw widget

    @property
    def firstVisibleRow(self):
        return self._firstVisibleRow

    def scrollToFirstRow(self, source_row_index):
        if self._firstVisibleRow != source_row_index:
            self._firstVisibleRow = source_row_index
            self.model.scrollFrame(self._firstVisibleRow)

    @property
    def lastFullVisibleRow(self):
        if self._fullVisibleRows > 0:
            return self._firstVisibleRow + self._fullVisibleRows - 1
        return self._firstVisibleRow

    @property
    def lastVisibleRow(self):
        if self._visibleRows > 0:
            return self._firstVisibleRow + self._visibleRows - 1
        return self._firstVisibleRow

    @property
    def visibleRows(self):
        return self._visibleRows

    @property
    def fullVisibleRows(self):
        return self._fullVisibleRows

    @property
    def firstVisibleIndex(self):
        return self.model.index(0, 0)

    @property
    def lastVisibleIndex(self):
        return self.model.lastRowIndex(self._visibleRows - 1)

    @property
    def lastFullVisibleIndex(self):
        return self.model.lastRowIndex(self._fullVisibleRows - 1)

    @property
    def font(self):
        return self._font

    @font.setter
    def font(self, new_font):
        self._font = new_font
        self._fontMetrics = QFontMetricsF(new_font)
        if hasattr(self.sourceModel, 'renderFont'):  # well, this is hack until i invent better solution...
            self.sourceModel.renderFont = new_font
        self._documentBackend.clear()
        self._updateGeometry()

    @property
    def editor(self):
        return self.sourceModel.editor

    @property
    def spaced(self):
        return self._spaced

    @spaced.setter
    def spaced(self, new_spaced):
        if self._spaced != new_spaced:
            self._spaced = new_spaced
            self._updateCache()

    def getRowCachedData(self, visible_row_index):
        return self._cache[visible_row_index] if 0 <= visible_row_index < len(self._cache) else None

    def getIndexCachedData(self, index):
        index = self.model.fromSourceIndex(index)
        row_data = self.getRowCachedData(index.row)
        if row_data is not None and index.column < len(row_data.items):
            return row_data.items[index.column]
        return None

    def getRectForIndex(self, index):
        index = self.model.fromSourceIndex(index)
        return self._documentBackend.rectForCell(index.row, index.column)

    # def getRectForRow(self, visible_row_index):
    #     if 0 <= visible_row_index < self._visibleRows:
    #         block = self._document.findBlockByLineNumber(visible_row_index)
    #         if block.isValid():
    #             rect = block.layout().lineAt(0).rect()
    #             rect.translate(VisualSpace, 0)
    #             return rect
    #     return QRectF()

    def getPolygonsForRange(self, first_index, last_index):
        first_index = self.model.toSourceIndex(first_index)
        last_index = self.model.toSourceIndex(last_index)

        first_visible_source_index = self.model.toSourceIndex(self.firstVisibleIndex)
        last_visible_source_index = self.model.toSourceIndex(self.lastVisibleIndex)
        if not first_index or not last_index or (first_index > last_visible_source_index or
                                                         last_index < first_visible_source_index):
            return tuple()

        first_index = max(first_index, first_visible_source_index)
        last_index = min(last_index, last_visible_source_index)

        if first_index == last_index:
            return QPolygonF(self.getRectForIndex(first_index)),
        else:
            r1 = self.getRectForIndex(first_index)
            r2 = self.getRectForIndex(last_index)

            if first_index.row == last_index.row:
                return QPolygonF(QRectF(r1.topLeft(), r2.bottomRight())),
            elif first_index.row + 1 == last_index.row and r1.left() > r2.right():
                first_row_last_index = self.sourceModel.lastRowIndex(first_index.row)
                return (
                    QPolygonF(QRectF(r1.topLeft(), self.getRectForIndex(first_row_last_index).bottomRight())),
                    QPolygonF(QRectF(QPointF(VisualSpace, r2.top()), r2.bottomRight()))
                )
            else:
                range_polygon = QPolygonF()

                for row_index in range(first_index.row, last_index.row):
                    index = self.sourceModel.index(row_index, self.sourceModel.columnCount(row_index) - 1)
                    index_rect = self.getRectForIndex(index)
                    range_polygon.append(index_rect.topRight())
                    range_polygon.append(index_rect.bottomRight())

                range_polygon.append(r2.topRight())
                range_polygon.append(r2.bottomRight())
                range_polygon.append(self.getRectForIndex(self.sourceModel.index(last_index.row, 0)).bottomLeft())
                range_polygon.append(self.getRectForIndex(self.sourceModel.index(first_index.row, 0)).bottomLeft())
                range_polygon.append(r1.bottomLeft())
                range_polygon.append(r1.topLeft())

                return range_polygon,

    def indexFromPoint(self, point):
        row, column = self._documentBackend.cellForPoint(point)
        return self.model.toSourceIndex(self.model.index(row, column))

    def paint(self, paint_data, is_leading):
        painter = paint_data.painter
        painter.save()

        painter.setPen(TextColor if is_leading else InactiveTextColor)
        painter.translate(VisualSpace, 0)

        # little trick to quickly change default text color for document without re-generating it
        paint_context = QAbstractTextDocumentLayout.PaintContext()
        paint_context.palette.setColor(QPalette.Text, TextColor if is_leading else InactiveTextColor)
        # standard QTextDocument.draw also sets clip rect here, but we already have one
        self._documentBackend.document.documentLayout().draw(painter, paint_context)

        painter.restore()

    def paintCaret(self, paint_data, is_leading, caret_position):
        painter = paint_data.painter
        if caret_position >= 0:
            caret_index = self.model.fromSourceIndex(self.sourceModel.indexFromPosition(caret_position))
            if caret_index and self.isIndexVisible(caret_index, False):
                caret_rect = self.getRectForIndex(caret_index)
                painter.setBrush(QBrush(CaretBackgroundColor))
                painter.setPen(CaretBorderColor)
                painter.drawRect(caret_rect)

    def paintSelection(self, paint_data, is_leading, selection):
        if selection is not None and len(selection) > 0:
            painter = paint_data.painter
            for sel_polygon in self.getPolygonsForRange(self.sourceModel.indexFromPosition(selection.start),
                                        self.sourceModel.indexFromPosition(selection.start + len(selection) -1)):
                painter.setBrush(SelectionBackgroundColor)
                painter.setPen(QPen(QBrush(SelectionBorderColor), 2.0))
                painter.drawPolygon(sel_polygon)

    def _updateGeometry(self):
        self._fullVisibleRows = int(self._geom.height() // self._fontMetrics.height())
        self._visibleRows = self._fullVisibleRows + bool(int(self._geom.height()) % int(self._fontMetrics.height()))
        self.model.resizeFrame(self._visibleRows)

    def _updateCache(self):
        self._cache = [None for row in range(self._visibleRows)]
        self._documentBackend.clear()

        for row in range(self._visibleRows):
            self._updateCachedRow(row)

        self.updateRequested.emit()

    def _onDocumentUpdated(self):
        ideal_width = self._documentBackend._document.idealWidth() + VisualSpace * 2
        if ideal_width != self._geom.width():
            self.resizeRequested.emit(QSizeF(ideal_width, self._geom.height()))

    def _updateCachedRow(self, row_index):
        assert(0 <= row_index < len(self._cache))

        row_data = RowData()
        column_count = self.model.columnCount(row_index)
        for column_index in range(column_count):
            index = self.model.toSourceIndex(self.model.index(row_index, column_index))
            index_data = IndexData(index)
            index_data.firstCharIndex = len(row_data.text)
            index_data.firstHtmlCharIndex = len(row_data.html)
            index_text = index_data.data()

            if index_text is not None:
                cell_classes = []
                if index.flags & ColumnModel.FlagModified:
                    cell_classes.append('cell-mod')

                prepared_text = html.escape(index_text)
                prepared_text = prepared_text.replace(' ', '&nbsp;')
                if cell_classes:
                    # unfortunately, HTML subset supported by Qt does not include multiclasses
                    for css_class in cell_classes:
                        index_html = '<span class={0}>{1}</span>'.format(css_class, prepared_text)
                else:
                    index_html = '<span>{0}</span>'.format(prepared_text)

                row_data.html += index_html
                row_data.text += index_text
                index_data.html = index_html

            if self.spaced and column_index + 1 < column_count:
                row_data.text += ' '
                row_data.html += '<span> </span>'

            row_data.items.append(index_data)

        self._cache[row_index] = row_data
        self._documentBackend.updateRow(row_index, row_data)

    def _onFrameScrolled(self, new_first_row, old_first_row):
{'fraction_non_alphanumeric': 0.08540529078089382, 'fraction_numerical': 0.004442559508031596, 'mean_word_length': 2.332720848056537, 'pattern_counts': {'":': 0, '<': 71, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4513262', 'n_tokens_mistral': 26192, 'n_tokens_neox': 25065, 'n_words': 6297}