<commit_before># Expose (a subset of) the reading functionality GNU idutils within Kate.
#
# Copyright (C) 2012 <NAME> <<EMAIL>>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) version 3.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public License
# along with this library; see the file COPYING.LIB.  If not, write to
# the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
# Boston, MA 02110-1301, USA.

__title__ = "gid"
__author__ = "<NAME> <<EMAIL>>"
__license__ = "LGPL"

import kate
import kate.gui

from PyQt4.QtCore import *
from PyQt4.QtGui import *
from PyKDE4.kdecore import *
from PyKDE4.kdeui import *
from PyKDE4.ktexteditor import KTextEditor
from idutils import Lookup

import sip

searchBar = None

class ConfigWidget(QWidget):
    """Configuration widget for this plugin."""
    #
    # Location of ID file.
    #
    idFile = None
    #
    # Completion string minimum size.
    #
    keySize = None
    #
    # Original file prefix.
    #
    srcIn = None
    #
    # Replacement file prefix.
    #
    srcOut = None

    def __init__(self, parent = None, name = None):
	super(ConfigWidget, self).__init__(parent)

	lo = QGridLayout()
	self.setLayout(lo)

	self.idFile = KLineEdit()
	self.idFile.setWhatsThis(i18n("Name of ID file."))
	lo.addWidget(self.idFile, 0, 1, 1, 1)
	self.labelIdFile = QLabel(i18n("&ID file:"))
	self.labelIdFile.setBuddy(self.idFile)
	lo.addWidget(self.labelIdFile, 0, 0, 1, 1)

	self.keySize = QSpinBox()
	self.keySize.setMinimum(3)
	self.keySize.setMaximum(16)
	self.keySize.setWhatsThis(i18n("Minimum length of token before completions will be shown."))
	lo.addWidget(self.keySize, 0, 3, 1, 1)
	self.labelKeySize = QLabel(i18n("&Complete after:"))
	self.labelKeySize.setBuddy(self.keySize)
	lo.addWidget(self.labelKeySize, 0, 2, 1, 1)

	self.srcIn = KLineEdit()
	self.srcIn.setWhatsThis(i18n("If not empty, when looking in a file for matches, replace this prefix of the file name."))
	lo.addWidget(self.srcIn, 1, 1, 1, 3)
	self.labelSrcIn = QLabel(i18n("&Replace:"))
	self.labelSrcIn.setBuddy(self.srcIn)
	lo.addWidget(self.labelSrcIn, 1, 0, 1, 1)

	self.srcOut = KLineEdit()
	self.srcOut.setWhatsThis(i18n("Replacement prefix. Use %i to insert the prefix of the ID file."))
	lo.addWidget(self.srcOut, 2, 1, 1, 3)
	self.labelSrcOut = QLabel(i18n("&With this:"))
	self.labelSrcOut.setBuddy(self.srcOut)
	lo.addWidget(self.labelSrcOut, 2, 0, 1, 1)

	self.reset();

    def apply(self):
	kate.configuration["idFile"] = self.idFile.text().encode("utf-8")
	kate.configuration["keySize"] = self.keySize.value()
	kate.configuration["srcIn"] = self.srcIn.text().encode("utf-8")
	kate.configuration["srcOut"] = self.srcOut.text().encode("utf-8")
	kate.configuration.save()

    def reset(self):
	self.defaults()
	if "idFile" in kate.configuration:
	    self.idFile.setText(kate.configuration["idFile"])
	if "keySize" in kate.configuration:
	    self.keySize.setValue(kate.configuration["keySize"])
	if "srcIn" in kate.configuration:
	    self.srcIn.setText(kate.configuration["srcIn"])
	if "srcOut" in kate.configuration:
	    self.srcOut.setText(kate.configuration["srcOut"])

    def defaults(self):
	self.idFile.setText("/view/myview/vob/ID")
	self.keySize.setValue(5)
	self.srcIn.setText("/vob")
	self.srcOut.setText("%i/vob")

class ConfigPage(kate.Kate.PluginConfigPage, QWidget):
    """Kate configuration page for this plugin."""
    def __init__(self, parent = None, name = None):
	super(ConfigPage, self).__init__(parent, name)
	self.widget = ConfigWidget(parent)
	lo = parent.layout()
	lo.addWidget(self.widget)

    def apply(self):
	self.widget.apply()

    def reset(self):
	self.widget.reset()

    def defaults(self):
	self.widget.defaults()
	self.changed.emit()

class ConfigDialog(KDialog):
    """Standalong configuration dialog for this plugin."""
    def __init__(self, parent = None):
	super(ConfigDialog, self).__init__(parent)
	self.widget = ConfigWidget(self)
	self.setMainWidget(self.widget)
	self.setButtons(KDialog.ButtonCode(KDialog.Default | KDialog.Reset | KDialog.Ok | KDialog.Cancel))
	self.applyClicked.connect(self.apply)
	self.resetClicked.connect(self.reset)
	self.defaultClicked.connect(self.defaults)

    @pyqtSlot()
    def apply(self):
	self.widget.apply()

    @pyqtSlot()
    def reset(self):
	self.widget.reset()

    @pyqtSlot()
    def defaults(self):
	self.widget.defaults()

def transform(file):
    """Return the transformed file name."""
    transformationKey = kate.configuration["srcIn"]
    if len(transformationKey):
	#
	# A transformation of the file name is requested.
	#
	try:
	    left, right = file.split(transformationKey, 1)
	except ValueError:
	    #
	    # No transformation is applicable.
	    #
	    return file
	percentI = kate.configuration["srcOut"].find("%i")
	if percentI > -1:
	    insertLeft, discard = kate.configuration["idFile"].split(transformationKey, 1)
	    discard, insertRight = kate.configuration["srcOut"].split("%i", 1)
	    file = insertLeft + insertRight
	else:
	    file = kate.configuration["srcOut"] + right
    return file

def wordAtCursorPosition(line, cursor):
    ''' Get the word under the active view's cursor in the given document.
    Stolen from the expand plugin!'''
    wordBoundary = set(u'. \t"\';[]{}()#:/\\,+=!?%^|&*~`')
    # better to use word boundaries than to hardcode valid letters because
    # expansions should be able to be in any unicode character.
    start = end = cursor.column()
    if start == len(line) or line[start] in wordBoundary:
	start -= 1
    while start >= 0 and line[start] not in wordBoundary:
	start -= 1
    start += 1
    while end < len(line) and line[end] not in wordBoundary:
	end += 1
    return start, end

def wordAtCursor(document, view):
    ''' Get the word under the active view's cursor in the given document.
    Stolen from the expand plugin!'''
    cursor = view.cursorPosition()
    line = unicode(document.line(cursor.line()))
    start, end = wordAtCursorPosition(line, cursor)
    return line[start:end]

class TreeModel(QStandardItemModel):
    def __init__(self, dataSource):
	super(TreeModel, self).__init__()
	self.dataSource = dataSource
	self.setHorizontalHeaderLabels((i18n("Match"), i18n("Line"), i18n("Col")))

    def literalSearch(self, token):
	root = self.invisibleRootItem()
	root.removeRows(0, root.rowCount())
	try:
	    tokenFlags, hitCount, files = self.dataSource.literalSearch(token)
	    #
	    # For each file, list the lines where a match is found.
	    #
	    for fileName, fileFlags in files:
		fileName = transform(fileName)
		fileRow = QStandardItem(fileName)
		root.appendRow(fileRow)
		line = 0
		for text in open(fileName):
		    column = text.find(token)
		    if column > -1:
			resultRow = list()
			resultRow.append(QStandardItem(text[:-1]))
			resultRow.append(QStandardItem(str(line + 1)))
			resultRow.append(QStandardItem(str(column + 1)))<commit_msg>Temorary approach for 8-bit files.<commit_after># Expose (a subset of) the reading functionality GNU idutils within Kate.
#
# Copyright (C) 2012 <NAME> <<EMAIL>>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) version 3.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public License
# along with this library; see the file COPYING.LIB.  If not, write to
# the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
# Boston, MA 02110-1301, USA.

__title__ = "gid"
__author__ = "<NAME> <<EMAIL>>"
__license__ = "LGPL"

import kate
import kate.gui

from PyQt4.QtCore import *
from PyQt4.QtGui import *
from PyKDE4.kdecore import *
from PyKDE4.kdeui import *
from PyKDE4.ktexteditor import KTextEditor

import codecs
from idutils import Lookup

import sip

searchBar = None

class ConfigWidget(QWidget):
    """Configuration widget for this plugin."""
    #
    # Location of ID file.
    #
    idFile = None
    #
    # Completion string minimum size.
    #
    keySize = None
    #
    # Original file prefix.
    #
    srcIn = None
    #
    # Replacement file prefix.
    #
    srcOut = None

    def __init__(self, parent = None, name = None):
	super(ConfigWidget, self).__init__(parent)

	lo = QGridLayout()
	self.setLayout(lo)

	self.idFile = KLineEdit()
	self.idFile.setWhatsThis(i18n("Name of ID file."))
	lo.addWidget(self.idFile, 0, 1, 1, 1)
	self.labelIdFile = QLabel(i18n("&ID file:"))
	self.labelIdFile.setBuddy(self.idFile)
	lo.addWidget(self.labelIdFile, 0, 0, 1, 1)

	self.keySize = QSpinBox()
	self.keySize.setMinimum(3)
	self.keySize.setMaximum(16)
	self.keySize.setWhatsThis(i18n("Minimum length of token before completions will be shown."))
	lo.addWidget(self.keySize, 0, 3, 1, 1)
	self.labelKeySize = QLabel(i18n("&Complete after:"))
	self.labelKeySize.setBuddy(self.keySize)
	lo.addWidget(self.labelKeySize, 0, 2, 1, 1)

	self.srcIn = KLineEdit()
	self.srcIn.setWhatsThis(i18n("If not empty, when looking in a file for matches, replace this prefix of the file name."))
	lo.addWidget(self.srcIn, 1, 1, 1, 3)
	self.labelSrcIn = QLabel(i18n("&Replace:"))
	self.labelSrcIn.setBuddy(self.srcIn)
	lo.addWidget(self.labelSrcIn, 1, 0, 1, 1)

	self.srcOut = KLineEdit()
	self.srcOut.setWhatsThis(i18n("Replacement prefix. Use %i to insert the prefix of the ID file."))
	lo.addWidget(self.srcOut, 2, 1, 1, 3)
	self.labelSrcOut = QLabel(i18n("&With this:"))
	self.labelSrcOut.setBuddy(self.srcOut)
	lo.addWidget(self.labelSrcOut, 2, 0, 1, 1)

	self.reset();

    def apply(self):
	kate.configuration["idFile"] = self.idFile.text().encode("utf-8")
	kate.configuration["keySize"] = self.keySize.value()
	kate.configuration["srcIn"] = self.srcIn.text().encode("utf-8")
	kate.configuration["srcOut"] = self.srcOut.text().encode("utf-8")
	kate.configuration.save()

    def reset(self):
	self.defaults()
	if "idFile" in kate.configuration:
	    self.idFile.setText(kate.configuration["idFile"])
	if "keySize" in kate.configuration:
	    self.keySize.setValue(kate.configuration["keySize"])
	if "srcIn" in kate.configuration:
	    self.srcIn.setText(kate.configuration["srcIn"])
	if "srcOut" in kate.configuration:
	    self.srcOut.setText(kate.configuration["srcOut"])

    def defaults(self):
	self.idFile.setText("/view/myview/vob/ID")
	self.keySize.setValue(5)
	self.srcIn.setText("/vob")
	self.srcOut.setText("%i/vob")

class ConfigPage(kate.Kate.PluginConfigPage, QWidget):
    """Kate configuration page for this plugin."""
    def __init__(self, parent = None, name = None):
	super(ConfigPage, self).__init__(parent, name)
	self.widget = ConfigWidget(parent)
	lo = parent.layout()
	lo.addWidget(self.widget)

    def apply(self):
	self.widget.apply()

    def reset(self):
	self.widget.reset()

    def defaults(self):
	self.widget.defaults()
	self.changed.emit()

class ConfigDialog(KDialog):
    """Standalong configuration dialog for this plugin."""
    def __init__(self, parent = None):
	super(ConfigDialog, self).__init__(parent)
	self.widget = ConfigWidget(self)
	self.setMainWidget(self.widget)
	self.setButtons(KDialog.ButtonCode(KDialog.Default | KDialog.Reset | KDialog.Ok | KDialog.Cancel))
	self.applyClicked.connect(self.apply)
	self.resetClicked.connect(self.reset)
	self.defaultClicked.connect(self.defaults)

    @pyqtSlot()
    def apply(self):
	self.widget.apply()

    @pyqtSlot()
    def reset(self):
	self.widget.reset()

    @pyqtSlot()
    def defaults(self):
	self.widget.defaults()

def transform(file):
    """Return the transformed file name."""
    transformationKey = kate.configuration["srcIn"]
    if len(transformationKey):
	#
	# A transformation of the file name is requested.
	#
	try:
	    left, right = file.split(transformationKey, 1)
	except ValueError:
	    #
	    # No transformation is applicable.
	    #
	    return file
	percentI = kate.configuration["srcOut"].find("%i")
	if percentI > -1:
	    insertLeft, discard = kate.configuration["idFile"].split(transformationKey, 1)
	    discard, insertRight = kate.configuration["srcOut"].split("%i", 1)
	    file = insertLeft + insertRight
	else:
	    file = kate.configuration["srcOut"] + right
    return file

def wordAtCursorPosition(line, cursor):
    ''' Get the word under the active view's cursor in the given document.
    Stolen from the expand plugin!'''
    wordBoundary = set(u'. \t"\';[]{}()#:/\\,+=!?%^|&*~`')
    # better to use word boundaries than to hardcode valid letters because
    # expansions should be able to be in any unicode character.
    start = end = cursor.column()
    if start == len(line) or line[start] in wordBoundary:
	start -= 1
    while start >= 0 and line[start] not in wordBoundary:
	start -= 1
    start += 1
    while end < len(line) and line[end] not in wordBoundary:
	end += 1
    return start, end

def wordAtCursor(document, view):
    ''' Get the word under the active view's cursor in the given document.
    Stolen from the expand plugin!'''
    cursor = view.cursorPosition()
    line = unicode(document.line(cursor.line()))
    start, end = wordAtCursorPosition(line, cursor)
    return line[start:end]

class TreeModel(QStandardItemModel):
    def __init__(self, dataSource):
	super(TreeModel, self).__init__()
	self.dataSource = dataSource
	self.setHorizontalHeaderLabels((i18n("Match"), i18n("Line"), i18n("Col")))

    def literalSearch(self, token):
	root = self.invisibleRootItem()
	root.removeRows(0, root.rowCount())
	try:
	    tokenFlags, hitCount, files = self.dataSource.literalSearch(token)
	    #
	    # For each file, list the lines where a match is found.
	    #
	    for fileName, fileFlags in files:
		fileName = transform(fileName)
		fileRow = QStandardItem(fileName)
		root.appendRow(fileRow)
		line = 0
		#
		# Question: what encoding is this file? TODO A better approach
		# to this question.
		#
		for text in codecs.open(fileName, encoding="latin-1"):
		    column = text.find(token)
		    if column > -1:
			resultRow = list()
			resultRow.append(QStandardItem(text[:-1]))
			resultRow.append(QStandardItem(str(line + 1)))
			resultRow.append(QStandardItem(str(column + 1)))
{'fraction_non_alphanumeric': 0.1276209677419355, 'fraction_numerical': 0.013306451612903225, 'mean_word_length': 4.087521367521368, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2218541', 'n_tokens_mistral': 5143, 'n_tokens_neox': 4878, 'n_words': 1331}