<commit_before>from collections import Counter

import numpy as np
import pyqtgraph as pg
from PyQt4.QtCore import *
from PyQt4.QtGui import *

from Orange.clustering.hierarchical import dist_matrix_linkage
from Orange.data import Table, DiscreteVariable, Domain
from Orange.misc import DistMatrix
from Orange.widgets import gui, widget, settings
from Orange.widgets.utils.itemmodels import PyTableModel
from orangecontrib.text import Corpus


class IO:
    distances = 'Distances'
    corpus = 'Corpus'
    corpus_without_duplicates = 'Corpus Without Duplicates'
    duplicates = 'Duplicates Cluster'


class OWDuplicates(widget.OWWidget):
    name = 'Duplicate Detection'
    description = 'Detect & remove duplicates from a corpus.'
    icon = 'icons/Duplicates.svg'
    priority = 45

    inputs = [(IO.distances, DistMatrix, 'set_distances')]
    outputs = [
        (IO.corpus_without_duplicates, Corpus),
        (IO.duplicates, Corpus),
        (IO.corpus, Corpus),
    ]

    resizing_enabled = True

    LINKAGE = ['Single', 'Average', 'Complete', 'Weighted', 'Ward']
    linkage_method = settings.Setting(1)

    # Cluster variable domain role
    AttributeRole, ClassRole, MetaRole = 0, 1, 2
    CLUSTER_ROLES = ["Attributes", "Class", "Metas"]
    cluster_role = settings.Setting(2)

    def __init__(self):
        super().__init__()
        self.corpus = None              # corpus taken from distances
        self.linkage = None             # hierarchical clustering linkage as returned by Orange
        self.distances = None           # DistMatrix on input
        self.clustering_mask = None     # 1D array of clusters for self.corpus
        self.threshold = 0              # hierarchical clustering distance threshold
        self.threshold_spin = None

        # Info
        self.n_documents = ''
        self.n_unique = ''
        self.n_duplicates = ''
        info_box = gui.widgetBox(self.controlArea, box='Info')
        gui.label(info_box, self, 'Documents: %(n_documents)s')
        gui.label(info_box, self, '  ◦ unique: %(n_unique)s')
        gui.label(info_box, self, '  ◦ duplicates: %(n_duplicates)s')

        # Threshold Histogram & Cluster View
        self.histogram = Histogram(self)
        self.table_view = gui.TableView(selectionMode=QListView.SingleSelection)
        self.table_model = PyTableModel()
        self.table_model.setHorizontalHeaderLabels(['Cluster', 'Size'])
        self.table_view.setModel(self.table_model)
        self.table_view.selectionModel().selectionChanged.connect(self.send_duplicates)

        # Add to main area
        height = 300
        main_area = gui.hBox(self.mainArea)
        self.histogram.setMinimumWidth(500)
        self.histogram.setMinimumHeight(height)
        self.table_view.setFixedWidth(140)
        main_area.layout().addWidget(self.histogram)
        main_area.layout().addWidget(self.table_view)

        # Controls
        gui.comboBox(self.controlArea, self, 'linkage_method', items=self.LINKAGE, box='Linkage',
                     callback=self.recalculate_linkage, orientation=Qt.Horizontal)
        self.threshold_spin = gui.doubleSpin(self.controlArea, self, 'threshold',
                                             0, float('inf'), 0.01, decimals=2,
                                             label='Distance threshold', box='Distances',
                                             callback=self.threshold_changed,
                                             keyboardTracking=False, controlWidth=60)
        self.histogram.region.sigRegionChangeFinished.connect(self.threshold_from_histogram_region)
        self.threshold_spin.setEnabled(False)
        gui.rubber(self.controlArea)

        # Output
        gui.comboBox(self.controlArea, self, "cluster_role", box='Output',
                     label='Append Cluster IDs to:', callback=self.send_corpus,
                     items=self.CLUSTER_ROLES)

    def reset(self):
        self.corpus = None
        self.linkage = None
        self.distances = None
        self.clustering_mask = None
        self.n_documents = ''
        self.n_unique = ''
        self.n_duplicates = ''
        self.threshold = 0
        self.threshold_spin.setEnabled(False)
        self.table_model.clear()
        self.histogram.setValues([])

    def set_distances(self, distances):
        self.distances = distances
        if distances is None:
            self.reset()
            return

        self.corpus = self.distances.row_items
        self.n_documents = len(self.corpus)
        self.threshold_spin.setEnabled(True)
        self.recalculate_linkage()

    def threshold_from_histogram_region(self):
        _, self.threshold = self.histogram.getRegion()
        self.threshold_changed()

    def threshold_changed(self):
        self.threshold = np.clip(self.threshold, *self.histogram.boundary())
        self.histogram.setRegion(0, self.threshold)
        self.detect_duplicates()

    def recalculate_linkage(self):
        if self.distances is not None:
            self.linkage = dist_matrix_linkage(self.distances,
                                               self.LINKAGE[self.linkage_method].lower())

            # Magnitude of the spinbox's step is data-dependent
            vals = sorted(self.linkage[:, 2])
            low, up = vals[0], vals[-1]
            step = (up - low) / 20
<commit_msg>OWDuplicates: Raise error on distances between columns<commit_after>from collections import Counter

import numpy as np
import pyqtgraph as pg
from PyQt4.QtCore import *
from PyQt4.QtGui import *

from Orange.clustering.hierarchical import dist_matrix_linkage
from Orange.data import Table, DiscreteVariable, Domain
from Orange.misc import DistMatrix
from Orange.widgets import gui, widget, settings
from Orange.widgets.utils.itemmodels import PyTableModel
from Orange.widgets.widget import Msg, OWWidget
from orangecontrib.text import Corpus


class IO:
    distances = 'Distances'
    corpus = 'Corpus'
    corpus_without_duplicates = 'Corpus Without Duplicates'
    duplicates = 'Duplicates Cluster'


class OWDuplicates(widget.OWWidget):
    name = 'Duplicate Detection'
    description = 'Detect & remove duplicates from a corpus.'
    icon = 'icons/Duplicates.svg'
    priority = 45

    inputs = [(IO.distances, DistMatrix, 'set_distances')]
    outputs = [
        (IO.corpus_without_duplicates, Corpus),
        (IO.duplicates, Corpus),
        (IO.corpus, Corpus),
    ]

    resizing_enabled = True

    class Error(OWWidget.Error):
        dist_matrix_invalid_shape = Msg('Duplicate detection only supports '
                                        'distances calculated between rows.')

    LINKAGE = ['Single', 'Average', 'Complete', 'Weighted', 'Ward']
    linkage_method = settings.Setting(1)

    # Cluster variable domain role
    AttributeRole, ClassRole, MetaRole = 0, 1, 2
    CLUSTER_ROLES = ["Attributes", "Class", "Metas"]
    cluster_role = settings.Setting(2)

    def __init__(self):
        super().__init__()
        self.corpus = None              # corpus taken from distances
        self.linkage = None             # hierarchical clustering linkage as returned by Orange
        self.distances = None           # DistMatrix on input
        self.clustering_mask = None     # 1D array of clusters for self.corpus
        self.threshold = 0              # hierarchical clustering distance threshold
        self.threshold_spin = None

        # Info
        self.n_documents = ''
        self.n_unique = ''
        self.n_duplicates = ''
        info_box = gui.widgetBox(self.controlArea, box='Info')
        gui.label(info_box, self, 'Documents: %(n_documents)s')
        gui.label(info_box, self, '  ◦ unique: %(n_unique)s')
        gui.label(info_box, self, '  ◦ duplicates: %(n_duplicates)s')

        # Threshold Histogram & Cluster View
        self.histogram = Histogram(self)
        self.table_view = gui.TableView(selectionMode=QListView.SingleSelection)
        self.table_model = PyTableModel()
        self.table_model.setHorizontalHeaderLabels(['Cluster', 'Size'])
        self.table_view.setModel(self.table_model)
        self.table_view.selectionModel().selectionChanged.connect(self.send_duplicates)

        # Add to main area
        height = 300
        main_area = gui.hBox(self.mainArea)
        self.histogram.setMinimumWidth(500)
        self.histogram.setMinimumHeight(height)
        self.table_view.setFixedWidth(140)
        main_area.layout().addWidget(self.histogram)
        main_area.layout().addWidget(self.table_view)

        # Controls
        gui.comboBox(self.controlArea, self, 'linkage_method', items=self.LINKAGE, box='Linkage',
                     callback=self.recalculate_linkage, orientation=Qt.Horizontal)
        self.threshold_spin = gui.doubleSpin(self.controlArea, self, 'threshold',
                                             0, float('inf'), 0.01, decimals=2,
                                             label='Distance threshold', box='Distances',
                                             callback=self.threshold_changed,
                                             keyboardTracking=False, controlWidth=60)
        self.histogram.region.sigRegionChangeFinished.connect(self.threshold_from_histogram_region)
        self.threshold_spin.setEnabled(False)
        gui.rubber(self.controlArea)

        # Output
        gui.comboBox(self.controlArea, self, "cluster_role", box='Output',
                     label='Append Cluster IDs to:', callback=self.send_corpus,
                     items=self.CLUSTER_ROLES)

    def reset(self):
        self.corpus = None
        self.linkage = None
        self.distances = None
        self.clustering_mask = None
        self.n_documents = ''
        self.n_unique = ''
        self.n_duplicates = ''
        self.threshold = 0
        self.threshold_spin.setEnabled(False)
        self.table_model.clear()
        self.histogram.setValues([])

    def set_distances(self, distances):
        self.Error.clear()
        self.distances = distances
        if distances is None:
            self.reset()
            return

        self.corpus = self.distances.row_items
        self.n_documents = len(self.corpus)
        if distances.shape != (self.n_documents, self.n_documents):
            self.Error.dist_matrix_invalid_shape()
            self.reset()
            return
        self.threshold_spin.setEnabled(True)
        self.recalculate_linkage()

    def threshold_from_histogram_region(self):
        _, self.threshold = self.histogram.getRegion()
        self.threshold_changed()

    def threshold_changed(self):
        self.threshold = np.clip(self.threshold, *self.histogram.boundary())
        self.histogram.setRegion(0, self.threshold)
        self.detect_duplicates()

    def recalculate_linkage(self):
        if self.distances is not None:
            self.linkage = dist_matrix_linkage(self.distances,
                                               self.LINKAGE[self.linkage_method].lower())

            # Magnitude of the spinbox's step is data-dependent
            vals = sorted(self.linkage[:, 2])
            low, up = vals[0], vals[-1]
            step = (up - low) / 20

{'fraction_non_alphanumeric': 0.09603712959657265, 'fraction_numerical': 0.006069260978222063, 'mean_word_length': 2.3739837398373984, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 21, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6427880', 'n_tokens_mistral': 3303, 'n_tokens_neox': 3105, 'n_words': 652}