<commit_before>import nltk
from simhash import Simhash
import numpy as np

from orangecontrib.text.vectorization.base import BaseVectorizer


class SimhashVectorizer(BaseVectorizer):
    name = "Simhash"
    max_f = 1024

    def __init__(self, shingle_len=10, f=64, hashfunc=None):
        """
        Args:
            shingle_len(int): Length of a shingle.
            f(int): Length of a document fingerprints
            hashfunc(callable): A function that accepts a string and returns
                a unsigned integer
        """
        self.f = f
        self._bin_format = '{:0%db}' % self.f
        self.hashfunc = hashfunc
        self.ngram_len = shingle_len

    @staticmethod
    def get_shingles(tokens, n):
        return map(lambda x: ''.join(x), nltk.ngrams(tokens, n))

    def compute_hash(self, tokens):
        return Simhash(self.get_shingles(tokens, self.ngram_len), f=self.f, hashfunc=self.hashfunc).value

    def int2binarray(self, num):
        return [int(x) for x in self._bin_format.format(num)]

    def _transform(self, corpus):
        """ Computes simhash values from the given corpus
        and creates a new one with a simhash attribute.

        Args:
            corpus (Corpus): a corpus with tokens.

        Returns:
            Corpus with `simhash` variable
        """

        X = np.array([self.int2binarray(self.compute_hash(doc)) for doc in corpus.tokens])
        corpus = corpus.copy()
        corpus.extend_attributes(X, ('simhash_{}'.format(i) for i in range(self.f)),
                                 var_attrs={'hidden': True})
        return corpus

    def report(self):
        return (('Hash length', self.f),
                ('Shingle length', self.ngram_len))<commit_msg>Simhash: change X type to float.<commit_after>import nltk
from simhash import Simhash
import numpy as np

from orangecontrib.text.vectorization.base import BaseVectorizer


class SimhashVectorizer(BaseVectorizer):
    name = "Simhash"
    max_f = 1024

    def __init__(self, shingle_len=10, f=64, hashfunc=None):
        """
        Args:
            shingle_len(int): Length of a shingle.
            f(int): Length of a document fingerprints
            hashfunc(callable): A function that accepts a string and returns
                a unsigned integer
        """
        self.f = f
        self._bin_format = '{:0%db}' % self.f
        self.hashfunc = hashfunc
        self.ngram_len = shingle_len

    @staticmethod
    def get_shingles(tokens, n):
        return map(lambda x: ''.join(x), nltk.ngrams(tokens, n))

    def compute_hash(self, tokens):
        return Simhash(self.get_shingles(tokens, self.ngram_len), f=self.f, hashfunc=self.hashfunc).value

    def int2binarray(self, num):
        return [int(x) for x in self._bin_format.format(num)]

    def _transform(self, corpus):
        """ Computes simhash values from the given corpus
        and creates a new one with a simhash attribute.

        Args:
            corpus (Corpus): a corpus with tokens.

        Returns:
            Corpus with `simhash` variable
        """

        X = np.array([self.int2binarray(self.compute_hash(doc)) for doc in corpus.tokens], dtype=np.float)
        corpus = corpus.copy()
        corpus.extend_attributes(X, ('simhash_{}'.format(i) for i in range(self.f)),
                                 var_attrs={'hidden': True})
        return corpus

    def report(self):
        return (('Hash length', self.f),
                ('Shingle length', self.ngram_len))
{'fraction_non_alphanumeric': 0.10413694721825963, 'fraction_numerical': 0.006276747503566334, 'mean_word_length': 2.355023923444976, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2224388', 'n_tokens_mistral': 1078, 'n_tokens_neox': 1029, 'n_words': 291}