<commit_before>#!/usr/bin/env python

from itertools import product
from collections import defaultdict
from redis import Redis
import sys
import json

from Bio import SeqIO

from nearpy import Engine
from nearpy import hashes
from nearpy.storage import RedisStorage
from nearpy.filters import DistanceThresholdFilter

import numpy as np

# The distance ratio of an ANN y is it's distance to the minimal hypersphere
# around the query vector x, that contains all exact nearest neighbours n,
# clamped to zero and normalized with this hypersphere's radius

class FeatureGenerator:

    def __init__(self, **kwargs):
        ''' '''

        self.alphabet = [ 'A', 'R', 'N', 'D', 'C', 'E', 'Q', 'G', 'H', 'I', 'L', 'K',
                     'M', 'F', 'P', 'S', 'T', 'W', 'Y', 'V', ]

        assert len(self.alphabet) == 20
        self.k = kwargs['K']

        self.feature_space = list(''.join(i) for i in product(self.alphabet,
            repeat=self.k))

        self.n_features = len(self.feature_space)


    def shingles(self, s, k):
        ''' return shingles of a given string given a k-mer size k '''
        return [ s[i : i + k ] for i in range(0, len(s) - k + 1) ]


    def vectorize(self, s):
        ''' convert shingles to features vector '''

        d = defaultdict(lambda: 0)

        for i in s:
            d[i] += 1

        return np.array([ d[i] for i in self.feature_space ])


    def transform(self, str):
        return self.vectorize(self.shingles(str, self.k))




class SimilarStringStore:

    def __init__(self):

        self.config = {}
        self.config['K'] = 2 # k-mer size for shingles
        self.config['P'] = 100 # number of Random Binary Projections
        self.config['MAX_DIST'] = 20 # maximum distance threshold

        self.transformer = FeatureGenerator(K = self.config['K'])

        self.hasher = hashes.RandomBinaryProjections('rbp', self.config['P'], rand_seed=42)

        self.backend = RedisStorage(Redis(host='localhost', port=6379, db=0))

        self.filters = [ DistanceThresholdFilter(self.config['MAX_DIST']) ]
<commit_msg>comments<commit_after>#!/usr/bin/env python

from itertools import product
from collections import defaultdict
from redis import Redis
import sys
import json

from Bio import SeqIO

from nearpy import Engine
from nearpy import hashes
from nearpy.storage import RedisStorage
from nearpy.filters import DistanceThresholdFilter

import numpy as np

# The distance ratio of an ANN y is it's distance to the minimal hypersphere
# around the query vector x, that contains all exact nearest neighbours n,
# clamped to zero and normalized with this hypersphere's radius

class FeatureGenerator:

    def __init__(self, **kwargs):
        ''' '''

        self.alphabet = [ 'A', 'R', 'N', 'D', 'C', 'E', 'Q', 'G', 'H', 'I', 'L', 'K',
                     'M', 'F', 'P', 'S', 'T', 'W', 'Y', 'V', ]

        assert len(self.alphabet) == 20
        self.k = kwargs['K']

        self.feature_space = list(''.join(i) for i in product(self.alphabet,
            repeat=self.k))

        self.n_features = len(self.feature_space)


    def shingles(self, s, k):
        ''' return shingles of a given string given a k-mer size k '''
        return [ s[i : i + k ] for i in range(0, len(s) - k + 1) ]


    def vectorize(self, s):
        ''' convert shingles to features vector '''

        d = defaultdict(lambda: 0)

        for i in s:
            d[i] += 1

        return np.array([ d[i] for i in self.feature_space ])


    def transform(self, str):
        return self.vectorize(self.shingles(str, self.k))




class SimilarStringStore:

    def __init__(self):

        self.config = {}
        self.config['K'] = 2 # k-mer size for shingles
        self.config['P'] = 100 # number of Random Binary Projections
        self.config['MAX_DIST'] = 20 # maximum distance threshold (euclidean)

        self.transformer = FeatureGenerator(K = self.config['K'])

        self.hasher = hashes.RandomBinaryProjections('rbp', self.config['P'], rand_seed=42)

        self.backend = RedisStorage(Redis(host='localhost', port=6379, db=0))

        self.filters = [ DistanceThresholdFilter(self.config['MAX_DIST']) ]

{'fraction_non_alphanumeric': 0.12843160658868466, 'fraction_numerical': 0.00907137741704464, 'mean_word_length': 2.822992700729927, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2849716', 'n_tokens_mistral': 1365, 'n_tokens_neox': 1270, 'n_words': 419}