<commit_before>from heapq import heappush, heapreplace
from collections import Counter

class MaxHashNgramSketch(object):
    """
    MaxHash Sketch
    """

    _anynew = None
    
    def __init__(self, nsize: int,
                 maxsize: int,
                 hashfun,
                 heap : list = None,
                 nvisited: int = 0):
        self._nsize = nsize
        self._maxsize = maxsize
        self._hashfun = hashfun
        if heap is None:
            self._heap = list()
        else:
            self._heap = heap
        self._heapset = set(self._heap)
        self._nvisited = nvisited

    @property
    def maxsize(self):
        return self._maxsize

    @property
    def nsize(self):
        return self._nsize

    @property
    def nvisited(self):
        return self._nvisited

    def _add(self, elt):
        self._heapset.add(elt)
        heappush(self._heap, elt)

    def _replace(self, elt):
        heapset = self._heapset
        heapset.add(elt)
        out = heapreplace(self._heap, elt)
        heapset.remove(out)
        return out

    def _make_elt(self, h, ngram):
        return (h, bytes(ngram))

    def __len__(self):
        return len(self._heap)
    
    def __contains__(self, elt):
        return elt in self._heapset

    def add_hashvalues(self, values):
        make_elt = self._make_elt
        heaptop = heap[0][0]
        for h in values:
            if h  >= heaptop:
                elt = make_elt(h, None)
                if elt not in heapset:
                    out = self._replace((h, ngram))
                    heaptop = heap[0][0]
        
    def add(self, s):
        hashfun = self._hashfun
        heap = self._heap
        heapset = self._heapset
        maxsize = self._maxsize
        nsize = self._nsize
        anynew = self._anynew
        i = None
        lheap = len(heap)
        if lheap > 0:
            heaptop = heap[0][0]
        else:
            heaptop = None
        for i in range(0, len(s)-nsize+1):
            ngram = s[i:(i+nsize)]<commit_msg>added docstrings<commit_after>from heapq import heappush, heapreplace
from collections import Counter

class MaxHashNgramSketch(object):
    """
    MaxHash Sketch
    """

    _anynew = None
    
    def __init__(self, nsize: int,
                 maxsize: int,
                 hashfun,
                 heap : list = None,
                 nvisited: int = 0):
        """
        - nsize: size of the ngrams / kmers
        - maxsize: maximum size for the sketch
        - hashfun: function used for hashing
        - heap: heapified list (if unsure about what it is, don't change the default)
        - nvisited: number of kmers visited so far
        """
        self._nsize = nsize
        self._maxsize = maxsize
        self._hashfun = hashfun
        if heap is None:
            self._heap = list()
        else:
            self._heap = heap
        self._heapset = set(self._heap)
        self._nvisited = nvisited

    @property
    def maxsize(self):
        """ Maximum size for the sketch. """
        return self._maxsize

    @property
    def nsize(self):
        """ Size of the ngrams / kmers. """
        return self._nsize

    @property
    def nvisited(self):
        """ Number of ngrams / kmers visited (considered for inclusion) so far. """
        return self._nvisited

    def _add(self, elt):
        self._heapset.add(elt)
        heappush(self._heap, elt)

    def _replace(self, elt):
        heapset = self._heapset
        heapset.add(elt)
        out = heapreplace(self._heap, elt)
        heapset.remove(out)
        return out

    def _make_elt(self, h, ngram):
        return (h, bytes(ngram))

    def __len__(self):
        return len(self._heap)
    
    def __contains__(self, elt):
        return elt in self._heapset

    def add_hashvalues(self, values):
        make_elt = self._make_elt
        heaptop = heap[0][0]
        for h in values:
            if h  >= heaptop:
                elt = make_elt(h, None)
                if elt not in heapset:
                    out = self._replace((h, ngram))
                    heaptop = heap[0][0]
        
    def add(self, s):
        """ Add all ngrams/kmers of length self.nsize found in the sequence "s".
        - s: a bytes-like sequence
        """
        hashfun = self._hashfun
        heap = self._heap
        heapset = self._heapset
        maxsize = self._maxsize
        nsize = self._nsize
        anynew = self._anynew
        i = None
        lheap = len(heap)
        if lheap > 0:
            heaptop = heap[0][0]
        else:
            heaptop = None
        for i in range(0, len(s)-nsize+1):
            ngram = s[i:(i+nsize)]
{'fraction_non_alphanumeric': 0.0936768149882904, 'fraction_numerical': 0.004258037044922291, 'mean_word_length': 1.5742465753424657, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1508834', 'n_tokens_mistral': 1467, 'n_tokens_neox': 1411, 'n_words': 390}