<commit_before>__all__ = ['CounterHashingVectorizer', 'ListHashingVectorizer', 'ListCountVectorizer', 'SpaceTokenizerTransformer']

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import HashingVectorizer

from .base import PureTransformer


class ListNGramAnalyzer(object):
    def __init__(self, ngram_range=(1, 1), ngram_delimiter=' ', **kwargs):
        if isinstance(ngram_range, tuple) and len(ngram_range) == 2: ngrams = list(range(ngram_range[0], ngram_range[1] + 1))
        elif isinstance(ngram_range, str): ngrams = [ngram_range]
        else: ngrams = ngram_range
        ngrams.sort()

        self.ngrams = ngrams
        self.ngram_delimiter = ngram_delimiter
    #end def

    def __call__(self, L):
        L = list(L)
        length = len(L)
        ngram_delimiter = self.ngram_delimiter

        for i in range(length):
            for n in self.ngrams:
                if i + n > length: break

                yield ngram_delimiter.join(L[i:i + n])
            #end for
        #end for
    #end def
#end def


def _counter_analyzer(C):
    for k, v in C.items():
        for _ in range(v):
            yield k
#end def


class ListCountVectorizer(CountVectorizer):
    def __init__(self, **kwargs):
        kwargs.setdefault('analyzer', ListNGramAnalyzer(**kwargs))

        super(ListCountVectorizer, self).__init__(**kwargs)
    #end def
#end class


class ListHashingVectorizer(HashingVectorizer):
    def __init__(self, **kwargs):
        kwargs.setdefault('analyzer', ListNGramAnalyzer(**kwargs))
        super(ListHashingVectorizer, self).__init__(**kwargs)
    #end def
#end class


class CounterHashingVectorizer(HashingVectorizer):
    def __init__(self, **kwargs):
        kwargs.setdefault('analyzer', _counter_analyzer)
        super(CounterHashingVectorizer, self).__init__(**kwargs)
    #end def
#end class


class SpaceTokenizerTransformer(PureTransformer):
    def transform_one(self, text):
        return text.split()
#end class
<commit_msg>Add `ListTfidfVectorizer` and debug messages with vocabulary sizes<commit_after>__all__ = ['CounterHashingVectorizer', 'ListHashingVectorizer', 'ListCountVectorizer', 'ListNGramAnalyzer', 'ListTfidfVectorizer', 'SpaceTokenizerTransformer']

import logging

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import HashingVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer

from .base import PureTransformer

logger = logging.getLogger(__name__)


class ListNGramAnalyzer(PureTransformer):
    def __init__(self, ngram_range=(1, 1), ngram_delimiter=' ', **kwargs):
        nparray = kwargs.pop('nparray', False)
        super(ListNGramAnalyzer, self).__init__(nparray=nparray, **kwargs)

        if isinstance(ngram_range, tuple) and len(ngram_range) == 2: ngrams = list(range(ngram_range[0], ngram_range[1] + 1))
        elif isinstance(ngram_range, str): ngrams = [ngram_range]
        else: ngrams = ngram_range
        ngrams.sort()

        self.ngrams = ngrams
        self.ngram_delimiter = ngram_delimiter
    #end def

    def __call__(self, L):
        L = list(L)
        length = len(L)
        ngram_delimiter = self.ngram_delimiter

        for i in range(length):
            for n in self.ngrams:
                if i + n > length: break

                yield ngram_delimiter.join(L[i:i + n])
            #end for
        #end for
    #end def

    def transform_one(self, tokens):
        yield from self.__call__(tokens)
#end def


def _counter_analyzer(C):
    for k, v in C.items():
        for _ in range(v):
            yield k
#end def


class ListCountVectorizer(CountVectorizer):
    def __init__(self, **kwargs):
        kwargs.setdefault('analyzer', ListNGramAnalyzer(**kwargs))
        super(ListCountVectorizer, self).__init__(**kwargs)
    #end def

    def fit(self, *args, **kwargs):
        super(ListCountVectorizer, self).fit(*args, **kwargs)
        logger.debug('There are {} vocabulary items in <{}>.'.format(len(self.vocabulary_), self))
    #end def

    def fit_transform(self, *args, **kwargs):
        super(ListCountVectorizer, self).fit_transform(*args, **kwargs)
        logger.debug('There {} vocabulary items in <{}>.'.format(len(self.vocabulary_), self))
    #end def
#end class


class ListTfidfVectorizer(TfidfVectorizer):
    def __init__(self, **kwargs):
        kwargs.setdefault('analyzer', ListNGramAnalyzer(**kwargs))
        super(ListTfidfVectorizer, self).__init__(**kwargs)
    #end def

    def fit(self, *args, **kwargs):
        super(ListTfidfVectorizer, self).fit(*args, **kwargs)
        logger.debug('There are {} vocabulary items in <{}>.'.format(len(self.vocabulary_), self))
    #end def

    def fit_transform(self, *args, **kwargs):
        super(ListTfidfVectorizer, self).fit_transform(*args, **kwargs)
        logger.debug('There {} vocabulary items in <{}>.'.format(len(self.vocabulary_), self))
    #end def
#end class


class ListHashingVectorizer(HashingVectorizer):
    def __init__(self, **kwargs):
        kwargs.setdefault('analyzer', ListNGramAnalyzer(**kwargs))
        super(ListHashingVectorizer, self).__init__(**kwargs)
    #end def
#end class


class CounterHashingVectorizer(HashingVectorizer):
    def __init__(self, **kwargs):
        kwargs.setdefault('analyzer', _counter_analyzer)
        super(CounterHashingVectorizer, self).__init__(**kwargs)
    #end def
#end class


class SpaceTokenizerTransformer(PureTransformer):
    def transform_one(self, text):
        return text.split()
#end class

{'fraction_non_alphanumeric': 0.11534334763948498, 'fraction_numerical': 0.002145922746781116, 'mean_word_length': 3.5471544715447156, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2196558', 'n_tokens_mistral': 1793, 'n_tokens_neox': 1727, 'n_words': 394}