<commit_before><commit_msg>Added PcaEncoder<commit_after>from sklearn.decomposition import PCA
import numpy

class PcaEncoder(object):

    def __init__(self, dimension):
        self.dimension = dimension
        
    def train(self, data):
        self.model = PCA(n_components=self.dimension,whiten=True)
        self.model.fit(data)
        #print(self.model.explained_variance_ratio_)
        
    def encode(self, vector):
        return self.model.transform(vector)
        
if __name__ == '__main__':
    encoder = PcaEncoder(dimension=2)
    encoder.train(numpy.array([[1,1,1,0,0,0],[1,0,1,0,0,0],[1,1,1,0,0,0],[0,0,1,1,1,0], [0,0,1,1,0,0],[0,0,1,1,1,0]]))
    encoded=encoder.encode(numpy.array([1,1,1,1,1,1]))
    print(encoded)
{'fraction_non_alphanumeric': 0.16914749661705006, 'fraction_numerical': 0.058186738836265225, 'mean_word_length': 3.933333333333333, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6168373', 'n_tokens_mistral': 292, 'n_tokens_neox': 278, 'n_words': 35}