<commit_before># -*- coding: utf-8 -*-

import warnings
import numpy as np

from data.data import Data
from classifier.kmeans import KMeans
from classifier.cnn import CNN
from score import online_score

warnings.filterwarnings("ignore")

CATEGORIES = (
    'artifacts',
    'chaetognaths',
    'copepods',
    'ctenophores',
    'detritus',
    'diatoms',
    'echinoderms',
    'fish',
    'gastropods',
    'gelatinous zooplankton',
    'hydromedusae',
    'invert larvae',
    'pelagic tunicates',
    'protists',
    'shrimp',
    'siphonophores',
    'trichodesmium',
)

NB_CLUSTERS = 3 # len(CATEGORIES)

if __name__ == '__main__':

    d = Data(size=28)
    d2 = d  # change to another size of pictures
    kclf = KMeans(clusters=NB_CLUSTERS-1).train(d.train_X)

    # TODO: Is it a good idea to train the CNN based on data that the Kmeans
    # classifies without having seen it ?
    print 'creating the datasets'
    train_X = [[] for i in xrange(NB_CLUSTERS)]
    train_Y = [[] for i in xrange(NB_CLUSTERS)]
    for X, y in zip(d2.train_X, d2.train_Y):
        idx = kclf.predict(X)
        train_X[idx].append(X)
        train_Y[idx].append(y)

    # creating a CNN for each
    print 'creating and training the cnn'
    cnns = [CNN(batch_size=1, train_X=train_X[i], train_Y=train_Y[
                i], epochs=2).train()for i in xrange(NB_CLUSTERS)]

    # Benchmark the algo:
    print 'benchmarking the algorithm'
    predictions = []
    for X in d2.valid_X:
        idx = kclf.predict(X)
        predictions.append(cnns[idx].predict(X))
    online_score(predictions, d2.valid_Y)<commit_msg>Fixed for good<commit_after># -*- coding: utf-8 -*-

import warnings
import numpy as np

from data.data import Data
from classifier.kmeans import KMeans
from classifier.cnn import CNN
from score import online_score

warnings.filterwarnings("ignore")

CATEGORIES = (
    'artifacts',
    'chaetognaths',
    'copepods',
    'ctenophores',
    'detritus',
    'diatoms',
    'echinoderms',
    'fish',
    'gastropods',
    'gelatinous zooplankton',
    'hydromedusae',
    'invert larvae',
    'pelagic tunicates',
    'protists',
    'shrimp',
    'siphonophores',
    'trichodesmium',
)

NB_CLUSTERS = 3  # len(CATEGORIES)

if __name__ == '__main__':

    d = Data(size=28)
    d2 = d  # change to another size of pictures
    kclf = KMeans(clusters=NB_CLUSTERS - 1).train(d.train_X)

    # TODO: Is it a good idea to train the CNN based on data that the Kmeans
    # classifies without having seen it ?
    print 'creating the datasets'
    train_X = [[] for i in xrange(NB_CLUSTERS)]
    train_Y = [[] for i in xrange(NB_CLUSTERS)]
    for X, y in zip(d2.train_X, d2.train_Y):
        idx = kclf.predict(X)
        train_X[idx].append(X)
        train_Y[idx].append(y)

    # creating a CNN for each
    print 'creating and training the cnn'
    cnns = [CNN(batch_size=1, train_X=train_X[i], train_Y=train_Y[
                i], epochs=2).train()for i in xrange(NB_CLUSTERS)]

    # Benchmark the algo:
    print 'benchmarking the algorithm'
    predictions = []
    for X in d2.valid_X:
        idx = kclf.predict(X)
        predictions.append(cnns[idx].predict([X, ]))
    online_score(predictions, d2.valid_Y)
{'fraction_non_alphanumeric': 0.11762880198634389, 'fraction_numerical': 0.0074487895716946, 'mean_word_length': 3.0136986301369864, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3248570', 'n_tokens_mistral': 1239, 'n_tokens_neox': 1162, 'n_words': 296}