<commit_before>
import cPickle
import numpy as np
from sklearn.externals import joblib
from nolearn.dbn import DBN

def load(name):
    with open(name, 'rb') as f:
        return cPickle.load(f)

dataset1 = load('/Home/gp/data/cifar10/cifar-10-batches-py/data_batch_1')
dataset2 = load('/Home/gp/data/cifar10/cifar-10-batches-py/data_batch_2')
dataset3 = load('/Home/gp/data/cifar10/cifar-10-batches-py/data_batch_3')
dataset4 = load('/Home/gp/data/cifar10/cifar-10-batches-py/data_batch_4')
dataset5 = load('/Home/gp/data/cifar10/cifar-10-batches-py/data_batch_5')
test_batch = load('/Home/gp/data/cifar10/cifar-10-batches-py/test_batch')

data_train = np.vstack([dataset1['data'], dataset2['data'], dataset3['data'],dataset4['data'],dataset5['data']])
labels_train = np.hstack([dataset1['labels'], dataset2['labels'],dataset3['labels'],dataset4['labels'],dataset5['labels']])

data_train = data_train.astype('float') / 255.
labels_train = labels_train
data_test = test_batch['data'].astype('float') / 255.
labels_test = np.array(test_batch['labels'])
n_feat = data_train.shape[1]
n_targets = labels_train.max() + 1

net = DBN(
    [n_feat, n_feat / 3, n_targets],
    epochs=1000,
    learn_rates=0.01,
    learn_rate_decays=0.99,
    learn_rate_minimums=0.005,
    verbose=1,
    )
net.fit(data_train, labels_train)

from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix

expected = labels_test
predicted = net.predict(data_test)

print "Classification report for classifier %s:\n%s\n" % (
    net, classification_report(expected, predicted))
print "Confusion matrix:\n%s" % confusion_matrix(expected, predicted)
print "prediction over expected" % predicted/expected

joblib.dump(net, 'nl_dbn.pkl', compress=9)
#nl_clone = joblib.load('nl_dbn.pkl')
<commit_msg>Restructure data paths for my use<commit_after>
import cPickle
import numpy as np
from sklearn.externals import joblib
from nolearn.dbn import DBN

def load(name):
    with open(name, 'rb') as f:
        return cPickle.load(f)

dataset1 = load('/Home/gp/data/cifar-10-batches-py/data_batch_1')
dataset2 = load('/Home/gp/data/cifar-10-batches-py/data_batch_2')
dataset3 = load('/Home/gp/data/cifar-10-batches-py/data_batch_3')
dataset4 = load('/Home/gp/data/cifar-10-batches-py/data_batch_4')
dataset5 = load('/Home/gp/data/cifar-10-batches-py/data_batch_5')
test_batch = load('/Home/gp/data/cifar-10-batches-py/test_batch')

data_train = np.vstack([dataset1['data'], dataset2['data'], dataset3['data'],dataset4['data'],dataset5['data']])
labels_train = np.hstack([dataset1['labels'], dataset2['labels'],dataset3['labels'],dataset4['labels'],dataset5['labels']])

data_train = data_train.astype('float') / 255.
labels_train = labels_train
data_test = test_batch['data'].astype('float') / 255.
labels_test = np.array(test_batch['labels'])
n_feat = data_train.shape[1]
n_targets = labels_train.max() + 1

net = DBN(
    [n_feat, n_feat / 3, n_targets],
    epochs=1000,
    learn_rates=0.01,
    learn_rate_decays=0.99,
    learn_rate_minimums=0.005,
    verbose=1,
    )
net.fit(data_train, labels_train)

from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix

expected = labels_test
predicted = net.predict(data_test)

print "Classification report for classifier %s:\n%s\n" % (
    net, classification_report(expected, predicted))
print "Confusion matrix:\n%s" % confusion_matrix(expected, predicted)
print "prediction over expected" % predicted/expected

joblib.dump(net, 'nl_dbn.pkl', compress=9)
#nl_clone = joblib.load('nl_dbn.pkl')

{'fraction_non_alphanumeric': 0.152428810720268, 'fraction_numerical': 0.035175879396984924, 'mean_word_length': 5.61070110701107, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5947241', 'n_tokens_mistral': 1378, 'n_tokens_neox': 1294, 'n_words': 219}