<jupyter_start><jupyter_text>Sklearn on the full dataset<jupyter_code>import h5py
import numpy
from sklearn import linear_model
from sklearn.preprocessing import OneHotEncoder
with h5py.File('data/chorales.hdf5', "r", libver='latest') as f:
    Xtrain = f['Xtrain'].value
    ytrain = f['ytrain'].value
    Xdev = f['Xdev'].value
    ydev = f['ydev'].value
    Xtest = f['Xtest'].value
    ytest = f['ytest'].value

Xtrain, Xdev, Xtest = Xtrain[:, range(10)], Xdev[:, range(10)], Xtest[:, range(10)]
Xtrain = numpy.vstack((Xtrain, Xdev))
Xall = numpy.vstack((Xtrain, Xtest))

with h5py.File('data/chorales_sm.hdf5', "r", libver='latest') as f:
    ytrain = f['ytrainfeat'].value
    ytest = f['ytestfeat'].value
    yall = f['yallfeat'].value

Xtrain.shape, Xall.shape
Xtestex, ytestex = [], []
assert len(Xtest) == len(ytest)
for idx, x in enumerate(ytest):
    if x in ytrain:
        Xtestex.append(Xtest[idx])
        ytestex.append(ytest[idx])
Xtestex = numpy.matrix(Xtestex)
ytestex = numpy.array(ytestex)
logistic = linear_model.LogisticRegression(multi_class='multinomial', solver='lbfgs', C=1000)
encoder = OneHotEncoder()
encoder.fit(Xall)
Xtrainsparse = encoder.transform(Xtrain)
Xtestsparse = encoder.transform(Xtest)
Xtestexsparse = encoder.transform(Xtestex)
logistic.fit(Xtrainsparse, ytrain)
logistic.score(Xtrainsparse, ytrain)
logistic.score(Xtestsparse, ytest)
logistic.score(Xtestexsparse, ytestex)
from sklearn.ensemble import RandomForestClassifier
RF = RandomForestClassifier(10, "entropy")
RF.fit(Xtrainsparse,ytrain)
print RF.score(Xtrainsparse, ytrain)
print RF.score(Xtestsparse, ytest)
print RF.score(Xtestexsparse, ytestex)
%matplotlib inline
import numpy as np
import scipy as sp
import matplotlib as mpl
import matplotlib.cm as cm
import matplotlib.pyplot as plt

def plotNLLandConfidence(filename):
    with open(filename, 'r') as f:
        lines = f.readlines()
    formatted = map(lambda x: x[:-2] if '\t\n' in x else x[:-1], lines)
    formatted = [map(float, x.split('\t')) for x in formatted]
    nll = [x[0] for x in formatted]
    per = [x[1] for x in formatted]
    fig, ax = plt.subplots(1, 2, figsize=(10,5))
    fig.tight_layout()
    ax[0].hist(per)
    ax[0].set_xlabel("Probability assigned to the correct decision")
    ax[1].hist(nll)
    ax[1].set_xlabel("NLL of the parameters given each outcome in the test set")
    plt.show()
plotNLLandConfidence("smerror.txt")
plotNLLandConfidence("smerror_train.txt")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1395255327704061, 'fraction_numerical': 0.012062726176115802, 'mean_word_length': 5.038834951456311, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '208057', 'license': 'Apache-2.0', 'n_tokens_mistral': 939, 'n_tokens_neox': 923, 'n_words': 210, 'path': 'bach_code/legacy/Initial Experiments.ipynb', 'repo_name': 'glasperfan/thesis', 'size': '2487'}