<jupyter_start><jupyter_text>Neural Networks and Deep Learning for Life Sciences and Health Applications - An introductory course about theoretical fundamentals, case studies and implementations in python and tensorflow (C) <NAME> 2018 - <EMAIL> github repository: https://github.com/michelucci/zhaw-dlcourse-spring2019Spring Semester 2019  Error Analysis Diagram Code (EAD)<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
eps = [1,3,9,5]
colors = [()]

fig, ax = plt.subplots()
bar = ax.bar(x = [1,2,3,4], height = eps, width = 1.0, color = (0.9,0.9,0.9), edgecolor = 'Black')
bar[0].set_hatch('/')
bar[0].set_color((0.9,0.9,0.9))
bar[0].set_edgecolor('Black')
bar[1].set_color((0.8,0.8,0.8))
bar[1].set_hatch('/')
bar[1].set_edgecolor('Black')
bar[2].set_hatch('/')
bar[2].set_color((0.7,0.7,0.7))
bar[2].set_edgecolor('Black')
bar[3].set_hatch('/')
bar[3].set_color((0.6,0.6,0.6))
bar[3].set_edgecolor('Black')
plt.xticks([1,2,3,4], ['Avoidable Bias','Variance','Data Mismatch\nProblem','Overfitting\non Dev'], 
           rotation = 'vertical', fontsize = 14)
plt.yticks([0,2,4,6,8,10],['','2','4','6','8','10'], fontsize = 14)
plt.ylim(0,11)
plt.xlim(0.5,4.5)

for i,v in enumerate(eps):
    ax.text(i+0.9, v+0.4, str(v)+'%', color = 'Black', fontsize = 14)

#plt.axhline(y=2, color = 'Black')
#plt.axhline(y=4, color = 'Black')
#plt.axhline(y=6, color = 'Black')
#plt.axhline(y=8, color = 'Black')
plt.axvline(x=0.5, color = 'Black')
plt.axvline(x=1.5, color = 'Black')
plt.axvline(x=2.5, color = 'Black')
plt.axvline(x=3.5, color = 'Black')
plt.axvline(x=4.5, color = 'Black')<jupyter_output><empty_output><jupyter_text>Figure 6-4<jupyter_code>eps = [9]
fig, ax = plt.subplots(figsize = (8,5))
bar = ax.barh([1,2,3,4], eps, height= 1.0, color = (0.9,0.9,0.9), edgecolor = 'Black')
bar[0].set_hatch('/')
bar[0].set_color((0.9,0.9,0.9))
bar[0].set_edgecolor('Black')
#bar[1].set_color((0.75,0.75,0.75))
#bar[1].set_hatch('/')
#bar[1].set_edgecolor('Black')
#bar[2].set_hatch('/')
#bar[2].set_color((0.6,0.6,0.6))
#bar[2].set_edgecolor('Black')
#bar[3].set_hatch('/')
#bar[3].set_color((0.45,0.45,0.45))
#bar[3].set_edgecolor('Black')
plt.yticks([1], ['Bias / $\Delta \epsilon_{Bias}$'], 
            fontsize = 14)
plt.xlim(0,11)
plt.ylim(0.5,1.5)

for i,v in enumerate(eps):
    ax.text(v+0.4, i+0.88, str(v)+'%', color = 'Black', fontsize = 14)

    
plt.xticks([0,2,4,6,8,10],['0','2','4','6','8','10'], fontsize = 14)

plt.axhline(y=0.5, color = 'Black')
#plt.axhline(y=1.5, color = 'Black')
#plt.axhline(y=2.5, color = 'Black')
#plt.axhline(y=3.5, color = 'Black')
#plt.axhline(y=4.5, color = 'Black')

plt.title ('Metric Analysis Diagram (MAD)', fontsize = 14)
plt.show()

fig.savefig('Figure_6-4'+'.png', format='png', dpi=300,bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>Figure 6-5<jupyter_code>eps = [4,6]
fig, ax = plt.subplots(figsize = (8,5))
bar = ax.barh([1,2], eps, height= 1.0, color = (0.9,0.9,0.9), edgecolor = 'Black')
bar[0].set_hatch('/')
bar[0].set_color((0.9,0.9,0.9))
bar[0].set_edgecolor('Black')
bar[1].set_color((0.75,0.75,0.75))
bar[1].set_hatch('/')
bar[1].set_edgecolor('Black')
#bar[2].set_hatch('/')
#bar[2].set_color((0.6,0.6,0.6))
#bar[2].set_edgecolor('Black')
#bar[3].set_hatch('/')
#bar[3].set_color((0.45,0.45,0.45))
#bar[3].set_edgecolor('Black')
plt.yticks([1,2], ['Overfitting of training dataset / $\Delta \epsilon_{overfitting\ train}$', 'Bias / $\Delta \epsilon_{Bias}$'], 
            fontsize = 14)
plt.xlim(0,11)
plt.ylim(0.5,2.5)

for i,v in enumerate(eps):
    ax.text(v+0.4, i+0.88, str(v)+'%', color = 'Black', fontsize = 14)

    
plt.xticks([0,2,4,6,8,10],['0','2','4','6','8','10'], fontsize = 14)

plt.axhline(y=0.5, color = 'Black')
plt.axhline(y=1.5, color = 'Black')
#plt.axhline(y=2.5, color = 'Black')
#plt.axhline(y=3.5, color = 'Black')
#plt.axhline(y=4.5, color = 'Black')

plt.title ('Metric Analysis Diagram (MAD)', fontsize = 14)
plt.show()

fig.savefig('Figure_6-5'+'.png', format='png', dpi=300,bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>Figure 6-6<jupyter_code>eps = [8,4,6]
fig, ax = plt.subplots(figsize = (8,5))
bar = ax.barh([1,2,3], eps, height= 1.0, color = (0.9,0.9,0.9), edgecolor = 'Black')
bar[0].set_hatch('/')
bar[0].set_color((0.9,0.9,0.9))
bar[0].set_edgecolor('Black')
bar[1].set_color((0.75,0.75,0.75))
bar[1].set_hatch('/')
bar[1].set_edgecolor('Black')
bar[2].set_hatch('/')
bar[2].set_color((0.6,0.6,0.6))
bar[2].set_edgecolor('Black')
#bar[3].set_hatch('/')
#bar[3].set_color((0.45,0.45,0.45))
#bar[3].set_edgecolor('Black')
plt.yticks([1,2,3], ['Overfitting of dev dataset / $\Delta \epsilon_{overfitting\ dev}$',
                   'Overfitting of training dataset / $\Delta \epsilon_{overfitting\ train}$', 
                   'Bias / $\Delta \epsilon_{Bias}$'], 
            fontsize = 14)
plt.xlim(0,11)
plt.ylim(0.5,3.5)

for i,v in enumerate(eps):
    ax.text(v+0.4, i+0.88, str(v)+'%', color = 'Black', fontsize = 14)

    
plt.xticks([0,2,4,6,8,10],['0','2','4','6','8','10'], fontsize = 14)

plt.axhline(y=0.5, color = 'Black')
plt.axhline(y=1.5, color = 'Black')
plt.axhline(y=2.5, color = 'Black')
#plt.axhline(y=3.5, color = 'Black')
#plt.axhline(y=4.5, color = 'Black')

plt.title ('Metric Analysis Diagram (MAD)', fontsize = 14)
plt.show()

fig.savefig('Figure_6-6'+'.png', format='png', dpi=300,bbox_inches='tight')
fig, ax = plt.subplots(figsize = (8,5))
bar = ax.barh([1,2,3,4], eps, height= 1.0, color = (0.9,0.9,0.9), edgecolor = 'Black')
bar[0].set_hatch('/')
bar[0].set_color((0.9,0.9,0.9))
bar[0].set_edgecolor('Black')
bar[1].set_color((0.75,0.75,0.75))
bar[1].set_hatch('/')
bar[1].set_edgecolor('Black')
bar[2].set_hatch('/')
bar[2].set_color((0.6,0.6,0.6))
bar[2].set_edgecolor('Black')
bar[3].set_hatch('/')
bar[3].set_color((0.45,0.45,0.45))
bar[3].set_edgecolor('Black')
plt.yticks([1,2,3,4], ['Overfitting of dev dataset / $\Delta \epsilon_{overfitting\ dev}$',
                   'Data mismatch (between train and dev) / $\Delta \epsilon_{train-dev}$', 
                   'Overfitting of training dataset / $\Delta \epsilon_{overfitting\ train}$', 
                   'Bias / $\Delta \epsilon_{Bias}$'], 
            fontsize = 14)
plt.xlim(0,11)
plt.ylim(0.5,4.5)

for i,v in enumerate(eps):
    ax.text(v+0.4, i+0.88, str(v)+'%', color = 'Black', fontsize = 14)

    
plt.xticks([0,2,4,6,8,10],['0','2','4','6','8','10'], fontsize = 14)

plt.axhline(y=0.5, color = 'Black')
plt.axhline(y=1.5, color = 'Black')
plt.axhline(y=2.5, color = 'Black')
plt.axhline(y=3.5, color = 'Black')
plt.axhline(y=4.5, color = 'Black')

plt.title ('Error Analysis Diagram (EAD)', fontsize = 14)
plt.show()

fig.savefig('Figure_6-X'+'.png', format='png', dpi=300,bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>Figure 6-8<jupyter_code>eps = [3,12,7,2]
fig, ax = plt.subplots(figsize = (8,5))
bar = ax.barh([1,2,3,4], eps, height= 1.0, color = (0.9,0.9,0.9), edgecolor = 'Black')
bar[0].set_hatch('/')
bar[0].set_color((0.9,0.9,0.9))
bar[0].set_edgecolor('Black')
bar[1].set_color((0.75,0.75,0.75))
bar[1].set_hatch('/')
bar[1].set_edgecolor('Black')
bar[2].set_hatch('/')
bar[2].set_color((0.6,0.6,0.6))
bar[2].set_edgecolor('Black')
bar[3].set_hatch('/')
bar[3].set_color((0.45,0.45,0.45))
bar[3].set_edgecolor('Black')
plt.yticks([1,2,3,4], ['Overfitting of dev dataset / $\Delta \epsilon_{overfitting\ dev}$',
                   'Data mismatch (between train and dev) / $\Delta \epsilon_{train-dev}$', 
                   'Overfitting of training dataset / $\Delta \epsilon_{overfitting\ train}$', 
                   'Bias / $\Delta \epsilon_{Bias}$'], 
            fontsize = 14)
plt.xlim(0,15)
plt.ylim(0.5,4.5)

for i,v in enumerate(eps):
    ax.text(v+0.4, i+0.88, str(v)+'%', color = 'Black', fontsize = 14)

    
plt.xticks([0,2,4,6,8,10,12,14],['0','2','4','6','8','10','12','14'], fontsize = 14)

plt.axhline(y=0.5, color = 'Black')
plt.axhline(y=1.5, color = 'Black')
plt.axhline(y=2.5, color = 'Black')
plt.axhline(y=3.5, color = 'Black')
plt.axhline(y=4.5, color = 'Black')

plt.title ('Error Analysis Diagram (EAD)', fontsize = 14)
plt.show()

fig.savefig('Figure_6-8'+'.png', format='png', dpi=300,bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>Figure 6-1<jupyter_code>x = np.arange(0,1000,10)
y = 90-95*np.exp(-x/70)+(x/200-2)*(x>500)
plt.rc('font', family='arial')
plt.rc('xtick', labelsize='x-small')
plt.rc('ytick', labelsize='x-small')
    
plt.tight_layout()

fig = plt.figure(figsize=(3.9, 3.1))
ax = fig.add_subplot(1, 1, 1)

ax.axhline(y = 95, color = 'Black', ls = '-')
ax.axhline(y = 85, color = 'Black', ls = '--')
ax.plot(x, y, lw = 2.0, ls = '-.', color = 'black', label = 'MSE Train')
#ax.plot(lambd_x, mse_dev_y, lw = 2.0, ls = '--', color = 'black', label = 'MSE Dev')

ax.spines['right'].set_visible(False)
ax.spines['top'].set_visible(False)

plt.xticks([])

ax.set_xlabel('Time invested in Research')
ax.set_ylabel('Accuracy')
ax.set_xlim(0,1000)
ax.set_ylim(0,100)
ax.text(400, 100, r'100 - $\epsilon_{Bayes}$', fontsize = 13)
ax.text(400, 75, r'100 - $\epsilon_{hlp}$', fontsize = 13)
#ax.text(12.5, 10, 'More\nregularization', fontsize = 13)
#plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)


fig.savefig('Figure_6-1'+'.png', format='png', dpi=300,bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>MNIST Shifted<jupyter_code>import numpy as np
from sklearn.datasets import fetch_mldata

%matplotlib inline

import matplotlib
import matplotlib.pyplot as plt

from random import *
mnist = fetch_mldata('MNIST original')
mnist
Xinput,yinput = mnist["data"], mnist["target"]
def plot_digit(some_digit):
    
    some_digit_image = some_digit.reshape(28,28)

    plt.imshow(some_digit_image, cmap = matplotlib.cm.binary, interpolation = "nearest")
    #plt.axis("off")
    plt.show()
    
plot_digit(Xinput[36003])
Xtrain = X_train_tr
ytrain = y_train_shifted
plot_digit(Xtrain[:,1005])<jupyter_output><empty_output><jupyter_text>Let's shift the images<jupyter_code>Xtrain_shifted = np.zeros_like(Xtrain)
for i in range(Xtrain.shape[1]):
    tmp = Xtrain[:,i].reshape(28,28)
    tmp_shifted = np.zeros_like(tmp)
    tmp_shifted[:,10:28] = tmp[:,0:18]
    Xtrain_shifted[:,i] = tmp_shifted.reshape(784)
plot_digit(Xtrain[:,7000])
plot_digit(Xtrain_shifted[:,7000])<jupyter_output><empty_output><jupyter_text>Figure 6-7<jupyter_code>f = plt.figure(figsize=(12,6));
plt.subplot(1,2,1)
plt.imshow(Xtrain[:,1345].reshape(28,28), cmap = matplotlib.cm.binary, interpolation = "nearest")
plt.subplot(1,2,2)
plt.imshow(Xtrain_shifted[:,1345].reshape(28,28), cmap = matplotlib.cm.binary, interpolation = "nearest")
f.savefig('Figure_6-7'+'.png', format='png', dpi=300,bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>Model  Let's create a train, a train-dev and a dev dataset<jupyter_code>X_ = Xinput[np.any([y == 1,y == 2], axis = 0)]
y_ = yinput[np.any([y == 1,y == 2], axis = 0)]
print(X_.shape)
np.random.seed(42)
rnd_train = np.random.rand(len(y_)) < 0.8

X_train = X_[rnd_train,:]
y_train = y_[rnd_train]
X_dev = X_[~rnd_train,:]
y_dev = y_[~rnd_train]

X_train_normalised = X_train/255.0
X_dev_normalised = X_dev/255.0
X_train_tr = X_train_normalised.transpose()
y_train_tr = y_train.reshape(1,y_train.shape[0])

n_dim = X_train_tr.shape[0]
dim_train = X_train_tr.shape[1]

X_dev_tr = X_dev_normalised.transpose()
y_dev_tr = y_dev.reshape(1,y_dev.shape[0])

y_train_shifted = y_train_tr - 1
y_dev_shifted = y_dev_tr - 1
Xtrain = X_train_tr
ytrain = y_train_shifted

Xdev = X_dev_tr
ydev = y_dev_shifted
print(Xtrain.shape)
print(Xdev.shape)<jupyter_output>(784, 11893)
(784, 2974)<jupyter_text>Let's create the train-dev dataset shifted<jupyter_code>Xtraindev = np.zeros_like(Xdev)
for i in range(Xdev.shape[1]):
    tmp = Xdev[:,i].reshape(28,28)
    tmp_shifted = np.zeros_like(tmp)
    tmp_shifted[:,10:28] = tmp[:,0:18]
    Xtraindev[:,i] = tmp_shifted.reshape(784)
    
ytraindev = ydev
print(Xtraindev.shape)
n_dim
import tensorflow as tf
tf.reset_default_graph()

X = tf.placeholder(tf.float32, [n_dim, None])
Y = tf.placeholder(tf.float32, [1, None])
learning_rate = tf.placeholder(tf.float32, shape=())

W = tf.Variable(tf.zeros([1, n_dim]))
b = tf.Variable(tf.zeros(1))

init = tf.global_variables_initializer()
y_ = tf.sigmoid(tf.matmul(W,X)+b)
cost = - tf.reduce_mean(Y * tf.log(y_)+(1-Y) * tf.log(1-y_))
training_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)
def run_logistic_model(learning_r, training_epochs, train_obs, train_labels, debug = False):
    sess = tf.Session()
    sess.run(init)
    
    cost_history = np.empty(shape=[0], dtype = float)

    for epoch in range(training_epochs+1):
        
        sess.run(training_step, feed_dict = {X: train_obs, Y: train_labels, learning_rate: learning_r})

        cost_ = sess.run(cost, feed_dict={ X:train_obs, Y: train_labels, learning_rate: learning_r})
        cost_history = np.append(cost_history, cost_)
        
        if (epoch % 10 == 0) & debug:
            print("Reached epoch",epoch,"cost J =", str.format('{0:.6f}', cost_))
            
    return sess, cost_history
sess, cost_history = run_logistic_model(learning_r = 0.01, 
                                training_epochs = 100, 
                                train_obs = Xtrain, 
                                train_labels = ytrain, 
                                debug = True)
correct_prediction1=tf.equal(tf.greater(y_, 0.5), tf.equal(Y,1))
accuracy1 = tf.reduce_mean(tf.cast(correct_prediction1, tf.float32))
print(sess.run(accuracy1, feed_dict={X:Xtrain, Y: ytrain, learning_rate: 0.05}))
correct_prediction1=tf.equal(tf.greater(y_, 0.5), tf.equal(Y,1))
accuracy1 = tf.reduce_mean(tf.cast(correct_prediction1, tf.float32))
print(sess.run(accuracy1, feed_dict={X:Xdev, Y: ydev, learning_rate: 0.05}))
correct_prediction1=tf.equal(tf.greater(y_, 0.5), tf.equal(Y,1))
accuracy1 = tf.reduce_mean(tf.cast(correct_prediction1, tf.float32))
print(sess.run(accuracy1, feed_dict={X:Xtraindev, Y: ytraindev, learning_rate: 0.05}))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.21691020002837283, 'fraction_numerical': 0.06688892041424316, 'mean_word_length': 4.874583333333334, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 47, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '297281', 'license': 'MIT', 'n_tokens_mistral': 6518, 'n_tokens_neox': 6080, 'n_words': 945, 'path': 'Week 8 - Metric Analysis/Week 8 - Metric Analysis - REFERENCE.ipynb', 'repo_name': 'michelucci/dlcourse_spring2019', 'size': '14131'}