<jupyter_start><jupyter_text>1.2 Tensorflow operations<jupyter_code>import tensorflow as tf
x = tf.constant([22,21,32], name='x')
d=tf.constant([12,23,43],name='x')
y = tf.Variable(x * d, name='y')
print(y)
 
x = tf.constant([22,21,32], name='x')
d=tf.constant([12,23,43],name='d')
y = tf.Variable(x * d, name='y')
 
model = tf.initialize_all_variables()
 
with tf.Session() as session:
   session.run(model)
   print(session.run(y))
 
a = tf.placeholder(tf.int8)
b = tf.placeholder(tf.int8)
sess = tf.Session()
sess.run(a+b, feed_dict={a: 111, b: 222})<jupyter_output><empty_output><jupyter_text>GPU Computing (Caution, this only works with CUDA corresponding GPU card)  Linear Regression with SGD<jupyter_code>import tensorflow as tf
import numpy as np
 
 
X = tf.placeholder("float") # create symbolic variables
Y = tf.placeholder("float")
X_train = np.asarray([1,2.2,3.3,4.1,5.2])
Y_train =  np.asarray([2,3,3.3,4.1,3.9,1.6])
 
def model(X, w):
   return tf.mul(X, w)
 
 
w = tf.Variable(0.0, name="weights")
y_model = model(X, w) # our predicted values
 
cost = (tf.pow(Y-y_model, 2)) # squared error cost
 
train_op = tf.train.GradientDescentOptimizer(0.01).minimize(cost) #sgd optimization
sess = tf.Session()
init = tf.initialize_all_variables()
sess.run(init)
 
for trials in range(50):  #
   for (x, y) in zip(X_train, Y_train):
       sess.run(train_op, feed_dict={X: x, Y: y})
 
print(sess.run(w))<jupyter_output>0.844732<jupyter_text>Neural Network from scratch in Tensorflow:<jupyter_code>import tensorflow as tf
import numpy as np
from sklearn import cross_validation
from sklearn.cross_validation import train_test_split
from sklearn.preprocessing import OneHotEncoder
from sklearn.utils import shuffle
from sklearn import preprocessing
from sklearn import datasets
import os
import pandas as pd
from datetime import datetime as dt
import logging
 
iris = datasets.load_iris()
X = np.asarray(iris.data, 'float32')
 
Y = iris.target
 
from sklearn import preprocessing
X= preprocessing.scale(X)
min_max_scaler = preprocessing.MinMaxScaler()
X = min_max_scaler.fit_transform(X)
 
lb = preprocessing.LabelBinarizer()
Y=lb.fit_transform(iris.target)
 
X_train, x_test, y_train, y_test = train_test_split(X,Y,test_size=0.3,random_state=22)
 
 
def init_weights(shape):
   return tf.Variable(tf.random_normal(shape, stddev=0.01))
 
# here we can see the feedforward pass
def model(X, w_h, w_o):
   h = tf.nn.sigmoid(tf.matmul(X, w_h))
   return tf.matmul(h, w_o)
 
 
 
X = tf.placeholder("float", [None, 4])
Y = tf.placeholder("float", [None, 3])
 
#here we set up our layer architecture with 1 hidden layer.
w_h = init_weights([4, 4])
w_o = init_weights([4, 3])
py_x = model(X, w_h, w_o)
 
 
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(py_x, Y)) # compute costs
train_op = tf.train.GradientDescentOptimizer(learning_rate=0.01).minimize(cost) # construct an optimizer
 
predict_op = tf.argmax(py_x, 1)
 
sess = tf.Session()
init = tf.initialize_all_variables()
sess.run(init)
 
   
for i in range(500):
   for start, end in zip(range(0, len(X_train),1 ), range(1, len(X_train),1)):
       sess.run(train_op, feed_dict={X: X_train[start:end], Y: y_train[start:end]})
   if i % 100 == 0:
       print i, np.mean(np.argmax(y_test, axis=1) ==
                    sess.run(predict_op, feed_dict={X: x_test, Y: y_test}))<jupyter_output>0 0.288888888889
100 0.777777777778
200 0.933333333333
300 0.977777777778
400 0.955555555556<jupyter_text>Machine Learning on Tensorflow with SKflow:<jupyter_code>import numpy as np
from sklearn.metrics import accuracy_score
import skflow            
import urllib2
url = 'https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/wine.scale'
set1 = urllib2.Request(url)
wine = urllib2.urlopen(set1)    
 
   
from sklearn.datasets import load_svmlight_file
X_train, y_train = load_svmlight_file(wine)
X_train=X_train.toarray()
 
from sklearn.cross_validation import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X_train,
y_train, test_size=0.30, random_state=4)
 
 
classifier = skflow.TensorFlowLinearClassifier(n_classes=4,learning_rate=0.01, optimizer='SGD',continue_training=True, steps=1000)
classifier.fit(X_train, y_train)
score = accuracy_score(y_train, classifier.predict(X_train))
d=classifier.predict(X_test)
print("Accuracy: %f" % score)
 
c=accuracy_score(d,y_test)
print('validation/test accuracy: %f' % c)<jupyter_output>Step #1, avg. loss: 1.58672
Step #101, epoch #25, avg. loss: 1.45840
Step #201, epoch #50, avg. loss: 1.09080
Step #301, epoch #75, avg. loss: 0.84564
Step #401, epoch #100, avg. loss: 0.68503
Step #501, epoch #125, avg. loss: 0.57680
Step #601, epoch #150, avg. loss: 0.50120
Step #701, epoch #175, avg. loss: 0.44486
Step #801, epoch #200, avg. loss: 0.40151
Step #901, epoch #225, avg. loss: 0.36760
Accuracy: 0.967742
validation/test accuracy: 0.981481<jupyter_text>Neural Networks on SKflow (Pima Indians, diabetes dataset)<jupyter_code>import tensorflow
import tensorflow as tf
import numpy as np
import urllib
import skflow
from sklearn.preprocessing import Normalizer
from sklearn import datasets, metrics, cross_validation
from sklearn.cross_validation import train_test_split
# Pima Indians Diabetes dataset (UCI Machine Learning Repository)
url = "http://archive.ics.uci.edu/ml/machine-learning-databases/pima-indians-diabetes/pima-indians-diabetes.data"
# download the file
raw_data = urllib.urlopen(url)
dataset = np.loadtxt(raw_data, delimiter=",")
print(dataset.shape)
X = dataset[:,0:7]
y = dataset[:,8]
X_train, X_test, y_train, y_test = train_test_split(X, y,
                                                   test_size=0.2,
                                                   random_state=0)
 
 
from sklearn import preprocessing
X= preprocessing.scale(X)
min_max_scaler = preprocessing.MinMaxScaler()
X = min_max_scaler.fit_transform(X)<jupyter_output>(768, 9)<jupyter_text>Setting up exponential decay<jupyter_code>def exp_decay(global_step):
   return tf.train.exponential_decay(
       learning_rate=0.01, global_step=global_step,
       decay_steps=steps, decay_rate=0.01)
steps = 5000
classifier = skflow.TensorFlowDNNClassifier(
   hidden_units=[5,4],
   n_classes=2,
   batch_size=300,
   steps=steps,
   optimizer='Adam',#SGD  #RMSProp
   learning_rate=exp_decay #here is the decay function
    )
classifier.fit(X_train,y_train)
score1a = metrics.accuracy_score(y_train, classifier.predict(X_train))
print("Accuracy: %f" % score1a)
score1b = metrics.accuracy_score(y_test, classifier.predict(X_test))
print("Validation Accuracy: %f" % score1b)<jupyter_output>Step #1, avg. loss: 12.83679
Step #501, epoch #167, avg. loss: 0.69306
Step #1001, epoch #333, avg. loss: 0.56356
Step #1501, epoch #500, avg. loss: 0.54453
Step #2001, epoch #667, avg. loss: 0.54554
Step #2501, epoch #833, avg. loss: 0.53300
Step #3001, epoch #1000, avg. loss: 0.53266
Step #3501, epoch #1167, avg. loss: 0.52815
Step #4001, epoch #1333, avg. loss: 0.52639
Step #4501, epoch #1500, avg. loss: 0.52721
Accuracy: 0.754072
Validation Accuracy: 0.740260<jupyter_text>Improve accuracy with PCA & whitening<jupyter_code>from sklearn.decomposition import PCA
from sklearn import linear_model, decomposition, datasets
from sklearn.pipeline import Pipeline
from sklearn.metrics import accuracy_score
 
pca = PCA(n_components=4,whiten=True)
 
lr = pca.fit(X)
classifier = skflow.TensorFlowDNNClassifier(
   hidden_units=[5,4],
   n_classes=2,
   batch_size=300,
   steps=steps,
   optimizer='Adam',#SGD  #RMSProp
   learning_rate=exp_decay
    )
 
 
pipe = Pipeline(steps=[('pca', pca), ('NNET', classifier)])
 
X_train, X_test, Y_train, Y_test = train_test_split(X, y,
                                                   test_size=0.2,
                                                   random_state=0)
pipe.fit(X_train, Y_train)
 
 
 
score2 = metrics.accuracy_score(Y_test, pipe.predict(X_test))
print("Accuracy Validation, with pca: %f" % score2)<jupyter_output>Step #1, avg. loss: 1.07512
Step #501, epoch #167, avg. loss: 0.54236
Step #1001, epoch #333, avg. loss: 0.50186
Step #1501, epoch #500, avg. loss: 0.49243
Step #2001, epoch #667, avg. loss: 0.48541
Step #2501, epoch #833, avg. loss: 0.46982
Step #3001, epoch #1000, avg. loss: 0.47928
Step #3501, epoch #1167, avg. loss: 0.47598
Step #4001, epoch #1333, avg. loss: 0.47464
Step #4501, epoch #1500, avg. loss: 0.47712
Accuracy Validation, with pca: 0.805195<jupyter_text>Deep Learning with Large Files First we generate some example data and write it to disk:<jupyter_code>import numpy as np
import pandas as pd
import skflow
from sklearn.datasets import make_classification
import random
from sklearn.cross_validation import train_test_split
import gc
import tensorflow as tf
from sklearn.metrics import accuracy_score
X, y = make_classification(n_samples=5000000,n_features=10, n_classes=4,n_informative=6,random_state=222,n_clusters_per_class=1)
X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.2, random_state=22)
 
Big_trainm=pd.DataFrame(X_train,y_train)
Big_testm = pd.DataFrame(X_test,y_test)
 
 
Big_trainm.to_csv('lsml-Bigtrainm', sep=',')
Big_testm.to_csv('lsml-Bigtestm', sep=',')
del(X,y,X_train,y_train,X_test)
gc.collect<jupyter_output><empty_output><jupyter_text>A function to random subsample data from disk (without loading the full dataset in memory)<jupyter_code>import pandas as pd
import random
def sample_file():
    global skip_idx
    global train_data
    global X_train
    global y_train
    big_train='lsml-Bigtrainm'

    # Count number of rows entire set
    num_lines = sum(1 for i in open(big_train))

    # Sample size - fraction of the train-set 1/3
    size = int(num_lines / 3)

    # skip indexes and keep indices
    skip_idx = random.sample(range(1, num_lines), num_lines - size)
    train_data = pd.read_csv(big_train, skiprows=skip_idx)  
    X_train=train_data.drop(train_data.columns[[0]], axis=1)
    y_train = train_data.ix[:,0]<jupyter_output><empty_output><jupyter_text>Setting up weight decay and our neural network<jupyter_code>def exp_decay(global_step):
   return tf.train.exponential_decay(
       learning_rate=0.01, global_step=global_step,
       decay_steps=steps, decay_rate=0.01)

steps = 5000
clf = skflow.TensorFlowDNNClassifier(
   hidden_units=[5,4,4],
   n_classes=4,
   batch_size=300,
   steps=steps,
   optimizer='Adam',
   learning_rate=exp_decay
    )

epochs=3
for i in range(epochs):
   sample_file()
   clf.partial_fit(X_train,y_train)
 
 
test_data = pd.read_csv('lsml-Bigtestm',sep=',')  
X_test=test_data.drop(test_data.columns[[0]], axis=1)
y_test = test_data.ix[:,0]  
score = accuracy_score(y_test, clf.predict(X_test))
print score<jupyter_output>Step #1, avg. loss: 1.61924
Step #501, avg. loss: 0.47881
Step #1001, avg. loss: 0.26629
Step #1501, avg. loss: 0.23577
Step #2001, avg. loss: 0.21795
Step #2501, avg. loss: 0.20363
Step #3001, avg. loss: 0.19687
Step #3501, avg. loss: 0.19372
Step #4001, avg. loss: 0.19267
Step #4501, epoch #1, avg. loss: 0.19268
Step #1, avg. loss: 1.68622
Step #501, avg. loss: 0.49493
Step #1001, avg. loss: 0.28307
Step #1501, avg. loss: 0.26928
Step #2001, avg. loss: 0.25467
Step #2501, avg. loss: 0.24505
Step #3001, avg. loss: 0.24192
Step #3501, avg. loss: 0.23625
Step #4001, avg. loss: 0.23650
Step #4501, epoch #1, avg. loss: 0.23614
Step #1, avg. loss: 1.64617
Step #501, avg. loss: 0.50067
Step #1001, avg. loss: 0.23848
Step #1501, avg. loss: 0.21030
Step #2001, avg. loss: 0.19792
Step #2501, avg. loss: 0.19607
Step #3001, avg. loss: 0.19411
Step #3501, avg. loss: 0.19351
Step #4001, avg. loss: 0.19286
Step #4501, epoch #1, avg. loss: 0.19193
0.9424<jupyter_text>Keras and Tensorflow<jupyter_code>import numpy as np
import keras
from sklearn.datasets import make_classification
from sklearn.cross_validation import train_test_split
from sklearn.preprocessing import OneHotEncoder
from keras.utils import np_utils, generic_utils
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation
from keras.optimizers import SGD
import time
nb_classes=3
X, y = make_classification(n_samples=1000, n_features=4, n_classes=nb_classes,n_informative=3, n_redundant=0, random_state=101)
y=np_utils.to_categorical(y,nb_classes)
print y
x_train, x_test, y_train, y_test = train_test_split(X, y,test_size=0.30, random_state=101)
#use this for reproducibility 
seed = 22
np.random.seed(seed)

model = Sequential()
model.add(Dense(4, input_shape=(4,)))
model.add(Activation('relu'))
model.add(Dense(3))
model.add(Activation('relu'))
model.add(Dense(3))
model.add(Activation('softmax'))

sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy', optimizer=sgd)
model.fit(x_train, y_train, verbose=1, batch_size=100, nb_epoch=50,show_accuracy=True,validation_data=(x_test, y_test))
time.sleep(0.1)

adam=keras.optimizers.Adam(lr=0.01)
model.compile(loss='categorical_crossentropy', optimizer=adam)
model.fit(x_train, y_train, verbose=1, batch_size=100, nb_epoch=200,show_accuracy=True,validation_data=(x_test, y_test))
time.sleep(0.1)<jupyter_output>Train on 700 samples, validate on 300 samples
Epoch 1/50
700/700 [==============================] - 0s - loss: 1.4109 - acc: 0.3257 - val_loss: 1.1695 - val_acc: 0.3267
Epoch 2/50
700/700 [==============================] - 0s - loss: 1.1322 - acc: 0.3714 - val_loss: 1.0529 - val_acc: 0.5000
Epoch 3/50
700/700 [==============================] - 0s - loss: 1.0494 - acc: 0.5214 - val_loss: 1.0364 - val_acc: 0.5133
Epoch 4/50
700/700 [==============================] - 0s - loss: 1.0314 - acc: 0.5443 - val_loss: 1.0221 - val_acc: 0.5433
Epoch 5/50
700/700 [==============================] - 0s - loss: 1.0075 - acc: 0.5786 - val_loss: 0.9960 - val_acc: 0.5733
Epoch 6/50
700/700 [==============================] - 0s - loss: 0.9747 - acc: 0.6100 - val_loss: 0.9618 - val_acc: 0.6167
Epoch 7/50
700/700 [==============================] - 0s - loss: 0.9387 - acc: 0.6386 - val_loss: 0.9272 - val_acc: 0.6400
Epoch 8/50
700/700 [==============================] - 0s - loss: 0.9060 - acc: 0.6457 - val_l[...]<jupyter_text>Convolutional Neural Networks with Keras<jupyter_code>from keras.datasets import cifar10
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.layers.convolutional import Convolution2D, MaxPooling2D
from keras.optimizers import SGD
from keras.utils import np_utils

batch_size = 32
nb_classes = 10
nb_epoch = 5 #these are the number of epochs, watch out because it might set your #cpu/gpu on fire.


# input image dimensions
img_rows, img_cols = 32, 32
# the CIFAR10 images are RGB
img_channels = 3

# the data, shuffled and split between train and test sets
(X_train, y_train), (X_test, y_test) = cifar10.load_data()
print('X_train shape:', X_train.shape)
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')

#remember we need to encode the target variable
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)

model = Sequential()

#this is the first convolutional layer, we set the filter size
model.add(Convolution2D(32, 3, 3, border_mode='same',
                        input_shape=(img_channels, img_rows, img_cols)))
model.add(Activation('relu'))
#the second convolutional layer
model.add(Convolution2D(32, 3, 3))
model.add(Activation('relu'))
#here we specify the pooling layer
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))


#first we flatten the input towards the fully connected layer into the softmax function
model.add(Flatten())
model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))

# let's train the model using SGD + momentum like we have done before.
sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy', optimizer=sgd)

X_train = X_train.astype('float32')
X_test = X_test.astype('float32')

#Here we apply scaling to the features 
X_train /= 255
X_test /= 255

for epoch in xrange(nb_epoch):
    model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=1,show_accuracy=True
             ,validation_data=(X_test, Y_test), shuffle=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13707563844700432, 'fraction_numerical': 0.08094106174193942, 'mean_word_length': 4.9000359582883855, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 5, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '441476', 'license': 'MIT', 'n_tokens_mistral': 7069, 'n_tokens_neox': 6146, 'n_words': 1494, 'path': 'Module 3/Chapter 5/Chapter_5_code.ipynb', 'repo_name': 'saicharanabhishek/machinelearning_examples', 'size': '16407'}