<jupyter_start><jupyter_text>Import modules<jupyter_code>import hashlib
import os
import pickle
from urllib.request import urlretrieve
import numpy as np
from PIL import Image
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelBinarizer, MinMaxScaler
from sklearn.utils import resample
from tqdm import tqdm
from zipfile import ZipFile
import matplotlib.pyplot as plt
import tensorflow as tf
import math

%matplotlib inline

print('All modules imported')<jupyter_output>/home/li/miniconda3/envs/carnd-term1/lib/python3.5/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
  from ._conv import register_converters as _register_converters<jupyter_text>Download `notMNIST` data set<jupyter_code>def download(url, file):
    if not os.path.isfile(file):
        print('Downloading {}'.format(file))
        urlretrieve(url, file)
        print('Download finished')

train_url = 'https://s3.amazonaws.com/udacity-sdc/notMNIST_train.zip'
train_file = 'notMNIST_train-1.zip'
test_url = 'https://s3.amazonaws.com/udacity-sdc/notMNIST_test.zip'
test_file = 'notMNIST_test-1.zip'

download(train_url, train_file)
download(test_url, test_file)

assert hashlib.md5(open(train_file, 'rb').read()).hexdigest() \
== 'c8673b3f28f489e9cdf3a3d74e2ac8fa', '{} is corrupted.'.format(train_file)
assert hashlib.md5(open(test_file, 'rb').read()).hexdigest() \
== '5d3c7e653e63471c88df796156a9dfa9', '{} is corrupted.'.format(test_file)
def uncompress_features_labels(file):
    """
    Uncompress features and labels from a zip file
    return np.array
    """
    features = []
    labels = []

    with ZipFile(file) as zipf:
        filenames_bar = tqdm(zipf.namelist(), unit='files')

        for filename in filenames_bar:
            if not filename.endswith('/'):
                with zipf.open(filename) as imagefile:
                    image = Image.open(imagefile)
                    image.load()
                    feature = np.array(image, dtype=np.float32).flatten()
                label = os.path.split(filename)[1][0]
                features.append(feature)
                labels.append(label)

    features = np.array(features)
    labels = np.array(labels)
    
    return features, labels
train_features, train_labels = uncompress_features_labels('notMNIST_train-1.zip')
test_features, test_labels = uncompress_features_labels('notMNIST_test-1.zip')
print('Train size: {}, test size: {}'.format(train_features.shape[0], test_features.shape[0]))
print(train_features.shape, train_labels.shape, test_features.shape, test_labels.shape)
print(train_labels[0])
img = train_features[0].reshape((28, 28)).astype(np.uint8)
Image.fromarray(img)
%matplotlib inline
np.random.seed(0)
img_idx = np.random.randint(train_features.shape[0], size=10)
print('Image indexes:', img_idx)
for n in img_idx:
    img = train_features[n].reshape((28, 28)).astype(np.uint8)
    plt.figure(figsize=(1, 1))
    plt.imshow(img, 'gray')
    plt.title('Label = {}'.format(train_labels[n]))
    plt.show()<jupyter_output>Image indexes: [199340  43567 173685 117952 176963 152315  95939  97639  41993 122579]<jupyter_text>Normalize features<jupyter_code>scaler = MinMaxScaler(feature_range=(0.1, 0.9))
train_features = scaler.fit_transform(train_features)
test_features = scaler.fit_transform(test_features)<jupyter_output><empty_output><jupyter_text>Binarize labels<jupyter_code>lb = LabelBinarizer()
train_labels = lb.fit_transform(train_labels)
test_labels = lb.fit_transform(test_labels)<jupyter_output><empty_output><jupyter_text>Save pickle files<jupyter_code>pickle_file = 'notMNIST.pickle'
if not os.path.isfile(pickle_file):
    print('Saving data to pickle file...')
    try:
        with open(pickle_file, 'wb') as pfile:
            pickle.dump({
                'train_features': train_features, 
                'train_labels': train_labels,
                'test_features': test_features,
                'test_labels': test_labels,
                        }, pfile, pickle.HIGHEST_PROTOCOL)
    except:
        print('Unable to save data to', pickle_file, ':', e)
        raise
        
print('Data cashed in pickle file.')<jupyter_output>Saving data to pickle file...
Data cashed in pickle file.<jupyter_text>Reload data from pickle file<jupyter_code>pickle_file = 'notMNIST.pickle'
with open(pickle_file, 'rb') as pfile:
    pickle_data = pickle.load(pfile)
    train_features = pickle_data['train_features']
    train_labels = pickle_data['train_labels']
    test_features = pickle_data['test_features']
    test_labels = pickle_data['test_labels']
    del pickle_data
    
print('Data loaded.')<jupyter_output>Data loaded.<jupyter_text>Create tensors<jupyter_code>features_count = 784
labels_count = 10

features = tf.placeholder(tf.float32)
labels = tf.placeholder(tf.float32)

weights = tf.Variable(tf.truncated_normal((features_count, labels_count)))
biases = tf.Variable(tf.zeros(labels_count))

biases.get_shape()
train_features, valid_features, train_labels, valid_labels = train_test_split(train_features, 
                                                                              train_labels, 
                                                                              test_size=0.05, 
                                                                              random_state=0)
train_feed_dict = {features: train_features, labels: train_labels}
valid_feed_dict = {features: valid_features, labels: valid_labels}
test_feed_dict = {features: test_features, labels: test_labels}

logits = tf.matmul(features, weights) + biases

prediction = tf.maximum(tf.nn.softmax(logits), 1.0e-30)

cross_entropy = -tf.reduce_sum(labels * tf.log(prediction), axis=1)

loss = tf.reduce_mean(cross_entropy, axis=0)

init = tf.global_variables_initializer()

is_correct_prediction = tf.equal(tf.argmax(prediction, 1), tf.argmax(labels, 1))

accuracy = tf.reduce_mean(tf.cast(is_correct_prediction, tf.float32), axis=0)


with tf.Session() as session:
    session.run(init)
    p = session.run(prediction, feed_dict=train_feed_dict)
    l = session.run(loss, feed_dict=train_feed_dict)
    d = session.run(logits, feed_dict=train_feed_dict)
    a = session.run(accuracy, feed_dict=train_feed_dict)
    idx = np.argmax(np.isnan(l)==True)
#     if idx!=0:
#         print(idx)
#         print(p[idx])
# #         print(d[idx])
#         ll = session.run(tf.log(p[idx]))
#         print(ll)
    print(p.min())
    print(a)
# 3907 is nan
import math

epochs = 20
batch_size = 2000
learning_rate = .1

optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)

test_accuracy = 0.0

log_batch_step = 50
batches = []
loss_batch = []
train_acc_batch = []
test_acc_batch = []

with tf.Session() as session:
    session.run(init)
    batch_count = int(math.ceil(train_features.shape[0]/batch_size))
    
    for epoch_i in range(epochs):
        
        batches_pbar = tqdm(range(batch_count), desc='Epoc {:>2}/{}'.format(epoch_i+1, epochs), unit='batches')
        
        for batch_i in batches_pbar:
            batch_start = batch_i * batch_size
            batch_features = train_features[batch_start:(batch_start + batch_size)]
            batch_labels = train_labels[batch_start:(batch_start + batch_size)]
            
            session.run(optimizer, feed_dict={features: batch_features, labels: batch_labels})
            l = session.run(loss, feed_dict={features: batch_features, labels: batch_labels})
            
            if not batch_i % log_batch_step:
                train_acc = session.run(accuracy, feed_dict=train_feed_dict)
                test_acc = session.run(accuracy, feed_dict=test_feed_dict)
                
                previous_batch = batches[-1] if batches else 0
                batches.append(log_batch_step + previous_batch)
                loss_batch.append(l)
                train_acc_batch.append(train_acc)
                test_acc_batch.append(test_acc)
                
    test_acc = session.run(accuracy, feed_dict=test_feed_dict)
    test_loss = session.run(loss, feed_dict=test_feed_dict)
        
loss_plot = plt.subplot(211)
loss_plot.set_title('Loss')
loss_plot.plot(batches, np.log10(loss_batch), 'g')
loss_plot.set_xlim([batches[0], batches[-1]])
acc_plot = plt.subplot(212)
acc_plot.set_title('Accuracy')
acc_plot.plot(batches, train_acc_batch, 'r', label='Training Accuracy')
acc_plot.plot(batches, test_acc_batch, 'x', label='Test Accuracy')
acc_plot.set_ylim([0, 1.0])
acc_plot.set_xlim([batches[0], batches[-1]])
acc_plot.legend(loc='lower right')
plt.tight_layout()
plt.show()
print(test_acc)
print(test_loss)
loss_plot = plt.subplot(211)
loss_plot.set_title('Loss')
loss_plot.plot(batches, np.log10(loss_batch), 'g')
loss_plot.set_xlim([batches[0], batches[-1]])
acc_plot = plt.subplot(212)
acc_plot.set_title('Accuracy')
acc_plot.plot(batches, train_acc_batch, 'r', label='Training Accuracy')
acc_plot.plot(batches, test_acc_batch, 'x', label='Test Accuracy')
acc_plot.set_ylim([0, 1.0])
acc_plot.set_xlim([batches[0], batches[-1]])
acc_plot.legend(loc='lower right')
plt.tight_layout()
plt.show()
print(test_acc)
print(test_loss)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1195069667738478, 'fraction_numerical': 0.022936763129689175, 'mean_word_length': 3.741361788617886, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 26, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '241105', 'license': 'MIT', 'n_tokens_mistral': 3318, 'n_tokens_neox': 3143, 'n_words': 557, 'path': 'lab-1.ipynb', 'repo_name': 'LiYan1988/tensorflow-lab', 'size': '9330'}