<jupyter_start><jupyter_text>Simple TrainThis Notebook is written to provide a self-contained overview over the data and network loading and trainingIt is organized in three parts1. Creation of a fake dataset with random values2. Loading this dataset using the Tensorflows [Dataset API](https://www.tensorflow.org/api_docs/python/tf/data/Dataset)3. Performing one training step load packages<jupyter_code>import numpy as np
from S2parser import S2parser
import os
import gzip
import shutil
import tensorflow as tf

parser=S2parser()<jupyter_output><empty_output><jupyter_text>1. Create a fake random dataset<jupyter_code>def write_random_sample(filepath,nobs=46, pix10=24, bands10=4, pix20=12, bands20=6, pix60=6, bands60=3):
    """create a fake sample with random values"""
    x10 = (np.random.random([nobs,pix10,pix10,bands10])*1e4).astype(np.int64)
    x20 = (np.random.random([nobs,pix20,pix20,bands20])*1e4).astype(np.int64)
    x60 = (np.random.random([nobs,pix60,pix60,bands60])*1e4).astype(np.int64)
    doy = (np.random.random([nobs])*365).astype(np.int64)
    year = np.round(np.random.random([nobs])+2016.5).astype(np.int64)
    label = (np.random.random([nobs,pix10,pix10])*17).astype(np.int64)

    # create instance of parser
    parser = S2parser()

    # write .tfrecord
    parser.write(filepath, x10, x20, x60, doy, year, label)

def ziptfrecord(infile,outfile):
    # gzip .tfrecord to .tfrecord.gz
    with open(infile, 'rb') as f_in:
        with gzip.open(outfile, 'wb') as f_out:
            shutil.copyfileobj(f_in, f_out)

    # remove unzipped .tfrecord
    #os.remove(infile)
# define directory to store the fake dataset
directory="tmp"
if not os.path.exists(directory):
    os.makedirs(directory)

filepaths=["{}/{}.tfrecord".format(directory,i) for i in range(10)]
zippedfilepaths=["{}/{}.tfrecord.gz".format(directory,i) for i in range(10)]
print(filepaths)

# create the dataset
for filepath,zippedfilepath in zip(filepaths,zippedfilepaths):
    write_random_sample(filepath)
    print("writing "+filepath)
    ziptfrecord(filepath,zippedfilepath)
    print("compressing "+filepath+" -> "+zippedfilepath)<jupyter_output>['tmp/0.tfrecord', 'tmp/1.tfrecord', 'tmp/2.tfrecord', 'tmp/3.tfrecord', 'tmp/4.tfrecord', 'tmp/5.tfrecord', 'tmp/6.tfrecord', 'tmp/7.tfrecord', 'tmp/8.tfrecord', 'tmp/9.tfrecord']
writing tmp/0.tfrecord
compressing tmp/0.tfrecord -> tmp/0.tfrecord.gz
writing tmp/1.tfrecord
compressing tmp/1.tfrecord -> tmp/1.tfrecord.gz
writing tmp/2.tfrecord
compressing tmp/2.tfrecord -> tmp/2.tfrecord.gz
writing tmp/3.tfrecord
compressing tmp/3.tfrecord -> tmp/3.tfrecord.gz
writing tmp/4.tfrecord
compressing tmp/4.tfrecord -> tmp/4.tfrecord.gz
writing tmp/5.tfrecord
compressing tmp/5.tfrecord -> tmp/5.tfrecord.gz
writing tmp/6.tfrecord
compressing tmp/6.tfrecord -> tmp/6.tfrecord.gz
writing tmp/7.tfrecord
compressing tmp/7.tfrecord -> tmp/7.tfrecord.gz
writing tmp/8.tfrecord
compressing tmp/8.tfrecord -> tmp/8.tfrecord.gz
writing tmp/9.tfrecord
compressing tmp/9.tfrecord -> tmp/9.tfrecord.gz<jupyter_text>2. Create a Tensorflow Dataset IteratorSee https://www.tensorflow.org/programmers_guide/datasets for more details- Essentially includes the data in the processing graph as input nodes.- These commands set up the processing graph of the input pipeline.- No data is processed,yet.- Some errors may remain undetected until data is inferred with sess.run(.)<jupyter_code>batchsize=2

print("Reset all previously loaded graphs (in case this or the next cells have been executed twice)")
tf.reset_default_graph()

print("creating dataset object")
dataset = tf.data.TFRecordDataset(zippedfilepaths, compression_type="GZIP")
#dataset = tf.data.TFRecordDataset(filepaths)


def normalize(serialized_feature):
    """ normalize stored integer values to floats approx. [0,1] """
    x10, x20, x60, doy, year, labels = serialized_feature
    x10 = tf.scalar_mul(1e-4, tf.cast(x10, tf.float32))
    x20 = tf.scalar_mul(1e-4, tf.cast(x20, tf.float32))
    x60 = tf.scalar_mul(1e-4, tf.cast(x60, tf.float32))
    doy = tf.cast(doy, tf.float32) / 365
    year = tf.cast(year, tf.float32) - 2016

    return x10, x20, x60, doy, year, labels

def mapping_function(serialized_feature):
    # read data from .tfrecords
    serialized_feature = parser.parse_example(serialized_feature)
    return normalize(serialized_feature)

print("applying the mapping function on all samples (will read tfrecord file and normalize the values)")
dataset = dataset.map(mapping_function)

print("repeat forever until externally stopped")
dataset = dataset.repeat()

print("combine samples to batches")
dataset = dataset.apply(tf.contrib.data.batch_and_drop_remainder(int(batchsize)))

print("make iterator")
iterator = dataset.make_initializable_iterator()<jupyter_output>Reset all previously loaded graphs (in case this or the next cells have been executed twice)
creating dataset object
applying the mapping function on all samples (will read tfrecord file and normalize the values)
repeat forever until externally stopped
combine samples to batches
make iterator<jupyter_text>3a. Retrieve one sample<jupyter_code>import matplotlib.pyplot as plt
%matplotlib inline


with tf.Session() as sess:
    sess.run(iterator.initializer)
    print("retrieving one sample as numpy array (just for fun)")
    x10, x20, x60, doy, year, labels = sess.run(iterator.get_next())
    print("x10.shape: " + str(x10.shape))
    plt.imshow(x10[0,0,:,:,0])
    plt.title("x10")<jupyter_output>retrieving one sample as numpy array (just for fun)
x10.shape: (2, 46, 24, 24, 4)<jupyter_text>3b Perform one training step<jupyter_code># define the network to be loaded
# if not yet created, make one with python modelzoo/seqencoder.py script, as described in the readme.md
graph="tmp/convgru128/graph.meta"

with tf.Session() as sess:

    print("loading network graph definition")
    tf.train.import_meta_graph(graph)

    print("initializing variables, tables and the data iterator")
    sess.run([tf.global_variables_initializer(), tf.local_variables_initializer(), tf.tables_initializer(),iterator.initializer])

    print("getting one string handle from the iterator that can be fed to the network")
    iterator_handle = sess.run(iterator.string_handle())

    print("making some processing nodes accessible to python")
    def get_operation(name):
        return tf.get_default_graph().get_operation_by_name(name).outputs[0]

    iterator_handle_op = get_operation("data_iterator_handle")
    is_train_op = get_operation("is_train")
    train_op = get_operation("train_op")

    print("performing one training step")
    sess.run(train_op,feed_dict={iterator_handle_op: iterator_handle, is_train_op: True})<jupyter_output>loading network graph definition
initializing variables, tables and the data iterator
getting one string handle from the iterator that can be fed to the network
making some processing nodes accessible to python
performing one training step
{'fraction_non_alphanumeric': 0.11269020958943439, 'fraction_numerical': 0.03158196956646569, 'mean_word_length': 5.095363079615048, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 28, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '173474', 'license': 'MIT', 'n_tokens_mistral': 2380, 'n_tokens_neox': 2262, 'n_words': 598, 'path': 'SimpleTrain.ipynb', 'repo_name': 'TUM-LMF/ijgi18', 'size': '6966'}