<jupyter_start><jupyter_text>Simple Example of using WebLoader with Keras<jupyter_code>#!test -f training.tgz || curl http://storage.googleapis.com/lpr-imagenet-augmented/imagenet_train-0000-000.tgz -o training.tgz
#!test -f testing.tgz || curl http://storage.googleapis.com/lpr-imagenet-augmented/imagenet_val-0000.tgz -o testing.tgz
from importlib import reload
import keras
from keras.datasets import cifar10
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Conv2D, MaxPooling2D
from keras import models, layers
import os
import numpy as np
import webloader as wl
mean = np.array([0.485, 0.456, 0.406], "f")
std = np.array([0.229, 0.224, 0.225], "f")
def norm_image(xs):
    return (xs-mean[None,None,None,:])/std[None,None,None,:]
def norm_cls(ys):
    return keras.utils.to_categorical(ys-1, 1000)
urls = "http://storage.googleapis.com/lpr-imagenet-augmented/imagenet_val-0000.tgz"
#urls = "testing.tgz"
test_batch_size = 256
test_num_batches = 50000//test_batch_size
testing = wl.WebLoader(urls, 50000,
                       fields="ppm;png;jpg cls",
                       batch_transforms=[norm_image, norm_cls],
                       batch_size=test_batch_size)
for xs, ys in testing: break
print(xs.shape, xs.dtype, np.amin(xs), np.amax(xs))
print(ys.shape, ys.dtype, np.amin(ys), np.amax(ys))
urls = "http://storage.googleapis.com/lpr-imagenet-augmented/imagenet_train-{0000..0147}-{000..019}.tgz"
#urls = "training.tgz"
batch_size = 32
num_batches = 1000000//batch_size
training = wl.WebLoader(urls, 1000000,
                       fields="ppm;png;jpg cls",
                       batch_transforms=[norm_image, norm_cls],
                       batch_size=batch_size)
xs, ys = next(iter(training))
print(xs.shape, xs.dtype, np.amin(xs), np.amax(xs))
print(ys.shape, ys.dtype, np.amin(ys), np.amax(ys))
batch_norm = True
dropout = 0.5
nclasses = 1000
shape = (224, 224, 3)
batchsize = 32
lr = 0.001 # was: 0.0001

def vgg_block(d, r=2, name=None, size=(3, 3), mp_size=(2, 2), mp_strides=(2, 2)):
    result = []
    for i in range(r):
        name1 = name + "-{}".format(i) if name is not None else None
        if batch_norm:
            result += [
                layers.Conv2D(d, size, padding="same", name=name1),
                layers.BatchNormalization(),
                layers.Activation("relu")
            ]
        else:
            result += [layers.Conv2D(d, size, activation="relu", padding="same", name=name1)]        
    name1 = name + "-mp" if name is not None else None
    result += [layers.MaxPooling2D(mp_size, mp_strides, name=name1)]
    return result

model = models.Sequential(
    [layers.Lambda(lambda x: x + 0, input_shape=shape)] +         
    vgg_block(64, 2) + 
    vgg_block(128, 2) +
    vgg_block(256, 3) +
    vgg_block(512, 3) +
    vgg_block(512, 3) +
    [layers.Flatten(name="flatten"),
     layers.Dense(4096, activation="relu", name="fc1"),
     layers.Dropout(rate=1-dropout),
     layers.Dense(4096, activation="relu", name="fc2"),
     layers.Dropout(rate=1-dropout),
     layers.Dense(nclasses, activation="softmax", name="prediction")])

opt = keras.optimizers.rmsprop(lr=lr, decay=1e-6)

model.compile(loss='categorical_crossentropy',
              optimizer=opt,
              metrics=['accuracy'])

model.fit_generator(iter(training),
              epochs=1,
              steps_per_epoch=num_batches, #num_batches,
              validation_data=iter(testing),
              validation_steps=test_num_batches, #test_num_batches,
              shuffle=True)<jupyter_output>WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.
Instructions for updating:
Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.
Epoch 1/1
{'fraction_non_alphanumeric': 0.1298729741568112, 'fraction_numerical': 0.04095488392466053, 'mean_word_length': 3.539761431411531, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '292425', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1697, 'n_tokens_neox': 1549, 'n_words': 337, 'path': 'examples/keras.ipynb', 'repo_name': 'NVlabs/webloader', 'size': '4566'}