<jupyter_start><jupyter_text>Dogs and Cats Example1. Download dataset from https://www.kaggle.com/c/dogs-vs-cats/leaderboard2. We have two files    - train.zip    - test1.zip2. Unzip these files to data folder3. Create a two folder     - train    - validation4. From train1 folder move cats 1-2000 and dogs 1-2000 to train folder5. From train1 folder move remaining ones to validation folder6. From test1 folder move all of them to the test/unknown folder7. We should have following folders at the end    - train      - dogs      - cats    - validation      - dogs      - cats    - test       - unknown<jupyter_code>import mxnet as mx
import os
from mxnet.gluon.data.vision import  transforms
from mxnet import gluon, autograd, ndarray
from mxnet.gluon import nn
from mxnet import gluon, init, nd
import time
from mxnet.gluon import data as gdata, loss as gloss, nn, utils as gutils
%matplotlib inline
from matplotlib.pylab import imshow
from multiprocessing import cpu_count
CPU_COUNT = cpu_count()
CPU_COUNT
data_folder = "data"
dataset_name = "dogs-vs-cats"
training_path = os.path.join(data_folder, dataset_name,"train")
validation_path = os.path.join(data_folder, dataset_name,"validation")
test_path = os.path.join(data_folder, dataset_name,"test1")
training_path
train_dataset = mx.gluon.data.vision.datasets.ImageFolderDataset(training_path)

validation_dataset = mx.gluon.data.vision.datasets.ImageFolderDataset(validation_path)
test_dataset = mx.gluon.data.vision.datasets.ImageFolderDataset(test_path)
batch_size = 32
def load_data(batch_size, resize=None):
    transformer = []
    if resize:
        transformer += [mx.gluon.data.vision.transforms.Resize(resize)]
    transformer += [mx.gluon.data.vision.transforms.ToTensor()]
    transformer = mx.gluon.data.vision.transforms.Compose(transformer)
    num_workers = 0 
    train_iter = mx.gluon.data.DataLoader(
        train_dataset.transform_first(transformer), batch_size, shuffle=True,
        num_workers=num_workers)
    validation_iter = mx.gluon.data.DataLoader(
        validation_dataset.transform_first(transformer), batch_size, shuffle=False,
        num_workers=num_workers)
    test_iter = mx.gluon.data.DataLoader(
        test_dataset.transform_first(transformer), batch_size, shuffle=False,
        num_workers=num_workers)
    return train_iter, validation_iter,test_iter
train_iter, validation_iter,test_iter = load_data(batch_size, resize=224)


sample_idx = 1250
sample = train_dataset[sample_idx]
sample_data = sample[0]
label = sample_data[1]
data_type = sample_data.dtype
data_shape = sample_data.shape

imshow(sample_data.asnumpy(), cmap='gray')
print(f"Data type: {data_type}, data size: {data_shape} ")


#for batch_idx, (data, label) in enumerate(train_data_loader):
#    print(data.shape)


def construct_net():
    num_fc = 512
    num_outputs = 2
    net = nn.Sequential()
    # Alexnet example
    # Here, we use a larger 11 x 11 window to capture objects. At the same time,
    # we use a stride of 4 to greatly reduce the height and width of the output.
    # Here, the number of output channels is much larger than that in LeNet
    net.add(nn.Conv2D(96, kernel_size=11, strides=4, activation='relu'),
        nn.MaxPool2D(pool_size=3, strides=2),
        # Make the convolution window smaller, set padding to 2 for consistent
        # height and width across the input and output, and increase the
        # number of output channels
        nn.Conv2D(256, kernel_size=5, padding=2, activation='relu'),
        nn.MaxPool2D(pool_size=3, strides=2),
        # Use three successive convolutional layers and a smaller convolution
        # window. Except for the final convolutional layer, the number of
        # output channels is further increased. Pooling layers are not used to
        # reduce the height and width of input after the first two
        # convolutional layers
        nn.Conv2D(384, kernel_size=3, padding=1, activation='relu'),
        nn.Conv2D(384, kernel_size=3, padding=1, activation='relu'),
        nn.Conv2D(256, kernel_size=3, padding=1, activation='relu'),
        nn.MaxPool2D(pool_size=3, strides=2),
        # Here, the number of outputs of the fully connected layer is several
        # times larger than that in LeNet. Use the dropout layer to mitigate
        # overfitting
        nn.Dense(4096, activation="relu"), nn.Dropout(0.5),
        nn.Dense(4096, activation="relu"), nn.Dropout(0.5),
        # Output layer. Since we are using Fashion-MNIST, the number of
        # classes is 10, instead of 1000 as in the paper
        nn.Dense(num_outputs))
    return net

# construct and initialize network.
ctx =  mx.gpu() if mx.test_utils.list_gpus() else mx.cpu()

net = construct_net()
net.initialize(mx.init.Xavier(), ctx=ctx)
# define loss and trainer.
criterion = gluon.loss.SoftmaxCrossEntropyLoss()
trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.1})
X = nd.random.uniform(shape=(1, 3, 224, 224))
net.initialize()
for layer in net:
    X = layer(X)
    print(layer.name, 'output shape:\t', X.shape)
def _get_batch(batch, ctx):
    """Return features and labels on ctx."""
    features, labels = batch
    if labels.dtype != features.dtype:
        labels = labels.astype(features.dtype)
    return (gutils.split_and_load(features, ctx),
            gutils.split_and_load(labels, ctx), features.shape[0])

def evaluate_accuracy(data_iter, net, ctx=[mx.cpu()]):
    """Evaluate accuracy of a model on the given data set."""
    if isinstance(ctx, mx.Context):
        ctx = [ctx]
    acc_sum, n = nd.array([0]), 0
    for batch in data_iter:
        features, labels, _ = _get_batch(batch, ctx)
        for X, y in zip(features, labels):
            y = y.astype('float32')
            acc_sum += (net(X).argmax(axis=1) == y).sum().copyto(mx.cpu())
            n += y.size
        acc_sum.wait_to_read()
    return acc_sum.asscalar() / n

def train_ch5(net, train_iter, test_iter, batch_size, trainer, ctx,
              num_epochs):
    """Train and evaluate a model with CPU or GPU."""
    print('training on', ctx)
    loss = mx.gluon.loss.SoftmaxCrossEntropyLoss()
    for epoch in range(num_epochs):
        train_l_sum, train_acc_sum, n, start = 0.0, 0.0, 0, time.time()
        for X, y in train_iter:
            X, y = X.as_in_context(ctx), y.as_in_context(ctx)
            with autograd.record():
                y_hat = net(X)
                l = loss(y_hat, y).sum()
            l.backward()
            trainer.step(batch_size)
            y = y.astype('float32')
            train_l_sum += l.asscalar()
            train_acc_sum += (y_hat.argmax(axis=1) == y).sum().asscalar()
            n += y.size
        test_acc = evaluate_accuracy(test_iter, net, ctx)
        print('epoch %d, loss %.4f, train acc %.3f, test acc %.3f, '
              'time %.1f sec'
              % (epoch + 1, train_l_sum / n, train_acc_sum / n, test_acc,
                 time.time() - start))

lr, num_epochs = 0.01, 1 # I train only one epoch since my computer is very slow.
net.initialize(force_reinit=True, ctx=ctx, init=init.Xavier())
trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': lr})
train_ch5(net, train_iter, validation_iter, batch_size, trainer, ctx,
              num_epochs)
preds = []
for data, label in test_iter:
    #print(data)
    output = net(data.as_in_context(ctx))
    #output = nd.softmax(net.output_new(output_features))
    preds.extend(output.asnumpy())
test_iter
len(preds)
ids = sorted(os.listdir(os.path.join(test_path,'unknown')))

with open('submission.csv', 'w') as f:
    f.write('id,label\n')
    for i, output in zip(ids, preds):
        f.write(i.split('.')[0] + ',' + ','.join(
            [str(num) for num in output]) + '\n')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11421221864951768, 'fraction_numerical': 0.0177491961414791, 'mean_word_length': 3.3056478405315612, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '151834', 'license': 'Apache-2.0', 'n_tokens_mistral': 2610, 'n_tokens_neox': 2535, 'n_words': 684, 'path': 'notebooks/old-mxnet/mxnet_image-classification-dogs-vs-cats-example01.ipynb', 'repo_name': 'ati-ozgur/kurs-neural-networks-deep-learning', 'size': '7775'}