<jupyter_start><jupyter_text>Version Control<jupyter_code>%%bash
function auth(){
    echo $(grep $1 ~/.auth_git | cut -d'"' -f 4 )
}
user=$(auth .user) pwd=$(auth .pwd) token=$(auth .token) email=$(auth .email) name=$(auth .name)
function git_clone(){
    clone_url="https://"$user":"$pwd$"@github.com/"$user"/"$1
    git clone $clone_url tmp && mv tmp/.git . && rm -rf tmp && git reset --hard
    git config --global user.email "$email" && git config --global user.name "$name"
    echo "https://github.com/"$user"/"$1
}
function git_push() { 
    git add -u && git commit -m "$2" && git push "https://"$token"@github.com/"$user"/"$1".git" 
}
# git_clone learn_torch
git pull
# git add README.md
# git add ex_main.py
# git add run_dataclass.py
# git_push learn_torch "edit"
# git status

#%%time
#!pip install -qr requirements.txt<jupyter_output>Updating 20686f3..c59dcca
Fast-forward
 mnist.ipynb | 5109 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 5109 insertions(+)
 create mode 100644 mnist.ipynb<jupyter_text>Dependency<jupyter_code>!pip install -qr tfrecord
# import torch, tensorflow as tf
# print('torch({}), tf({})'.format(torch.__version__, tf.__version__))
# import tensorflow_datasets as tfds
# mnist = tfds.load(name='mnist')
mnist['test']<jupyter_output><empty_output><jupyter_text>Get DataThe MNIST database of handwritten digits has 60,000 training examples, and 10,000 test examples. Each example included in the MNIST database is a 28x28 grayscale image of handwritten digit and its corresponding label(0-9).<jupyter_code>import torchvision.datasets as dset
dset.MNIST??
!ls ../data/MNIST/raw
../data/MNIST/raw/t10k-images-idx3-ubyte
../data/MNIST/raw/t10k-labels-idx1-ubyte
import codecs
def get_int(b: bytes) -> int:
    return int(codecs.encode(b, 'hex'), 16)

SN3_PASCALVINCENT_TYPEMAP = {
    8: (torch.uint8, np.uint8, np.uint8),
    9: (torch.int8, np.int8, np.int8),
    11: (torch.int16, np.dtype('>i2'), 'i2'),
    12: (torch.int32, np.dtype('>i4'), 'i4'),
    13: (torch.float32, np.dtype('>f4'), 'f4'),
    14: (torch.float64, np.dtype('>f8'), 'f8')
}


def read_sn3_pascalvincent_tensor(path: str, strict: bool = True) -> torch.Tensor:
    """Read a SN3 file in "Pascal Vincent" format (Lush file 'libidx/idx-io.lsh').
       Argument may be a filename, compressed filename, or file object.
    """
    # read
    with open(path, "rb") as f:
        data = f.read()
    # parse
    magic = get_int(data[0:4])
    nd = magic % 256
    ty = magic // 256
    assert 1 <= nd <= 3
    assert 8 <= ty <= 14
    m = SN3_PASCALVINCENT_TYPEMAP[ty]
    s = [get_int(data[4 * (i + 1): 4 * (i + 2)]) for i in range(nd)]
    parsed = np.frombuffer(data, dtype=m[1], offset=(4 * (nd + 1)))
    assert parsed.shape[0] == np.prod(s) or not strict
    return torch.from_numpy(parsed.astype(m[2], copy=False)).view(*s)


def read_label_file(path: str) -> torch.Tensor:
    x = read_sn3_pascalvincent_tensor(path, strict=False)
    assert(x.dtype == torch.uint8)
    assert(x.ndimension() == 1)
    return x.long()


def read_image_file(path: str) -> torch.Tensor:
    x = read_sn3_pascalvincent_tensor(path, strict=False)
    assert(x.dtype == torch.uint8)
    assert(x.ndimension() == 3)
    return x

data = read_image_file('../data/MNIST/raw/t10k-images-idx3-ubyte')
targets = read_label_file('../data/MNIST/raw/t10k-labels-idx1-ubyte')
train_loader = torch.utils.data.DataLoader(
                 dataset=(data, targets),
                 batch_size=batch_size,
                 shuffle=True)
from torchvision import datasets, transforms
transform = transforms.Compose([
                           transforms.ToTensor(),
                           transforms.Normalize((0.1307,), (0.3081,)),
                       ])
transform(image)
from torchvision import datasets, transforms
transform = transforms.Compose([
                                transforms.ToPILImage(),
                           transforms.ToTensor(),
                           transforms.Normalize((0.1307,), (0.3081,)),
                       ])
# data.shape

image = data[0,:,:]#.ToPILImage()
print(image.shape, type(image))
transform(image)


data[0,:,:].values
# type(data), type(targets)
# data.shape, targets.shape


# from torchvision import datasets, transforms
# dataset1 = datasets.MNIST('../data', train=True, download=True,
#                     transform=transform)
# dataset2 = datasets.MNIST('../data', train=False,
#                     transform=transform)
# !aws s3 cp s3://com.climate.production.users/people/jun.xiong/projects/mnist/ . --recursive
# loader
import torch, torchvision
from tfrecord.torch.dataset import TFRecordDataset
import matplotlib.pyplot as plt
batch_size = 64
tfrecord_path = "train/train.tfrecords.gz"
index_path = None
description = {"idx":"int", "image": "byte", "digit": "int"}
dataset = TFRecordDataset(tfrecord_path, index_path, description, compression_type='gzip')
loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size)

row = next(iter(loader))
# print(row)
# viz
data, target = row['image'].reshape(batch_size,1,28,28), row['digit'].reshape(batch_size)
img = torchvision.utils.make_grid(data)

img = img.numpy().transpose(1,2,0)
std = [0.5,0.5,0.5]
mean = [0.5,0.5,0.5]
img = img*std+mean
print([target[i] for i in range(64)])
plt.imshow(img)
# https://github.com/vahidk/tfrecord
import torch
from tfrecord.torch.dataset import TFRecordDataset
from torchvision import transforms
transform = transforms.Compose([transforms.ToTensor(),
                               transforms.Normalize(mean=[0.5,0.5,0.5],std=[0.5,0.5,0.5])])
def get_loader(tfrecord_path):
    index_path = None
    description = {"idx":"int", "image": "byte", "digit": "int"}
    dataset = TFRecordDataset(tfrecord_path, index_path, description, compression_type='gzip', transform=transform)
    return torch.utils.data.DataLoader(dataset, batch_size=64)

train_loader = get_loader("train/train.tfrecords.gz")
test_loader = get_loader("val/record/test.tfrecords.gz")
row = next(iter(train_loader))
print(row)
from __future__ import print_function
import argparse
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms
from torch.optim.lr_scheduler import StepLR

if True:
    train_kwargs = {'batch_size': 64}
    test_kwargs = {'batch_size': 1000}
    transform=transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize((0.1307,), (0.3081,))
        ])
    dataset1 = datasets.MNIST('../data', train=True, download=True,
                       transform=transform)
    dataset2 = datasets.MNIST('../data', train=False,
                       transform=transform)
    train_loader = torch.utils.data.DataLoader(dataset1,**train_kwargs)
    test_loader = torch.utils.data.DataLoader(dataset2, **test_kwargs)
# for batch_idx, (data, target) in enumerate(train_loader):
#     break
data = next(iter(train_loader))
data = next(iter(train_loader))
data, target = next(iter(train_loader))
data.shape, target.shape
data.shape, target.shape
img = torchvision.utils.make_grid(data)

img = img.numpy().transpose(1,2,0)
std = [0.5,0.5,0.5]
mean = [0.5,0.5,0.5]
img = img*std+mean
print([labels[i] for i in range(64)])
plt.imshow(img)<jupyter_output><empty_output><jupyter_text>Create Idx<jupyter_code># from glob import glob
# for f in glob('train/*.tfrecord'):
#     # !echo ${f} ${f}.idx
#     !python -m tfrecord.tools.tfrecord2idx train/${f} train/${f}.idx
for f in glob('val/*.tfrecord'):
    # !echo ${f} ${f}.idx
    !python -m tfrecord.tools.tfrecord2idx val/${f} val/${f}.idx
import torch
from tfrecord.torch.dataset import MultiTFRecordDataset

tfrecord_pattern = "train/{}.tfrecord"
index_pattern = "train/{}.tfrecord.idx"
splits = {
    "tfrecord_1": 1,
    "tfrecord_2": 1,
}
description = {"idx":"int", "image": "int", "digit": "int"}
dataset = MultiTFRecordDataset(tfrecord_pattern, index_pattern, splits, description)
loader = torch.utils.data.DataLoader(dataset, batch_size=32)

data = next(iter(loader))
print(data)
import itertools
dict(zip([f.split('/')[1].split('.')[0] for f in glob(f'train/tf*.tfrecord')], itertools.repeat(1)))
import torch
from tfrecord.torch.dataset import MultiTFRecordDataset
from glob import glob
import itertools

# transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize(mean=[0.5,0.5,0.5],std=[0.5,0.5,0.5])])
# def get_data_load(folder, batch_size=64):

#     tfrecord_pattern = f"{folder}/{{}}.tfrecord.gz"
#     # print(tfrecord_pattern)
#     index_pattern = f"{folder}/{{}}.tfrecord.idx"
#     flist = [f.split('/')[1].split('.')[0] for f in glob(f'{folder}/tf*.tfrecord')]
#     splits = dict(zip(flist, itertools.repeat(1)))

#     description = {"idx":"int", "image": "int", "digit": "int"}
#     dataset = MultiTFRecordDataset(tfrecord_pattern, index_pattern, splits, description, transform=transform)
#     return torch.utils.data.DataLoader(dataset, batch_size=batch_size)
tfrecord_path = "/tmp/data.tfrecord.gz"
index_path = None
description = {"idx":"int", "image": "byte", "label": "float"}
dataset = TFRecordDataset(tfrecord_path, index_path, description, )
loader = torch.utils.data.DataLoader(dataset, batch_size=32)


train_loader = get_data_load('train')
test_loader = get_data_load('val')
data = next(iter(train_loader))
print(data)
%run -i run_dataclass.py
train = MY_MNIST(transform=None)

# train[0]
for (cnt,i) in enumerate(train):
    image = i['img']
    label = i['target']
    ax = plt.subplot(4, 4, cnt+1)
    ax.axis('off')
    ax.imshow(image)
    ax.set_title(label)
    plt.pause(0.001)
    if cnt ==15:
        break
import numpy as np
import tensorflow as tf
!ls -l /content/record/train.tfrecords
import torch
from tfrecord.torch.dataset import TFRecordDataset

tfrecord_path = "/content/record/train.tfrecords"
index_path = None
description = {"idx":"int", "image": "byte", "digit": "int"}
dataset = TFRecordDataset(tfrecord_path, index_path, description)
loader = torch.utils.data.DataLoader(dataset, batch_size=32)

data = next(iter(loader))
print(data['image'].shape)<jupyter_output>torch.Size([32, 784])<jupyter_text>Create TFrecord from numpy<jupyter_code>%run -i run_numpy_array.py
import tensorflow.keras.datasets.mnist as mnist
# https://www.programcreek.com/python/?code=ddbourgin%2Fnumpy-ml%2Fnumpy-ml-master%2Fnumpy_ml%2Ftests%2Ftest_nn.py


(x_train, y_train), (x_test, y_test) = mnist.load_data()


class DatasetMNIST(torch.utils.data.Dataset):
    
    def __init__(self, X, y, transform=None):
        self.X = X
        self.y = y
        self.transform = transform
        
    def __len__(self):
        return self.X.shape[0]
    
    def __getitem__(self, index):
        image = torch.from_numpy(self.X[index, :, :]).reshape(1,28,28)
        label = self.y[index]
        
        if self.transform is not None:
            image = self.transform(image)
            
        return image, label

test_set = DatasetMNIST(x_train, y_train)
a, b = test_set[0]
type(a), type(b)
a.shape


y_train[0]
%run -i run_numpy_array.py
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train.shape, y_train.shape, x_test.shape, y_test.shape
x_train.shape[0]
%%time
def _int64_feature(value):
    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
def _bytes_feature(value):
    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
def write_to_tfrecord(X, y, filename):
    options = tf.io.TFRecordOptions(compression_type='GZIP')
    writer = tf.io.TFRecordWriter(filename, options=options)

    for i in range(X.shape[0]):
        image_raw = X[i].tobytes()
        example = tf.train.Example(features=tf.train.Features(
            feature={
                'idx': _int64_feature(i),
                'digit': _int64_feature(y[i]),
                'image': _bytes_feature(image_raw)
            }))
        writer.write(example.SerializeToString())
    writer.close()
write_to_tfrecord(x_train, y_train, 'record/train.tfrecords.gz')
write_to_tfrecord(x_test, y_test, 'record/test.tfrecords.gz')
!aws s3 cp record/test.tfrecords.gz s3://com.climate.production.users/people/jun.xiong/projects/mnist/val/
!aws s3 cp record/train.tfrecords.gz s3://com.climate.production.users/people/jun.xiong/projects/mnist/train/<jupyter_output>upload: record/test.tfrecords.gz to s3://com.climate.production.users/people/jun.xiong/projects/mnist/val/test.tfrecords.gz
upload: record/train.tfrecords.gz to s3://com.climate.production.users/people/jun.xiong/projects/mnist/train/train.tfrecords.gz
CPU times: user 4.95 s, sys: 80.8 ms, total: 5.03 s
Wall time: 8.76 s<jupyter_text>Insert<jupyter_code>class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 32, 3, 1)
        self.conv2 = nn.Conv2d(32, 64, 3, 1)
        self.dropout1 = nn.Dropout(0.25)
        self.dropout2 = nn.Dropout(0.5)
        self.fc1 = nn.Linear(9216, 128)
        self.fc2 = nn.Linear(128, 10)

    def forward(self, x):
        x = self.conv1(x)
        x = F.relu(x)
        x = self.conv2(x)
        x = F.relu(x)
        x = F.max_pool2d(x, 2)
        x = self.dropout1(x)
        x = torch.flatten(x, 1)
        x = self.fc1(x)
        x = F.relu(x)
        x = self.dropout2(x)
        x = self.fc2(x)
        output = F.log_softmax(x, dim=1)
        return output


def train(args, model, device, train_loader, optimizer, epoch):
    model.train()
    # for batch_idx, (data, target) in enumerate(train_loader):
    for batch_idx, row in enumerate(train_loader):
        data, target = row['image'].reshape(64,1,28,28), row['digit'].reshape(64)
        data, target = data.to(device), target.to(device)
        optimizer.zero_grad()
        output = model(data)
        loss = F.nll_loss(output, target)
        loss.backward()
        optimizer.step()
        if batch_idx % args.log_interval == 0:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch, batch_idx * len(data), len(train_loader.dataset),
                100. * batch_idx / len(train_loader), loss.item()))
            if args.dry_run:
                break


def test(model, device, test_loader):
    model.eval()
    test_loss = 0
    correct = 0
    with torch.no_grad():
        # for data, target in test_loader:
        for row in test_loader:
            data, target = row['image'].reshape(64,1,28,28), row['digit'].reshape(64)
            data, target = data.to(device), target.to(device)
            output = model(data)
            test_loss += F.nll_loss(output, target, reduction='sum').item()  # sum up batch loss
            pred = output.argmax(dim=1, keepdim=True)  # get the index of the max log-probability
            correct += pred.eq(target.view_as(pred)).sum().item()

    test_loss /= len(test_loader.dataset)

    print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
        test_loss, correct, len(test_loader.dataset),
        100. * correct / len(test_loader.dataset)))


if True:
    device = torch.device('cpu')
    model = Net().to(device)
    optimizer = optim.Adadelta(model.parameters(), lr=args.lr)

    scheduler = StepLR(optimizer, step_size=1, gamma=args.gamma)
    for epoch in range(1, args.epochs + 1):
        train(args, model, device, train_loader, optimizer, epoch)
        test(model, device, test_loader)
        scheduler.step()

    if args.save_model:
        torch.save(model.state_dict(), "mnist_cnn.pt")
args.epochs
data.reshape?
transform = transforms.Compose([transforms.ToTensor(),
                               transforms.Normalize(mean=[0.5,0.5,0.5],std=[0.5,0.5,0.5])])
data_train = datasets.MNIST(root = "./data/",
                            transform=transform,
                            train = True,
                            download = True)

data_test = datasets.MNIST(root="./data/",
                           transform = transform,
                           train = False)
data_loader_train = torch.utils.data.DataLoader(dataset=data_train,
                                                batch_size = 64,
                                                shuffle = True,
                                                 num_workers=2)

data_loader_test = torch.utils.data.DataLoader(dataset=data_test,
                                               batch_size = 64,
                                               shuffle = True,
                                                num_workers=2)
print(len(data_train))
images, labels = next(iter(data_loader_train))
img = torchvision.utils.make_grid(images)

img = img.numpy().transpose(1,2,0)
std = [0.5,0.5,0.5]
mean = [0.5,0.5,0.5]
img = img*std+mean
print([labels[i] for i in range(64)])
plt.imshow(img)
import torchvision
import matplotlib.pyplot as plt
row = next(iter(train_loader))
images, labels = row['image'], row['digit']
img = torchvision.utils.make_grid(images)

img = img.numpy().transpose(1,2,0)
std = [0.5,0.5,0.5]
mean = [0.5,0.5,0.5]
img = img*std+mean
print([labels[i] for i in range(64)])
plt.imshow(img)
data = next(iter(train_loader))
data
data['image'].shape
MultiTFRecordDataset?
# from torchvision import datasets, transforms
# transform=transforms.Compose([
#     transforms.ToTensor(),
#     transforms.Normalize((0.1307,), (0.3081,))
#     ])
# dataset1 = datasets.MNIST('../data', train=True, download=True, transform=transform)
# dataset2 = datasets.MNIST('../data', train=False,
#                     transform=transform)
!ls -l ../data/MNIST/raw/
from __future__ import print_function
import argparse
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms
from torch.optim.lr_scheduler import StepLR
from tfrecord.torch.dataset import MultiTFRecordDataset

class Net1(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 32, 3, 1)
        self.conv2 = nn.Conv2d(32, 64, 3, 1)
        self.dropout1 = nn.Dropout(0.25)
        self.dropout2 = nn.Dropout(0.5)
        self.fc1 = nn.Linear(9216, 128)
        self.fc2 = nn.Linear(128, 10)

    def forward(self, x):
        x = self.conv1(x)
        x = F.relu(x)
        x = self.conv2(x)
        x = F.relu(x)
        x = F.max_pool2d(x, 2)
        x = self.dropout1(x)
        x = torch.flatten(x, 1)
        x = self.fc1(x)
        x = F.relu(x)
        x = self.dropout2(x)
        x = self.fc2(x)
        output = F.log_softmax(x, dim=1)
        return output

class Net(nn.Module):
    
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = torch.nn.Sequential(torch.nn.Conv2d(1,64,kernel_size=3,stride=1,padding=1),
                                         torch.nn.ReLU(),
                                         torch.nn.Conv2d(64,128,kernel_size=3,stride=1,padding=1),
                                         torch.nn.ReLU(),
                                         torch.nn.MaxPool2d(stride=2,kernel_size=2))
        self.dense = torch.nn.Sequential(torch.nn.Linear(14*14*128,1024),
                                         torch.nn.ReLU(),
                                         torch.nn.Dropout(p=0.5),
                                         torch.nn.Linear(1024, 10))
    def forward(self, x):
        x = self.conv1(x)
        x = x.view(-1, 14*14*128)
        x = self.dense(x)
        return x

def train(args, model, device, train_loader, optimizer, epoch):
    model.train()
    # for batch_idx, (data, target) in enumerate(train_loader):
    for row in train_loader:
        batch_idx, data, target = row['idx'], row['image'], row['digit']
        data, target = data.to(device), target.to(device)
        optimizer.zero_grad()
        output = model(data)
        loss = F.nll_loss(output, target)
        loss.backward()
        optimizer.step()
        if batch_idx % args.log_interval == 0:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch, batch_idx * len(data), len(train_loader.dataset),
                100. * batch_idx / len(train_loader), loss.item()))
            if args.dry_run:
                break


def test(model, device, test_loader):
    model.eval()
    test_loss = 0
    correct = 0
    with torch.no_grad():
        # for idx, data, target in test_loader:
        for row in test_loader:
            batch_idx, data, target = row['idx'], row['image'], row['digit']
            data, target = data.to(device), target.to(device)
            output = model(data)
            test_loss += F.nll_loss(output, target, reduction='sum').item()  # sum up batch loss
            pred = output.argmax(dim=1, keepdim=True)  # get the index of the max log-probability
            correct += pred.eq(target.view_as(pred)).sum().item()

    test_loss /= len(test_loader.dataset)

    print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
        test_loss, correct, len(test_loader.dataset),
        100. * correct / len(test_loader.dataset)))

from collections import namedtuple
d = {'no_cuda':False, 'batch_size':64, 'test_batch_size':1000,'epochs':14, 'lr':1.0, 'gamma':0.7, 'dry_run':False, 'log_interval':10, 'save_model':True}
args = namedtuple('args', d.keys())(*d.values())

def main():
    # # Training settings
    # parser = argparse.ArgumentParser(description='PyTorch MNIST Example')
    # parser.add_argument('--batch-size', type=int, default=64, metavar='N',
    #                     help='input batch size for training (default: 64)')
    # parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N',
    #                     help='input batch size for testing (default: 1000)')
    # parser.add_argument('--epochs', type=int, default=14, metavar='N',
    #                     help='number of epochs to train (default: 14)')
    # parser.add_argument('--lr', type=float, default=1.0, metavar='LR',
    #                     help='learning rate (default: 1.0)')
    # parser.add_argument('--gamma', type=float, default=0.7, metavar='M',
    #                     help='Learning rate step gamma (default: 0.7)')
    # parser.add_argument('--no-cuda', action='store_true', default=False,
    #                     help='disables CUDA training')
    # parser.add_argument('--dry-run', action='store_true', default=False,
    #                     help='quickly check a single pass')
    # parser.add_argument('--seed', type=int, default=1, metavar='S',
    #                     help='random seed (default: 1)')
    # parser.add_argument('--log-interval', type=int, default=10, metavar='N',
    #                     help='how many batches to wait before logging training status')
    # parser.add_argument('--save-model', action='store_true', default=False,
    #                     help='For Saving the current Model')
    # args = parser.parse_args()
    # args =
    use_cuda = not args.no_cuda and torch.cuda.is_available()

    # torch.manual_seed(args.seed)

    device = torch.device("cuda" if use_cuda else "cpu")

    train_kwargs = {'batch_size': args.batch_size}
    test_kwargs = {'batch_size': args.test_batch_size}
    if use_cuda:
        cuda_kwargs = {'num_workers': 1,
                       'pin_memory': True,
                       'shuffle': True}
        train_kwargs.update(cuda_kwargs)
        test_kwargs.update(cuda_kwargs)

    transform=transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize((0.1307,), (0.3081,))
        ])
    # dataset1 = datasets.MNIST('../data', train=True, download=True,
    #                    transform=transform)
    # dataset2 = datasets.MNIST('../data', train=False,A
    #                    transform=transform)
    # # train_loader = torch.utils.data.DataLoader(dataset1,**train_kwargs)
    # # test_loader = torch.utils.data.DataLoader(dataset2, **test_kwargs)




    print(Net)
    model = Net().to(device)
    optimizer = optim.Adadelta(model.parameters(), lr=args.lr)

    scheduler = StepLR(optimizer, step_size=1, gamma=args.gamma)
    for epoch in range(1, args.epochs + 1):
        train(args, model, device, train_loader, optimizer, epoch)
        test(model, device, test_loader)
        scheduler.step()

    if args.save_model:
        torch.save(model.state_dict(), "mnist_cnn.pt")


# if __name__ == '__main__':
main()

# s3://com.climate.production.analytics/dsw/scratch/sagemaker/data/mnist-tfrecords/
# https://zhuanlan.zhihu.com/p/77952356

import tensorflow as tf
print (tf.__version__)
def tfrecord_parser(serialized_example):
    """Parses a single tf.Example into image and label tensors."""
 
    feature = {
        "idx": tf.FixedLenFeature([1], tf.int64),
        "image": tf.FixedLenFeature([28 * 28], tf.int64),
        "digit": tf.FixedLenFeature([1], tf.int64),
    }
 
    features = tf.parse_single_example(serialized_example, features=feature)
 
    # 28 x 28 is size of MNIST example
    image = tf.cast(tf.reshape(features["image"], [28 * 28]), tf.float32)
    digit = tf.reshape(features["digit"], [1])
 
    return {"image": image}, digit


batch_size = 256
num_shards = 1
shard_index = 0
num_epochs = 1
# tfrecord_glob_pattern = f"*.tfrecord"
filenames = ['s3://com.climate.production.analytics/dsw/scratch/sagemaker/data/mnist-tfrecords/train/tfrecord_1.tfrecord']
ds = tf.data.TFRecordDataset(filenames[:])
for x, y in ds.map(read_tfrecord):
    image = torch.from_numpy(x.numpy())
    digit = torch.from_numpy(y.numpy())
    break
image, digit
    # volume = torch.from_numpy(x.numpy())
    # segmentation = torch.from_numpy(y.numpy())

# return volume, segmentation
# ds = (
#     tf.data.Dataset.list_files(tfrecord_glob_pattern, shuffle=True)
#     .interleave(tf.data.TFRecordDataset, cycle_length=2)
#     .shard(num_shards=num_shards, index=shard_index)
#     .repeat(num_epochs)
#     .shuffle(buffer_size=100)
#     .map(tfrecord_parser, num_parallel_calls=4)
#     .batch(batch_size=batch_size)
# )

!aws s3 cp s3://com.climate.production.analytics/dsw/scratch/sagemaker/data/mnist-tfrecords/train/tfrecord_1.tfrecord .
import argparse
import os
import json
import ast
 
import tensorflow as tf
 
import logging as _logging
from tensorflow.python.platform import tf_logging
 
 
def model_fn(features, labels, mode, params):
    # model taken from https://www.kaggle.com/ilufei/mnist-with-tensorflow-dnn-97
    layer1 = tf.keras.layers.Dense(
        params["nr_neurons_first_layer"],
        activation="relu",
        input_shape=(params["batch_size"], 784),
        kernel_initializer=tf.contrib.layers.xavier_initializer(),
    )(features["image"])
    dropped_out = tf.layers.dropout(
        inputs=layer1, rate=0.4, training=(mode == tf.estimator.ModeKeys.TRAIN)
    )
    layer2 = tf.keras.layers.Dense(
        128,
        activation="relu",
        kernel_initializer=tf.contrib.layers.xavier_initializer(),
    )(dropped_out)
    layer3 = tf.keras.layers.Dense(
        64, activation="relu", kernel_initializer=tf.contrib.layers.xavier_initializer()
    )(layer2)
    layer4 = tf.keras.layers.Dense(
        32, activation="relu", kernel_initializer=tf.contrib.layers.xavier_initializer()
    )(layer3)
    layer5 = tf.keras.layers.Dense(
        16, activation="relu", kernel_initializer=tf.contrib.layers.xavier_initializer()
    )(layer4)
    logits = tf.keras.layers.Dense(
        10, kernel_initializer=tf.contrib.layers.xavier_initializer()
    )(layer5)
    predictions = tf.argmax(logits, 1)
 
    if mode == tf.estimator.ModeKeys.PREDICT:
        return tf.estimator.EstimatorSpec(
            mode=mode,
            predictions={"preds": predictions},
            export_outputs={
                "SIGNATURE_NAME": tf.estimator.export.PredictOutput(
                    {"preds": predictions}
                )
            },
        )
 
    cross_entropy = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)
 
    if mode == tf.estimator.ModeKeys.TRAIN:
        optimizer = tf.train.AdamOptimizer(learning_rate=1e-3)
        train_op = optimizer.minimize(
            loss=cross_entropy, global_step=tf.train.get_or_create_global_step()
        )
        return tf.estimator.EstimatorSpec(
            mode=mode, loss=cross_entropy, train_op=train_op
        )
 
    accuracy = tf.metrics.accuracy(
        labels=tf.cast(labels, tf.int64), predictions=tf.cast(predictions, tf.int64)
    )
    eval_metric_ops = {"accuracy": accuracy}
 
    # Provide an estimator spec for `ModeKeys.EVAL` mode.
    return tf.estimator.EstimatorSpec(
        mode=mode, loss=cross_entropy, eval_metric_ops=eval_metric_ops
    )
 
 
def tfrecord_parser(serialized_example):
    """Parses a single tf.Example into image and label tensors."""
 
    feature = {
        "idx": tf.FixedLenFeature([1], tf.int64),
        "image": tf.FixedLenFeature([28 * 28], tf.int64),
        "digit": tf.FixedLenFeature([1], tf.int64),
    }
 
    features = tf.parse_single_example(serialized_example, features=feature)
 
    # 28 x 28 is size of MNIST example
    image = tf.cast(tf.reshape(features["image"], [28 * 28]), tf.float32)
    digit = tf.reshape(features["digit"], [1])
 
    return {"image": image}, digit
 
 


def main():
    # tf.logging.set_verbosity(tf.logging.DEBUG)

    # TF 1.13 and 1.14 handle logging a bit different, so wrapping the logging setup in a try/except block
    try:
        tf_logger = tf_logging._get_logger()
        handler = tf_logger.handlers[0]
        handler.setFormatter(
            _logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
        )
    except:
        pass


main()
from box import Box
!ls ../data/MNIST/raw
!aws s3 cp s3://com.climate.production.analytics/dsw/scratch/sagemaker/data/mnist-tfrecords/val/ val --recursive

MultiTFRecordDataset??<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14769951480675925, 'fraction_numerical': 0.024861970888405555, 'mean_word_length': 2.97156146179402, 'pattern_counts': {'":': 33, '<': 26, '<?xml version=': 0, '>': 30, 'https://': 7, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 31, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '28330', 'license': 'MIT', 'n_tokens_mistral': 10553, 'n_tokens_neox': 10063, 'n_words': 1977, 'path': 'mnist.ipynb', 'repo_name': 'suredream/CNN-Sentinel', 'size': '29885'}