<jupyter_start><jupyter_text>MLP Data LoaderThe changes fot this data loader is that this dataloader jumps 4 frames for every 5 frames.<jupyter_code>import os
import torch
from torch.utils.data import Dataset
import numpy as np
from PIL import Image
import albumentations as A
import torchvision.transforms as transforms
from torch.utils.data import DataLoader
import string

augmentation = A.Compose([
    A.OneOf([
        A.IAAAdditiveGaussianNoise(p=0.9),
        A.GaussNoise(p=0.9),
    ], p=0.9),
    A.OneOf([
        A.MotionBlur(p=0.9),
        A.MedianBlur(blur_limit=3, p=0.9),
        A.Blur(blur_limit=4, p=0.9),
    ], p=0.9),
    A.OneOf([
        A.CLAHE(clip_limit=2, p=0.9),
        A.Sharpen(p=0.9),
        A.Emboss(p=0.9),
        A.RandomBrightnessContrast(p=0.95),
    ], p=0.9),
    A.OneOf(
        [
            A.HueSaturationValue(p=0.9),
            A.RandomGamma(p=0.9),
            A.Perspective(p=0.05),
        ], p=0.9,
    )
])


def build_transform(shape):
    transform = transforms.Compose([
        transforms.Resize((shape[0], shape[1])),
        transforms.ToTensor()
    ])
    return transform


class VideoDataset(Dataset):
    def __init__(self, folder_list, char_dict,
                 fixed_frame_num=200, fixed_max_len=6,
                 image_shape=(100, 100),
                 aug=augmentation):
        self.folders = folder_list
        np.random.shuffle(self.folders)
        self.fixed_frame_num = fixed_frame_num
        self.char_dict = char_dict
        self.fixed_max_len = fixed_max_len
        self.augmentation = aug
        self.image_shape = image_shape
        self.transform = build_transform(shape=self.image_shape)

    def __len__(self):
        return len(self.folders)

    def __getitem__(self, index):
        image_folder = self.folders[index]
        label = image_folder.split("/")[-1].split("_")[-1].strip(" ")
        label_digit = [self.char_dict[i] for i in label]
        assert len(label_digit) < self.fixed_max_len
        label_digit.append(self.char_dict["<eos>"])
        rest = self.fixed_max_len - len(label_digit)
        if rest:
            label_digit += [self.char_dict["<blank>"]] * rest

        image_list = [os.path.join(image_folder, i) for i in os.listdir(image_folder) if i.endswith(".jpg")]
        image_list = sorted(image_list)
        images = []

        if len(image_list) >= self.fixed_frame_num:
            image_list = image_list[:self.fixed_frame_num]
        else:
            image_list += ["pad"] * (self.fixed_frame_num - len(image_list))

        for index,i in enumerate(image_list):
            if index%5 == 0:
                if i != "pad":
                    img = Image.open(i).convert("RGB")
                    if self.augmentation is not None:
                        img = self.augmentation(image=np.array(img, dtype=np.uint8))["image"]
                        img = Image.fromarray(img)
                else:
                    img = Image.new("RGB", (self.image_shape[1], self.image_shape[0]))

                img = self.transform(img)

                images.append(img)

        x = torch.stack(images).flatten()
        y = torch.tensor(label_digit, dtype=torch.long)
        return x, y<jupyter_output>/home/allen/anaconda3/envs/pytorch/lib/python3.8/site-packages/albumentations/imgaug/transforms.py:252: FutureWarning: IAAAdditiveGaussianNoise is deprecated. Please use GaussNoise instead
  warnings.warn("IAAAdditiveGaussianNoise is deprecated. Please use GaussNoise instead", FutureWarning)<jupyter_text>Build MLP + RNN lipsreading  model<jupyter_code>import torch
import torch.nn.functional as F


class BidirectionalLSTM(torch.nn.Module):

    def __init__(self, nIn, nHidden, nOut):
        super(BidirectionalLSTM, self).__init__()

        self.rnn = torch.nn.LSTM(nIn, nHidden, bidirectional=True, batch_first=True)
        self.embedding = torch.nn.Linear(nHidden * 2, nOut)

    def forward(self, inputs):
        recurrent, _ = self.rnn(inputs)
        T, b, h = recurrent.size()
        t_rec = recurrent.reshape(T * b, h)
        output = self.embedding(t_rec)
        output = output.reshape(T, b, -1)
        output = F.softmax(output, dim=-1)
        return output


class VideoModel(torch.nn.Module):
    def __init__(self, number_classes=28, max_len=6, image_shape=(100, 100)):
        """

        :param number_classes:
        our char dictionary is:
        0: <blank>
        1: a
        2: b
        3: c
        ...
        26: z
        27: <eos>
        :param max_len: max_len = 6,
        Suppose we said abcde,
        the the label should be abcde<eos>
        abc -> abc<eos><blank><blank>
        number_classes = 28, 26 characters + <eos> + <blank>
        """
        super(VideoModel, self).__init__()
        self.number_classes = number_classes
        self.max_len = max_len
        
        #MLP processing layer
        self.mlp1= self.mlp(432000, 2000)
        self.mlp2 = self.mlp(2000,2000)
        self.mlp3 = self.mlp(2000, 2000*6)



        self.lstm_decoder = BidirectionalLSTM(nIn=2000,
                                              nHidden=256,
                                              nOut=number_classes)


    def forward(self, x):
        #x = x.permute(dims=(0, 2, 3, 4, 1))
        x = self.mlp1(x)
        x = self.mlp2(x)
        x = self.mlp3(x)
        shape = x.size()
        x = x.view(shape[0], self.max_len, -1)  # bs, max_len, rest

        x = self.lstm_decoder(x)
        return x

    def mlp(self, input_size, output_size):
        conv_block = torch.nn.Sequential(
            torch.nn.Linear(input_size, output_size),
            torch.nn.ReLU(),
            torch.nn.Linear(output_size, output_size),
            torch.nn.ReLU(),
        )
        return conv_block


if __name__ == '__main__':
    x = torch.rand(5, 432000)
    print('Input shape:', x.shape)
    model = VideoModel()
    y = model(x)
    print('Output shape:',y.size())  # [5, 6, 28]<jupyter_output>Input shape: torch.Size([5, 432000])
Output shape: torch.Size([5, 6, 28])<jupyter_text>Define the dataloader in this task<jupyter_code>import os

os.environ["CUDA_VISIBLE_DEVICES"] = "0"

import torch
from torch.utils.data import DataLoader
import string
import time
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print(device)


def make_char_dict():
    chars = string.ascii_lowercase
    char_dict = {"<blank>": 0}
    for idx, c in enumerate(chars):
        char_dict[c] = idx + 1
    current_len = len(list(char_dict.keys()))
    char_dict["<eos>"] = current_len
    print(char_dict)
    return char_dict


def get_train_test_folders():
    test = open("data/eval_lst.txt", "r", encoding="utf-8").readlines()
    train = open("data/train_lst.txt", "r", encoding="utf-8").readlines()
    train_folders = [os.path.join("data", "data_aligned", i.strip("\n")) for i in train]
    test_folders = [os.path.join("data", "data_aligned", i.strip("\n")) for i in test]
    print("train videos:{}".format(len(train_folders)))
    print("test videos:{}".format(len(test_folders)))
    return train_folders, test_folders


image_shape = (60, 60)

char_dict = make_char_dict()
train_folders, test_folders = get_train_test_folders()
train_dataset = VideoDataset(
    folder_list=train_folders,
    char_dict=char_dict,
    fixed_frame_num=200,
    fixed_max_len=6,
    image_shape=image_shape,
)
batch_size = 10
train_dataloader = DataLoader(
    train_dataset, batch_size=batch_size, shuffle=True
)
test_dataset = VideoDataset(
    folder_list=test_folders,
    char_dict=char_dict,
    fixed_frame_num=200,
    fixed_max_len=6,
    aug=None,  # No need to do data augmentation in testing dataset
    image_shape=image_shape,
)
test_dataloader = DataLoader(
    test_dataset, batch_size=batch_size, shuffle=True
)<jupyter_output>cuda
{'<blank>': 0, 'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5, 'f': 6, 'g': 7, 'h': 8, 'i': 9, 'j': 10, 'k': 11, 'l': 12, 'm': 13, 'n': 14, 'o': 15, 'p': 16, 'q': 17, 'r': 18, 's': 19, 't': 20, 'u': 21, 'v': 22, 'w': 23, 'x': 24, 'y': 25, 'z': 26, '<eos>': 27}
train videos:171
test videos:20<jupyter_text>Init our model<jupyter_code>model = VideoModel(number_classes=len(list(char_dict.keys())),
                   max_len=6,
                   image_shape=image_shape)
model = model.to(device)
print(model)<jupyter_output>VideoModel(
  (mlp1): Sequential(
    (0): Linear(in_features=432000, out_features=2000, bias=True)
    (1): ReLU()
    (2): Linear(in_features=2000, out_features=2000, bias=True)
    (3): ReLU()
  )
  (mlp2): Sequential(
    (0): Linear(in_features=2000, out_features=2000, bias=True)
    (1): ReLU()
    (2): Linear(in_features=2000, out_features=2000, bias=True)
    (3): ReLU()
  )
  (mlp3): Sequential(
    (0): Linear(in_features=2000, out_features=12000, bias=True)
    (1): ReLU()
    (2): Linear(in_features=12000, out_features=12000, bias=True)
    (3): ReLU()
  )
  (lstm_decoder): BidirectionalLSTM(
    (rnn): LSTM(2000, 256, batch_first=True, bidirectional=True)
    (embedding): Linear(in_features=512, out_features=28, bias=True)
  )
)<jupyter_text>Setup the loss function and optimizer<jupyter_code>criterion = torch.nn.CrossEntropyLoss().to(device)
optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)<jupyter_output><empty_output><jupyter_text>Setup the training epochs<jupyter_code># in real traning process, we set 300 epochs with early stopping,
# here we just use 10 epochs to show the entire pipeline for your reference
epochs = 10  # for easy running<jupyter_output><empty_output><jupyter_text>Setup learning rate scheduler for training<jupyter_code>scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
                                                       mode='min',
                                                       verbose=True,
                                                       factor=0.1,
                                                       patience=5,
                                                       threshold=0.00001)<jupyter_output><empty_output><jupyter_text>Define the acc metric function of our task<jupyter_code>def compute_val_acc(scores, y):
    num = scores.size(0)
    prediction = scores.argmax(dim=1)
    indicator = (prediction == y)
    num_matches = indicator.sum()
    return num_matches.float() / num<jupyter_output><empty_output><jupyter_text>Summarize the training process as a function<jupyter_code>def train_process():
    running_loss = 0
    num_batches = 0

    model.train()
    for idx, data in enumerate(train_dataloader):
        optimizer.zero_grad()

        x, y = data
        size = y.size()
        x = x.to(device)
        y = y.to(device)

        x.requires_grad_()

        scores = model(x)

        scores = scores.view(size[0] * size[1], -1)
        y = y.view(size[0] * size[1])
        loss = criterion(scores, y)

        loss.backward()
        torch.nn.utils.clip_grad_norm_(model.parameters(), 5)
        optimizer.step()
        running_loss += loss.detach().item()
        num_batches += 1
        print("time:{}, epoch: {} step: {}, avg running loss is {}".format(
            time.ctime(), epoch + 1, idx + 1, running_loss / num_batches
        ))
    return running_loss, num_batches<jupyter_output><empty_output><jupyter_text>summarize the validation process<jupyter_code>def testing_process():
    running_loss = 0
    num_batches = 0
    running_acc = 0
    model.eval()
    with torch.no_grad():
        for idx, data in enumerate(test_dataloader):
            x, y = data
            size = y.size()
            x = x.to(device)
            y = y.to(device)
            scores = model(x)

            scores = scores.view(size[0] * size[1], -1)
            y = y.view(size[0] * size[1])
            loss = criterion(scores, y)
            running_loss += loss.item()
            num_batches += 1
            running_acc += compute_val_acc(scores, y)
    return running_loss, num_batches, running_acc<jupyter_output><empty_output><jupyter_text>Training and evaluate models for each epoch   plus: using learning rate scheduler to improve the performance plus: using early stopping for real training task to avoid over-fitting<jupyter_code>lowest_loss = 100000000
lowest_loss_epoch = 0
c = 0
patiences = 10
for epoch in range(epochs):
    running_loss, num_batches = train_process()
    test_running_loss, test_num_batches, running_acc = testing_process()
    print("*" * 100)
    print("epoch: {}, avg training loss:{}, avg validation loss:{}, validation acc: {}".format(epoch + 1,
                                                                                               running_loss / num_batches,
                                                                                               test_running_loss / test_num_batches,
                                                                                               running_acc / test_num_batches))
    scheduler.step(test_running_loss / test_num_batches)
    print("*" * 100)
    # early stopping
    if test_running_loss / test_num_batches < lowest_loss:
        c = 0
        lowest_loss = test_running_loss / test_num_batches
        lowest_loss_epoch = epoch
        torch.save({
            'epoch': epoch,
            'model_state_dict': model.state_dict(),
            'optimizer_state_dict': optimizer.state_dict(),
            'loss': lowest_loss,
        }, "best_mlp_model.pt")
        print("save best model in best_mlp_model.pt")
    else:
        c += 1
    if c == patiences:
        print("no improvement for {} epochs, model is stopped".format(patiences))
        break<jupyter_output>time:Wed Nov 17 23:26:49 2021, epoch: 1 step: 1, avg running loss is 3.332216739654541
time:Wed Nov 17 23:26:56 2021, epoch: 1 step: 2, avg running loss is 3.332311749458313
time:Wed Nov 17 23:27:04 2021, epoch: 1 step: 3, avg running loss is 3.3322948614756265
time:Wed Nov 17 23:27:09 2021, epoch: 1 step: 4, avg running loss is 3.332262933254242
time:Wed Nov 17 23:27:15 2021, epoch: 1 step: 5, avg running loss is 3.332238960266113
time:Wed Nov 17 23:27:20 2021, epoch: 1 step: 6, avg running loss is 3.3322213888168335
time:Wed Nov 17 23:27:26 2021, epoch: 1 step: 7, avg running loss is 3.3321676594870433
time:Wed Nov 17 23:27:33 2021, epoch: 1 step: 8, avg running loss is 3.332140177488327
time:Wed Nov 17 23:27:39 2021, epoch: 1 step: 9, avg running loss is 3.3320784833696155
time:Wed Nov 17 23:27:46 2021, epoch: 1 step: 10, avg running loss is 3.33204185962677
time:Wed Nov 17 23:27:50 2021, epoch: 1 step: 11, avg running loss is 3.3320165764201772
time:Wed Nov 17 23:27:56 2021, epoch:[...]<jupyter_text>Inference process   load model from check point do evaluation  the trained model is too big for this case, we will not uploaed<jupyter_code>model = VideoModel(number_classes=len(list(char_dict.keys())),
                   max_len=6,
                   image_shape=image_shape)
optimizer = torch.optim.SGD(model.parameters(), lr=0.001, momentum=0.9)

checkpoint = torch.load("best_mlp_model.pt")
model.load_state_dict(checkpoint['model_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
epoch = checkpoint['epoch']
loss = checkpoint['loss']
model = model.to(device)
print(model)

model.eval()
acc = 0
count = 0
with torch.no_grad():
    for idx, data in enumerate(test_dataloader):
        x, y = data
        size = y.size()
        x = x.to(device)
        y = y.to(device)
        scores = model(x)

        scores = scores.view(size[0] * size[1], -1)
        y = y.view(size[0] * size[1])
        acc += compute_val_acc(scores, y)
        count += 1

print("Acc in inference process is {}".format(acc / count))<jupyter_output>VideoModel(
  (mlp1): Sequential(
    (0): Linear(in_features=432000, out_features=2000, bias=True)
    (1): ReLU()
    (2): Linear(in_features=2000, out_features=2000, bias=True)
    (3): ReLU()
  )
  (mlp2): Sequential(
    (0): Linear(in_features=2000, out_features=2000, bias=True)
    (1): ReLU()
    (2): Linear(in_features=2000, out_features=2000, bias=True)
    (3): ReLU()
  )
  (mlp3): Sequential(
    (0): Linear(in_features=2000, out_features=12000, bias=True)
    (1): ReLU()
    (2): Linear(in_features=12000, out_features=12000, bias=True)
    (3): ReLU()
  )
  (lstm_decoder): BidirectionalLSTM(
    (rnn): LSTM(2000, 256, batch_first=True, bidirectional=True)
    (embedding): Linear(in_features=512, out_features=28, bias=True)
  )
)
Acc in inference process is 0.31666669249534607
{'fraction_non_alphanumeric': 0.12456140350877193, 'fraction_numerical': 0.05045372050816697, 'mean_word_length': 2.6605403011514617, 'pattern_counts': {'":': 2, '<': 59, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '283211', 'license': 'MIT', 'n_tokens_mistral': 6199, 'n_tokens_neox': 5517, 'n_words': 1247, 'path': 'MLP_RNN_lipsreading.ipynb', 'repo_name': 'duguqiankun/lips_reading', 'size': '16530'}