<jupyter_start><jupyter_text>1.1 Preprocessing and Extraction of frames add the necessary libraries<jupyter_code>#get the names of the files
movie_files = []
video_dataset = "/content/drive/MyDrive/Game-Movie/video_data"
for dirname, _, filenames in os.walk(video_dataset):
    for filename in filenames:
        print(os.path.join(dirname, filename))
        if filename!="MafiaVideogame.mp4":
            movie_files.append(os.path.join(dirname, filename))
        else:
            game_file = os.path.join(dirname, filename)

def get_length(filename):
    result = subprocess.run(["ffprobe", "-v", "error", "-show_entries",
                             "format=duration", "-of",
                             "default=noprint_wrappers=1:nokey=1", filename],
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT)
    return float(result.stdout)

def get_secs(filename):
    video = cv2.VideoCapture(filename) 
    
    # count the number of frames 
    frames = video.get(cv2.CAP_PROP_FRAME_COUNT) 
    fps = int(video.get(cv2.CAP_PROP_FPS)) 

    seconds = int(frames / fps) 
    return seconds
def get_name(filename): #get just the name of the file for naming purposes
    name = filename.split(".")[0]
    name = name.split("/")[-1]
    return name

def get_frames(filename,amount,new_path):
    count=0
    Path(new_path).mkdir(parents=True, exist_ok=True)
    name = get_name(filename)
    total_secs = get_secs(filename)
    
    video = cv2.VideoCapture(filename)   # capturing the video from the given path
    frameRate = int(video.get(cv2.CAP_PROP_FPS)) #frame rate
    distance = (total_secs*frameRate)//amount #get a frame every distance seconds so that they are uniformly spaced out from each other
    while (video.isOpened()):
        frameID = video.get(1) #get the number of the current frame
        
        ret, frame = video.read()
        
        if (ret!=True):
            break
        if (frameID%distance==0):
            new_file = new_path+name+"_"+str(count)+".jpg"
            count+=1
            cv2.imwrite(new_file,frame)
    video.release()
    print("Frames extracted")
get_frames(game_file,3000,"/content/drive/MyDrive/Game-Movie/images/game/")
for movie in movie_files:
    get_frames(movie,1000,"/content/drive/MyDrive/Game-Movie/images/movies/")
#resize the images to 256x256
game_path = "/content/drive/MyDrive/Game-Movie/images/game/" #path for games and images
movies_path = "/content/drive/MyDrive/Game-Movie/images/movies/"
game_images = next(os.walk(game_path))[2]
movie_images = next(os.walk(movies_path))[2]

def fullNames(path,files):
    files = [path+x for x in files]
    return files
game_images = fullNames(game_path,game_images)
movie_images = fullNames(movies_path,movie_images)
def cropNsize(files,out_path):
    Path(out_path).mkdir(parents=True, exist_ok=True)
    for image in files:
        im = Image.open(image)
        name = image.split("/")[-1]
        im = im.crop((108, 0, 1173, 600)) 
        im = im.resize((256,256), Image.ANTIALIAS)
        im.save(out_path+name, 'JPEG', quality=90)
    print("All Images in folder resized")
cropNsize(game_images,"/content/drive/MyDrive/Game-Movie/images256/game/")
cropNsize(movie_images,"/content/drive/MyDrive/Game-Movie/images256/movies/")<jupyter_output>All Images in folder resized
All Images in folder resized<jupyter_text>1.1.1 Dataset<jupyter_code>def get_files(path):
    files = []
    for dirname, _, filenames in os.walk(path):
        for filename in filenames:
            files.append(os.path.join(dirname, filename))
    return files
            
class dataset(Dataset):
    def __init__(self, path,transform=None):
        self.transform = transform
        self.files = get_files(path)
        self.length = len(self.files)
    def __getitem__(self, index):
        image = Image.open(self.files[index])
        if self.transform:
            image = self.transform(image)
        return image
    def __len__(self):
        return self.length
game_path = "/content/drive/MyDrive/Game-Movie/images256/game/"
movies_path = "/content/drive/MyDrive/Game-Movie/images256/movies/"

trans = transforms.Compose([transforms.ToTensor(),])

all_game = dataset(path=game_path,transform=trans)
all_movie = dataset(path=movies_path,transform = trans)

game_train_dataset, game_test_dataset, game_validation_dataset = random_split(all_game,[2000,500,len(all_game)-2500], generator=torch.Generator().manual_seed(42))
movie_train_dataset, movie_test_dataset, movie_validation_dataset = random_split(all_movie,[2000,500,len(all_movie)-2500], generator=torch.Generator().manual_seed(42))

batch_size=4
game_train_loader = DataLoader(game_train_dataset,batch_size=batch_size, shuffle=True, num_workers=0)
movie_train_loader = DataLoader(movie_train_dataset,batch_size=batch_size, shuffle=True, num_workers=0)

Path("images/").mkdir(parents=True, exist_ok=True) #generate path to save produced images<jupyter_output><empty_output><jupyter_text>2.1 Frame-to-Frame Model  2.1.1 ResNet Architecture<jupyter_code>#code is adapted from: https://github.com/junyanz/pytorch-CycleGAN-and-pix2pix/blob/master/models/networks.py
use_bias=True
use_dropout=True
class Generator(nn.Module):
    def __init__(self,norm="instance"):
        super(Generator, self).__init__()
        #initial layer
        if norm=="instance":
            normalization = nn.InstanceNorm2d
        elif norm=="batch":
            normalization = nn.BatchNorm2d
        self.pad = nn.ReflectionPad2d(3)
        self.conv1 = nn.Conv2d(3,64,7,padding=0,bias=use_bias)
        self.batch64 = normalization(64)
        self.relu = nn.ReLU(inplace=True)
        
        #downsampling layers
        self.conv2 = nn.Conv2d(64, 128, 3, stride=2, padding=1,bias=use_bias)
        self.batch128 = normalization(128)

        self.conv3 = nn.Conv2d(128, 256, 3, stride=2, padding=1,bias=use_bias)
        self.batch256 = normalization(256)
        
        #Residual blocks
        self.resblock1 = ResidualBlock(256,norm="batch")
        self.resblock2 = ResidualBlock(256,norm="batch")
        self.resblock3 = ResidualBlock(256,norm="batch")
        self.resblock4 = ResidualBlock(256,norm="batch")
        self.resblock5 = ResidualBlock(256,norm="batch")
        self.resblock6 = ResidualBlock(256,norm="batch")
        self.resblock7 = ResidualBlock(256,norm="batch")
        self.resblock8 = ResidualBlock(256,norm="batch")
        self.resblock9 = ResidualBlock(256,norm="batch")
        
        #upsample
        self.convtrans1 = nn.ConvTranspose2d(256, 128, 3, stride=2, padding=1, output_padding=1,bias=use_bias)
        self.convtrans2 = nn.ConvTranspose2d(128, 64, 3, stride=2, padding=1, output_padding=1,bias=use_bias)
        #or to avoid artifacts:
        self.upsample1 = nn.Sequential(nn.Upsample(scale_factor = 2, mode='bilinear', align_corners=False), 
                      nn.ReflectionPad2d(1),                                               
                      nn.Conv2d(256, 128, kernel_size=3, stride=1, padding=0, bias=use_bias))
        self.upsample2 = nn.Sequential(nn.Upsample(scale_factor = 2, mode='bilinear', align_corners=False), 
                      nn.ReflectionPad2d(1),                                               
                      nn.Conv2d(128, 64, kernel_size=3, stride=1, padding=0, bias=use_bias))
        #output
        self.outconv = nn.Conv2d(64,3,7,padding=0)
        self.out = nn.Tanh()
        
    def forward(self,x):
        x = self.pad(x)
        x = self.conv1(x)
        x = self.relu(self.batch64(x))
        
        x = self.conv2(x)
        x = self.relu(self.batch128(x))
        
        x = self.conv3(x)
        x = self.relu(self.batch256(x))
        
        x = self.resblock1(x)
        x = self.resblock2(x)
        x = self.resblock3(x)
        x = self.resblock4(x)
        x = self.resblock5(x)
        x = self.resblock6(x)
        x = self.resblock7(x)
        x = self.resblock8(x)
        x = self.resblock9(x)

        x = self.convtrans1(x)
        x = self.relu(self.batch128(x))
        x = self.convtrans2(x)
        x = self.relu(self.batch64(x))
        
        
        # x = self.upsample1(x)
        # x = self.relu(self.batch128(x))
        # x = self.upsample2(x)
        # x = self.relu(self.batch64(x))

        x = self.pad(x)
        x = self.outconv(x)
        return self.out(x)
    
class ResidualBlock(nn.Module):
    def __init__(self, in_channels,norm="instance"):
        super(ResidualBlock, self).__init__()
        if norm=="instance":
            normalization = nn.InstanceNorm2d
        elif norm=="batch":
            normalization = nn.BatchNorm2d
        if use_dropout:
            self.res = nn.Sequential(nn.ReflectionPad2d(1),
                                 nn.Conv2d(in_channels, in_channels, 3,padding=0,bias=use_bias),
                                 normalization(in_channels),
                                 nn.ReLU(inplace=True),
                                 nn.Dropout(0.5),
                                 nn.ReflectionPad2d(1),
                                 nn.Conv2d(in_channels, in_channels, 3,padding=0,bias=use_bias),
                                 normalization(in_channels))
        else:
            self.res = nn.Sequential(nn.ReflectionPad2d(1),
                                 nn.Conv2d(in_channels, in_channels, 3,padding=0,bias=use_bias),
                                 nn.normalization(in_channels),
                                 nn.ReLU(inplace=True),
                                 nn.ReflectionPad2d(1),
                                 nn.Conv2d(in_channels, in_channels, 3,padding=0,bias=use_bias),
                                 nn.normalization(in_channels))

    def forward(self, x):
        return x + self.res(x)
gtm = Generator(norm="batch").to(device)
mtg = Generator(norm="batch").to(device)<jupyter_output><empty_output><jupyter_text>2.1.2 Discriminator<jupyter_code>class NLayerDiscriminator(nn.Module):
    def __init__(self, norm="instance"):
        super(NLayerDiscriminator, self).__init__()

        if norm=="instance":
            normalization = nn.InstanceNorm2d
        elif norm=="batch":
            normalization = nn.BatchNorm2d
        use_bias=False
        self.conv1 = nn.Conv2d(3,64,4,stride=2,padding=1)
        self.conv2 = nn.Conv2d(64,128,4,stride=2,padding=1,bias=use_bias)
        self.norm2 = normalization(128)
        self.conv3 = nn.Conv2d(128,256,4,stride=2,padding=1,bias=use_bias)
        self.norm3 = normalization(256)
        self.conv4 = nn.Conv2d(256,512,4,stride=2,padding=1,bias=use_bias)
        self.norm4 = normalization(512)
        self.conv5 = nn.Conv2d(512,1,4,stride=1,padding=1,bias=use_bias)
        self.out = nn.Sigmoid() 
        self.leakyrelu = nn.LeakyReLU(0.2, True)
        self.pad = nn.ZeroPad2d((1,0,1,0))

    def forward(self,x):
        x = self.leakyrelu(self.conv1(x))
        x = self.leakyrelu(self.norm2(self.conv2(x)))
        x = self.leakyrelu(self.norm3(self.conv3(x)))
        x = self.leakyrelu(self.norm4(self.conv4(x)))
        x = self.pad(x)
        x = self.conv5(x)
        x = self.out(x)
        return x

D_game = NLayerDiscriminator(norm="batch").to(device)
D_movie = NLayerDiscriminator(norm="batch").to(device)
summary(D_game,(3,256,256))
class BranchDiscriminator(nn.Module):
    def __init__(self,nlayer=True,pixel=True,norm = "instance"):
        super(BranchDiscriminator,self).__init__()
        if norm=="instance":
            normalization = nn.InstanceNorm2d
        elif norm=="batch":
            normalization = nn.BatchNorm2d
        use_bias=True
        self.nlayer=nn.Sequential(
            nn.Conv2d(3,64,4,stride=2,padding=1),
            nn.LeakyReLU(0.2, True),
            nn.Conv2d(64,128,4,stride=2,padding=1,bias=use_bias),
            normalization(128),
            nn.LeakyReLU(0.2, True),
            nn.Conv2d(128,256,4,stride=2,padding=1,bias=use_bias),
            normalization(256),
            nn.LeakyReLU(0.2, True),
            nn.Conv2d(256,512,4,stride=2,padding=1,bias=use_bias),
            normalization(512),
            nn.LeakyReLU(0.2, True),
            nn.ZeroPad2d((1, 0, 1, 0)),
            nn.Conv2d(512,1,4,stride=1,padding=1,bias=use_bias),
        )
        self.pixel=nn.Sequential(
            nn.Conv2d(3, 64, kernel_size=1, stride=1, padding=0),
            nn.LeakyReLU(0.2, True),
            nn.Conv2d(64, 128, kernel_size=1, stride=1, padding=0),
            normalization(128),
            nn.LeakyReLU(0.2, True),
            nn.Conv2d(128, 1, kernel_size=1, stride=1, padding=0),
            nn.MaxPool2d(16,16)
        )
        self.sigmoid = nn.Sigmoid()
    def forward(self,x):
        x_layer = self.nlayer(x)
        x_pixel = self.pixel(x)
        # print(x_pixel.shape)
        # x_layer_sigmoid =  self.sigmoid(x_layer)
        # x_pixel_sigmoid = self.sigmoid(x_pixel)
        # x_combined = self.sigmoid(x_layer_sigmoid+x_pixel_sigmoid)
        x_combined = self.sigmoid(x_layer+x_pixel)
        return x_combined
D_game = BranchDiscriminator(norm="batch").to(device)
D_movie = BranchDiscriminator(norm="batch").to(device)
summary(D_game,(3,256,256))
def init_weights(net, init_type='normal', init_gain=0.02):
    def init_func(m):  # define the initialization function
        classname = m.__class__.__name__
        if hasattr(m, 'weight') and (classname.find('Conv') != -1 or classname.find('Linear') != -1):
            if init_type == 'normal':
                init.normal_(m.weight.data, 0.0, init_gain)
            elif init_type == 'xavier':
                init.xavier_normal_(m.weight.data, gain=init_gain)
            elif init_type == 'kaiming':
                init.kaiming_normal_(m.weight.data, a=0, mode='fan_in')
            elif init_type == 'orthogonal':
                init.orthogonal_(m.weight.data, gain=init_gain)
            else:
                raise NotImplementedError('initialization method [%s] is not implemented' % init_type)
            if hasattr(m, 'bias') and m.bias is not None:
                init.constant_(m.bias.data, 0.0)
        elif classname.find('BatchNorm2d') != -1:  # BatchNorm Layer's weight is not a matrix; only normal distribution applies.
            init.normal_(m.weight.data, 1.0, init_gain)
            init.constant_(m.bias.data, 0.0)
    print('initialize network with %s' % init_type)
    net.apply(init_func)  # apply the initialization function <init_func>
    
init_weights(gtm)
init_weights(mtg)
init_weights(D_game)
init_weights(D_movie)
# init_weights(D_game_1x)
# init_weights(D_movie_1x)
lr=0.0002
beta1=0.5
G_optimiser = torch.optim.Adam(itertools.chain(gtm.parameters(), mtg.parameters()), lr=lr, betas=(beta1, 0.999))
D_optimiser = torch.optim.Adam(itertools.chain(D_game.parameters(), D_movie.parameters()), lr=lr/10, betas=(beta1, 0.999))
# D_1x_optimiser = torch.optim.Adam(itertools.chain(D_game_1x.parameters(), D_movie_1x.parameters()), lr=lr/2, betas=(beta1, 0.999))
criterion_idt = nn.L1Loss()
criterion_cycle = nn.L1Loss()
gan_loss = nn.MSELoss()
# gan_loss = nn.BCEWithLogitsLoss()
patch_loss = nn.BCELoss()
# G_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(G_optimiser, mode='min', factor=0.2, threshold=0.01, patience=5)
# D_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(D_optimiser, mode='min', factor=0.2, threshold=0.01, patience=5)
# D_1x_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(D_1x_optimiser, mode='min', factor=0.2, threshold=0.01, patience=5)

#define the training hyper parameters 
epoch = 0
max_epoch = 50
lambdaidt = 10
lambda_gtm = 10
lambda_mtg = 10<jupyter_output><empty_output><jupyter_text>2.1.3 Training<jupyter_code>wandb.init(project="GameToMovie",name="r")
real_label = torch.ones((batch_size,1,16,16),device=device)
fake_label = torch.zeros((batch_size,1,16,16),device=device)
while epoch<max_epoch:
    
    for _ in range(50):
        game_images = next(game_iter)
        movie_images = next(movie_iter)
        game_images = game_images.t`o(device)
        movie_images = movie_images.to(device)
        
        fake_movie = gtm(game_images)
        fake_game = mtg(movie_images)
        cycle_game = mtg(fake_movie) #cycle back
        cycle_movie = gtm(fake_game) #cycle back

        # real data label is 1, fake data label is 0.
        # real_label = torch.full((batch_size, 1), 1, device=device, dtype=torch.float32)
        # fake_label = torch.full((batch_size, 1), 0, device=device, dtype=torch.float32)

        #train gtm and mtg 
        #set the grads to zero
        # real_label = Variable(Tensor(np.ones((real_A.size(0), *D_A.output_shape))), requires_grad=False)
        # fake_label = Variable(Tensor(np.zeros((real_A.size(0), *D_A.output_shape))), requires_grad=False)
        #generate movie from games and check Discriminator
        for param in D_game.parameters():
            param.requires_grad = False
        for param in D_movie.parameters():
            param.requires_grad = False
        # for param in D_game_1x.parameters():
        #     param.requires_grad = False
        # for param in D_movie_1x.parameters():
        #     param.requires_grad = False
        G_optimiser.zero_grad()


        # loss_gtm_1x = patch_loss(D_movie_1x(gen_movie),real_label_256)
        # loss_mtg_1x = patch_loss(D_game_1x(gen_game),real_label_256)
        loss_gtm = l2_loss(D_movie.discriminate(fake_movie),real_label)
                #generate game from movies and check Disc
        loss_mtg = l2_loss(D_game.discriminate(fake_game),real_label)        

        
        cycle_loss_GTM = criterion_cycle(cycle_game,game_images)*lambda_gtm
        cycle_loss_MTG = criterion_cycle(cycle_movie,movie_images)*lambda_mtg

        #identity loss
        #passing a movie image to GameToMovie generator should give the identity
        if lambdaidt>0:
            idt_gtm = gtm(movie_images)
            loss_idt_gtm = criterion_idt(idt_gtm,movie_images)*lambda_mtg*lambdaidt
            idt_mtg = mtg(game_images)
            loss_idt_mtg = criterion_idt(idt_mtg,game_images)*lambda_gtm*lambdaidt
        else:
            loss_idt_gtm = 0
            loss_idt_mtg = 0
     
        loss_G = loss_gtm+loss_mtg+loss_idt_gtm+loss_idt_mtg+cycle_loss_GTM+cycle_loss_MTG#+loss_gtm_1x+loss_mtg_1x
        loss_G.backward()
        G_optimiser.step()
        
        
        for param in D_game.parameters():
            param.requires_grad = True
        for param in D_movie.parameters():
            param.requires_grad = True     
        # for param in D_game_1x.parameters():
        #     param.requires_grad = True
        # for param in D_movie_1x.parameters():
        #     param.requires_grad = True  
        D_optimiser.zero_grad()
        
        loss_real_game = l2_loss(D_game.discriminate(game_images),real_label)
        loss_real_movie = l2_loss(D_movie.discriminate(movie_images),real_label)

        loss_fake_game = l2_loss(D_game.discriminate(fake_game.detach()),fake_label)
        loss_fake_movie = l2_loss(D_movie.discriminate(fake_movie.detach()),fake_label)
        
        # loss_real_game_1x = patch_loss(D_game_1x(game_images),real_label)
        # loss_real_movie_1x = patch_loss(D_movie_1x(movie_images),real_label)
        # loss_fake_game_1x = patch_loss(D_game_1x(gen_game),fake_label)
        # loss_fake_movie_1x = patch_loss(D_movie_1x(gen_movie),fake_label)
        
        loss_D_game = (loss_real_game+loss_fake_game)*0.5
        loss_D_movie = (loss_real_movie+loss_fake_movie)*0.5
        
        # loss_D_game_1x = (loss_real_game_1x+loss_fake_game_1x)*0.5
        # loss_D_movie_1x = (loss_real_movie_1x+loss_fake_movie_1x)*0.5
       
        loss_D_game.backward()
        loss_D_movie.backward()
        # loss_D_game_1x.backward()
        # loss_D_movie_1x.backward()

        D_optimiser.step()
    wandb.log({ "Generator Loss":loss_G.item(),"Game Discriminator Loss":loss_D_game.item(),"Movie Discriminator Loss":loss_D_movie.item(),"Cycle Loss":(cycle_loss_GTM+cycle_loss_MTG).item(),
               "Identity Loss":(loss_idt_gtm+loss_idt_mtg).item(),"Gen Learning Rate":G_optimiser.param_groups[0]['lr'],"Disc Learning Rate":D_optimiser.param_groups[0]['lr']})
    epoch+=1
    G_scheduler.step(0)
    D_scheduler.step(0)
    # D_1x_scheduler.step(0)
    plt.figure(figsize = (30,30))
    imgList = []
    for i in range(4):
        imgList.extend([game_images[i],idt_mtg[i],fake_movie[i],cycle_game[i],movie_images[i],idt_gtm[i],fake_game[i],cycle_movie[i]])
    grid_img = torchvision.utils.make_grid(imgList, nrow=4)
    plt.imshow(grid_img.cpu().data.permute(0,2,1).contiguous().permute(2,1,0))
    name ="images/epoch"+"_"+str(epoch)+".png"
    plt.savefig(name)
    wandb.save(name)

def save_models(epoch):
    torch.save(D_game,"disc_game{}.h5".format(epoch))
    torch.save(D_movie,"disc_movie{}.h5".format(epoch))
    torch.save(gtm,"gtmGen{}.h5".format(epoch))
    torch.save(mtg,"mtgGen{}.h5".format(epoch))
    wandb.save("disc_game{}.h5".format(epoch))
    wandb.save("disc_movie{}.h5".format(epoch))
    wandb.save("gtmGen{}.h5".format(epoch))
    wandb.save("mtgGen{}.h5".format(epoch))
def cycle(iterable):
    while True:
        for x in iterable:
            yield x
game_iter = cycle(game_train_loader)
movie_iter = cycle(movie_train_loader)<jupyter_output><empty_output><jupyter_text>3 Face ModelFor this part we use the already extracted frames from the movies instead of going through the original videos. We perform this in the original resolution and then resize the images.  3.1 Face Extraction<jupyter_code>from facenet_pytorch import MTCNN
mtcnn = MTCNN(keep_all=False, image_size=256, margin=50,post_process=True,min_face_size=100,device=device)
def getFaces(images,path,out_path):
    Path(out_path).mkdir(parents=True, exist_ok=True)
    for image in images:
        filename = path+image
        new_name = out_path+image
        im = Image.open(filename)
        save_paths = new_name
        mtcnn(im, save_path=save_paths)
    print("All Faces Extracted")
getFaces(game_images,game_path,"/content/drive/MyDrive/Game-Movie/face_images/game/")
getFaces(movie_images,movies_path,"/content/drive/MyDrive/Game-Movie/face_images/movies/")<jupyter_output><empty_output><jupyter_text>Convert the dataset to h5 format for faster IO<jupyter_code>game_path = "/content/drive/MyDrive/Game-Movie/face_images/game/"
movie_path = "/content/drive/MyDrive/Game-Movie/face_images/movies/"
game_fileName = "/content/drive/MyDrive/Game-Movie/face_images/h5/game_faces.h5"
movie_fileName = "/content/drive/MyDrive/Game-Movie/face_images/h5/movie_faces.h5"
def converth5(path,output_path):
    #get names of the images and randomize
    # Path(output_path).mkdir(parents=True, exist_ok=True)
    files = [name for name in os.listdir(path)]
    random.shuffle(files)#inplace operation
    #define some variable according to the size
    size = 256
    numTrain = int(0.15*len(files))
    numTest = len(files)-numTrain
    #create the h5 files
    f = h5py.File(output_path, "w")
    with f as out:
        out.create_dataset("X_train",(numTrain,size,size,3),dtype='u1')   
        out.create_dataset("X_test",(numTest,size,size,3),dtype='u1')
        #populate the h5 file
    f = h5py.File(output_path, "a")
    with f as out:
        for index,img_name in enumerate(files[:numTrain]):
            img = Image.open(path+img_name)      
            out['X_train'][index] = np.asarray(img)
        for index,img_name in enumerate(files[numTrain:]):
            img = Image.open(path+img_name)      
            out['X_test'][index] = np.asarray(img)
    print("Conversion Complete")
converth5(game_path,game_fileName)
converth5(movie_path,movie_fileName)<jupyter_output>Conversion Complete
Conversion Complete<jupyter_text>Dataset and Training<jupyter_code>class dataset_h5(Dataset):
    def __init__(self, in_file, transform=None,train=True):
        super(dataset_h5, self).__init__()
        self.train = train
        self.file = h5py.File(in_file, 'r')
        self.transform = transform
 
    def __getitem__(self, index):
        if self.train:
            x = self.file['X_train'][index]
        else:
            x = self.file['X_test'][index]
        # Preprocessing each image
        if self.transform is not None:
            x = self.transform(x)   
        return x #x is original and p is inpainted
 
    def __len__(self):
        if self.train:
            return self.file['X_train'].shape[0]
        else:
            return self.file['X_test'].shape[0]

img_width = 256
game_fileName = "/content/drive/MyDrive/Game-Movie/face_images/h5/game_faces.h5"
movie_fileName = "/content/drive/MyDrive/Game-Movie/face_images/h5/movie_faces.h5"
img_transform = transforms.Compose([transforms.ToTensor(),transforms.RandomRotation(45),transforms.RandomHorizontalFlip(p=0.4)])
test_transform = transforms.Compose([transforms.ToTensor()])

game_trainset = dataset_h5(game_fileName,transform=img_transform)
movie_trainset = dataset_h5(movie_fileName,transform=img_transform)

game_testset = dataset_h5(game_fileName,transform=test_transform,train=False)
movie_testset = dataset_h5(movie_fileName,transform=test_transform,train=False)
Path("outputs/").mkdir(parents=True, exist_ok=True) #generate path to save produced images

batch_size=4

game_train_loader = DataLoader(game_trainset,batch_size=batch_size, shuffle=True, num_workers=0,drop_last=True)
movie_train_loader = DataLoader(movie_trainset,batch_size=batch_size, shuffle=True, num_workers=0,drop_last=True)

game_test_loader = DataLoader(game_testset,batch_size=batch_size, shuffle=True, num_workers=0,drop_last=True)
movie_test_loader = DataLoader(movie_testset,batch_size=batch_size, shuffle=True, num_workers=0,drop_last=True)

real_label = torch.ones((batch_size,1,16,16),device=device)
fake_label = torch.zeros((batch_size,1,16,16),device=device)
def cycle(iterable):
    while True:
        for x in iterable:
            yield x
game_iter = cycle(game_train_loader)
movie_iter = cycle(movie_train_loader)

game_test_iter = cycle(game_test_loader)
movie_test_iter = cycle(movie_test_loader)
wandb.init(project="GameToMovie",resume=True)<jupyter_output><empty_output><jupyter_text>code to load model an continue training<jupyter_code>run_path="23w33gvf"
epoch=50
D_game = wandb.restore("disc_game{}.h5".format(epoch),run_path="ccm/GameToMovie/{}".format(run_path))
D_movie = wandb.restore("disc_movie{}.h5".format(epoch),run_path="ccm/GameToMovie/{}".format(run_path))
gtm = wandb.restore("gtmGen{}.h5".format(epoch),run_path="ccm/GameToMovie/{}".format(run_path))
mtg = wandb.restore("mtgGen{}.h5".format(epoch),run_path="ccm/GameToMovie/{}".format(run_path))
D_game = torch.load(D_game.name).to(device)
D_movie = torch.load(D_movie.name).to(device)
gtm = torch.load(gtm.name).to(device)
mtg = torch.load(mtg.name).to(device)

while epoch<50:
    
    for _ in range(75):
        game_images = next(game_iter)
        movie_images = next(movie_iter)
        game_images = game_images.to(device)
        movie_images = movie_images.to(device)
        
        fake_movie = gtm(game_images)
        fake_game = mtg(movie_images)
        cycle_game = mtg(fake_movie) #cycle back
        cycle_movie = gtm(fake_game) #cycle back

        # real data label is 1, fake data label is 0.
        #set the grads to zero
        #generate movie from games and check Discriminator
        for param in D_game.parameters():
            param.requires_grad = False
        for param in D_movie.parameters():
            param.requires_grad = False
        # for param in D_game_1x.parameters():
        #     param.requires_grad = False
        # for param in D_movie_1x.parameters():
        #     param.requires_grad = False
        G_optimiser.zero_grad()


        # loss_gtm_1x = patch_loss(D_movie_1x(fake_movie),real_label_256)
        # loss_mtg_1x = patch_loss(D_game_1x(fake_game),real_label_256)
        loss_gtm = gan_loss(D_movie(fake_movie),real_label)
                #generate game from movies and check Disc
        loss_mtg = gan_loss(D_game(fake_game),real_label)        

        
        cycle_loss_GTM = criterion_cycle(cycle_game,game_images)*lambda_gtm
        cycle_loss_MTG = criterion_cycle(cycle_movie,movie_images)*lambda_mtg

        #identity loss
        #passing a movie image to GameToMovie generator should give the identity
        if lambdaidt>0:
            idt_gtm = gtm(movie_images)
            loss_idt_gtm = criterion_idt(idt_gtm,movie_images)*lambdaidt
            idt_mtg = mtg(game_images)
            loss_idt_mtg = criterion_idt(idt_mtg,game_images)*lambdaidt
        else:
            loss_idt_gtm = 0
            loss_idt_mtg = 0
     
        loss_G = loss_gtm+loss_mtg+loss_idt_gtm+loss_idt_mtg+cycle_loss_GTM+cycle_loss_MTG#+loss_gtm_1x+loss_mtg_1x
        loss_G.backward()
        G_optimiser.step()
        
        
        for param in D_game.parameters():
            param.requires_grad = True
        for param in D_movie.parameters():
            param.requires_grad = True     
        # for param in D_game_1x.parameters():
        #     param.requires_grad = True
        # for param in D_movie_1x.parameters():
        #     param.requires_grad = True  
        D_optimiser.zero_grad()
        
        loss_real_game = gan_loss(D_game(game_images.detach()),real_label)
        loss_real_movie = gan_loss(D_movie(movie_images.detach()),real_label)

        loss_fake_game = gan_loss(D_game(fake_game.detach()),fake_label)
        loss_fake_movie = gan_loss(D_movie(fake_movie.detach()),fake_label)
        
        # loss_real_game_1x = patch_loss(D_game_1x(game_images),real_label_256)
        # loss_real_movie_1x = patch_loss(D_movie_1x(movie_images),real_label_256)
        # loss_fake_game_1x = patch_loss(D_game_1x(fake_game),fake_label_256)
        # loss_fake_movie_1x = patch_loss(D_movie_1x(fake_movie),fake_label_256)
        
        loss_D_game = (loss_real_game+loss_fake_game)*0.5
        loss_D_movie = (loss_real_movie+loss_fake_movie)*0.5
        
        # loss_D_game_1x = (loss_real_game_1x+loss_fake_game_1x)*0.5
        # loss_D_movie_1x = (loss_real_movie_1x+loss_fake_movie_1x)*0.5
       
        loss_D_game.backward()
        loss_D_movie.backward()
        # loss_D_game_1x.backward()
        # loss_D_movie_1x.backward()

        D_optimiser.step()
    wandb.log({ "Generator Loss":loss_G.item(),"Game Discriminator Loss":loss_D_game.item(),"Movie Discriminator Loss":loss_D_movie.item(),"Cycle Loss":(cycle_loss_GTM+cycle_loss_MTG).item(),
               "Identity Loss":(loss_idt_gtm+loss_idt_mtg).item()})
    #,"Game Pixel Loss":loss_D_game_1x.item(),"Movie Pixel Loss":loss_D_movie_1x.item()})
    epoch+=1
    # G_scheduler.step(0)
    # D_scheduler.step(0)
    # D_1x_scheduler.step(0)
    if epoch % 5 ==0:
        plt.figure(figsize = (30,30))
        imgList = []
        for i in range(batch_size):
            imgList.extend([game_images[i],idt_mtg[i],fake_movie[i],cycle_game[i],movie_images[i],idt_gtm[i],fake_game[i],cycle_movie[i]])
        grid_img = torchvision.utils.make_grid(imgList, nrow=4)
        plt.imshow(grid_img.cpu().data.permute(0,2,1).contiguous().permute(2,1,0))
        name ="images/epoch"+"_"+str(epoch)+".png"
        save_models(epoch)
        plt.savefig(name)
        wandb.save(name)

torch.save(gtm.state_dict(),"/content/drive/MyDrive/Game-Movie/Models/game_to_movieGen.pt")
torch.save(mtg.state_dict(),"/content/drive/MyDrive/Game-Movie/Models/movie_to_gameGen.pt")
torch.save(D_game.state_dict(),"/content/drive/MyDrive/Game-Movie/Models/D_game.pt")
torch.save(D_movie.state_dict(),"/content/drive/MyDrive/Game-Movie/Models/D_movie.pt")
gtm.load_state_dict(torch.load("/content/drive/MyDrive/Game-Movie/Models/game_to_movieGen.pt"))
mtg.load_state_dict(torch.load("/content/drive/MyDrive/Game-Movie/Models/movie_to_gameGen.pt"))
D_game.load_state_dict(torch.load("/content/drive/MyDrive/Game-Movie/Models/D_game.pt"))
D_movie.load_state_dict(torch.load("/content/drive/MyDrive/Game-Movie/Models/D_movie.pt"))<jupyter_output><empty_output><jupyter_text>function to generate results and images<jupyter_code>with torch.no_grad():
    game_images = next(game_test_iter)
    movie_images = next(movie_test_iter)
    game_images = game_images.to(device)
    movie_images = movie_images.to(device)
    fake_movie = gtm(game_images)
    fake_game = mtg(movie_images.detach())
    idt_game = mtg(game_images.detach())
    idt_movie = gtm(movie_images.detach())
    cycle_game = mtg(fake_movie.detach())
    cycle_movie = gtm(fake_game.detach())

    imgList = []
    plt.figure(figsize = (30,30))
    for i in range(4):
        imgList.extend([game_images[i],idt_game[i],fake_movie[i],cycle_game[i],movie_images[i],idt_movie[i],fake_game[i],cycle_movie[i]])
    grid_img = torchvision.utils.make_grid(imgList, nrow=4)
    plt.imshow(grid_img.cpu().data.permute(0,2,1).contiguous().permute(2,1,0))
    name ="images/FINAL6.png"
    plt.savefig(name)
    wandb.save(name)<jupyter_output><empty_output><jupyter_text>4 Real World Application<jupyter_code>tr = transforms.ToTensor()
untr = transforms.ToPILImage()
!pip install facenet-pytorch
from facenet_pytorch import MTCNN
mtcnn = MTCNN(keep_all=False, image_size=256, margin=50,post_process=True,min_face_size=100,device=device)
import time 
count=0
start = 270*30 #start at 4:30
max_count = start + 300 #10 seconds at 30 fps

save_loc = "/content/drive/MyDrive/Game-Movie/FinalOut/"
Path(save_loc).mkdir(parents=True, exist_ok=True)

video = cv2.VideoCapture(game_file)   # capturing the video from the given path
frameRate = int(video.get(cv2.CAP_PROP_FPS)) #frame rate


while (video.isOpened()): 
    ret, frame = video.read()
    count+=1
    if (ret!=True):
        break
    if count>=start: 
        
        im_pil = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
        im = im_pil.crop((108, 0, 1173, 600)) 


        box,_ = mtcnn.detect(im)
        cropped = im.crop(box[0].round())
        resized = cropped.resize((256,256))

        with torch.no_grad():
            translated_im = gtm(tr(resized).unsqueeze_(0).to(device))
            img = plt.imshow(translated_im.squeeze().permute(1,2,0).cpu())
            plt.axis("off")
            new_file = save_loc+"_"+str(count)+".jpg"
            plt.savefig(new_file, bbox_inches='tight')


            # final = untr(translated_im.squeeze())
        # final.save(new_file)

    if count==max_count:
        break
video.release()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1240265192090559, 'fraction_numerical': 0.02622969804001042, 'mean_word_length': 3.2180974477958237, 'pattern_counts': {'":': 23, '<': 46, '<?xml version=': 0, '>': 47, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 17, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '333948', 'license': 'MIT', 'n_tokens_mistral': 12711, 'n_tokens_neox': 12070, 'n_words': 1772, 'path': 'Game_Movie (2).ipynb', 'repo_name': 'ChristosCM/Game-Movie-CycleGAN', 'size': '34541'}