<jupyter_start><jupyter_text>API Server<jupyter_code>from IPython.display import Javascript
display(Javascript('''google.colab.output.setIframeHeight(0, true, {maxHeight: 200})'''))

!pip install ninja opensimplex gradio moviepy==1.0.3 tts pyrebase
!apt install ffmpeg
!pip install git+https://github.com/1adrianb/face-alignment@v1.0.1
!pip install fbpca boto3 requests==2.23.0 #urllib3==1.25.11
# !git submodule update --init --recursive
!python -c "import nltk; nltk.download('wordnet')"
from google.colab import drive
drive.mount('/content/drive')
%cd /content/drive/MyDrive/Colab/fyp
try: # set up path
    import sys
    sys.path.append('/content/drive/MyDrive/Colab/fyp/ganspace')
    sys.path.append('/content/drive/MyDrive/Colab/fyp/first-order-model')
    sys.path.append('/content/drive/MyDrive/Colab/fyp/iPERCore')
    sys.path.append('/content/drive/MyDrive/Colab/fyp/Wav2Lip')
    print('Paths added')
except Exception as e:
    print(e)
    pass
#@title Install impersonator dependencies
# set CUDA_HOME, here we use CUDA 10.1
from IPython.display import Javascript
display(Javascript('''google.colab.output.setIframeHeight(0, true, {maxHeight: 200})'''))

import os
os.environ["CUDA_HOME"] = "/usr/local/cuda-10.1"

!echo $CUDA_HOME
%cd /content/drive/MyDrive/Colab/fyp/iPERCore/
!python setup.py develop
%cd /content/
#@title Define Cropping Functions

import os
from moviepy.video.io.ffmpeg_tools import ffmpeg_extract_subclip
import face_alignment
import imageio
import numpy as np
from skimage.transform import resize
import matplotlib.pyplot as plt
import matplotlib.animation as animation
from IPython.display import HTML, clear_output
import cv2
import shutil
import warnings
warnings.filterwarnings("ignore")
 
hog = cv2.HOGDescriptor()
hog.setSVMDetector(cv2.HOGDescriptor_getDefaultPeopleDetector())

fa = face_alignment.FaceAlignment(face_alignment.LandmarksType._2D, flip_input=True,
                                      device='cuda')

image_size = 512

def create_bounding_box(target_landmarks, expansion_factor=1):
    target_landmarks = np.array(target_landmarks)
    x_y_min = target_landmarks.reshape(-1, 68, 2).min(axis=1)
    x_y_max = target_landmarks.reshape(-1, 68, 2).max(axis=1)
    expansion_factor = (expansion_factor-1)/2
    bb_expansion_x = (x_y_max[:, 0] - x_y_min[:, 0]) * expansion_factor
    bb_expansion_y = (x_y_max[:, 1] - x_y_min[:, 1]) * expansion_factor
    x_y_min[:, 0] -= bb_expansion_x
    x_y_max[:, 0] += bb_expansion_x
    x_y_min[:, 1] -= bb_expansion_y
    x_y_max[:, 1] += bb_expansion_y
    return np.hstack((x_y_min, x_y_max-x_y_min))

def fix_dims(im):
    if im.ndim == 2:
        im = np.tile(im[..., None], [1, 1, 3])
    return im[...,:3]

def get_crop(im, center_face=True, crop_face=True, expansion_factor=1, landmarks=None):
    im = fix_dims(im)
    if (center_face or crop_face) and not landmarks:
        landmarks = fa.get_landmarks_from_image(im)
    if (center_face or crop_face) and landmarks:
        rects = create_bounding_box(landmarks, expansion_factor=expansion_factor)
        x0,y0,w,h = sorted(rects, key=lambda x: x[2]*x[3])[-1]
        if crop_face:
            s = max(h, w)
            x0 += (w-s)//2
            x1 = x0 + s
            y0 += (h-s)//2
            y1 = y0 + s
        else:
            img_h,img_w = im.shape[:2]
            img_s = min(img_h,img_w)
            x0 = min(max(0, x0+(w-img_s)//2), img_w-img_s)
            x1 = x0 + img_s
            y0 = min(max(0, y0+(h-img_s)//2), img_h-img_s)
            y1 = y0 + img_s            
    else:
        h,w = im.shape[:2]
        s = min(h,w)
        x0 = (w-s)//2
        x1 = x0 + s
        y0 = (h-s)//2
        y1 = y0 + s
    return int(x0),int(x1),int(y0),int(y1)

def pad_crop_resize(im, x0=None, x1=None, y0=None, y1=None, new_h=256, new_w=256):
    im = fix_dims(im)
    h,w = im.shape[:2]
    if x0 is None:
      x0 = 0
    if x1 is None:
      x1 = w
    if y0 is None:
      y0 = 0
    if y1 is None:
      y1 = h
    if x0<0 or x1>w or y0<0 or y1>h:
        im = np.pad(im, pad_width=[(max(-y0,0),max(y1-h,0)),(max(-x0,0),max(x1-w,0)),(0,0)], mode='edge')
    return resize(im[max(y0,0):y1-min(y0,0),max(x0,0):x1-min(x0,0)], (new_h, new_w))

def get_crop_body(im, center_body=True, crop_body=True, expansion_factor=1, rects=None):
    im = fix_dims(im)
    if (center_body or crop_body) and rects is None:
        rects, _ = hog.detectMultiScale(im, winStride=(4, 4),padding=(8,8), scale=expansion_factor)
    if (center_body or crop_body) and rects is not None and len(rects):
        x0,y0,w,h = sorted(rects, key=lambda x: x[2]*x[3])[-1]
        if crop_body:
            x0 += w//2-h//2
            x1 = x0+h
            y1 = y0+h
        else:
            img_h,img_w = im.shape[:2]
            x0 += (w-img_h)//2
            x1 = x0+img_h
            y0 = 0
            y1 = img_h
    else:
        h,w = im.shape[:2]
        x0 = (w-h)//2
        x1 = (w+h)//2
        y0 = 0
        y1 = h
    return int(x0),int(x1),int(y0),int(y1)

def crop_resize(im, size, crop=False):
  if im.shape[:2] == size:
    return im
  if size[0]<im.shape[0] or size[1]<im.shape[1]:
    interp = cv2.INTER_AREA
  else:
    interp = cv2.INTER_CUBIC
  if not crop:
    return np.clip(cv2.resize(im, size[::-1], interpolation=interp),0,1)
  ratio = max(size[0]/im.shape[0], size[1]/im.shape[1])
  im = np.clip(cv2.resize(im, (int(np.ceil(im.shape[1]*ratio)), int(np.ceil(im.shape[0]*ratio))), interpolation=interp),0,1)
  return im[(im.shape[0]-size[0])//2:(im.shape[0]-size[0])//2+size[0], (im.shape[1]-size[1])//2:(im.shape[1]-size[1])//2+size[1]]

#@title Define GANSpace functions
from ipywidgets import fixed

# Taken from https://github.com/alexanderkuk/log-progress
def log_progress(sequence, every=1, size=None, name='Items'):
    from ipywidgets import IntProgress, HTML, VBox
    from IPython.display import display

    is_iterator = False
    if size is None:
        try:
            size = len(sequence)
        except TypeError:
            is_iterator = True
    if size is not None:
        if every is None:
            if size <= 200:
                every = 1
            else:
                every = int(size / 200)     # every 0.5%
    else:
        assert every is not None, 'sequence is iterator, set every'

    if is_iterator:
        progress = IntProgress(min=0, max=1, value=1)
        progress.bar_style = 'info'
    else:
        progress = IntProgress(min=0, max=size, value=0)
    label = HTML()
    box = VBox(children=[label, progress])
    display(box)

    index = 0
    try:
        for index, record in enumerate(sequence, 1):
            if index == 1 or index % every == 0:
                if is_iterator:
                    label.value = '{name}: {index} / ?'.format(
                        name=name,
                        index=index
                    )
                else:
                    progress.value = index
                    label.value = u'{name}: {index} / {size}'.format(
                        name=name,
                        index=index,
                        size=size
                    )
            yield record
    except:
        progress.bar_style = 'danger'
        raise
    else:
        progress.bar_style = 'success'
        progress.value = index
        label.value = "{name}: {index}".format(
            name=name,
            index=str(index or '?')
        )

def name_direction(sender):
  if not text.value:
    print('Please name the direction before saving')
    return
    
  if num in named_directions.values():
    target_key = list(named_directions.keys())[list(named_directions.values()).index(num)]
    print(f'Direction already named: {target_key}')
    print(f'Overwriting... ')
    del(named_directions[target_key])
  named_directions[text.value] = [num, start_layer.value, end_layer.value]
  save_direction(random_dir, text.value)
  for item in named_directions:
    print(item, named_directions[item])

def save_direction(direction, filename):
  filename += ".npy"
  np.save(filename, direction, allow_pickle=True, fix_imports=True)
  print(f'Latent direction saved as {filename}')

def project_image(target_image, step, model, center=False, seed=303, use_clip=True, video=False):
    if model == 'portrait':
        pkl = '/content/drive/MyDrive/Colab/models/portrait-001000.pkl'
    elif model == 'character':
        pkl = '/content/drive/MyDrive/Colab/models/character-002600.pkl'
    elif model == 'model':
        pkl = '/content/drive/MyDrive/Colab/models/modelv4-001600.pkl'
    elif model == 'lookbook':
        pkl = '/content/drive/MyDrive/Colab/models/lookbook-001800.pkl'
    else:
        print('Model PKL file does not exists')

    !python /content/drive/MyDrive/Colab/fyp/stylegan2-ada-pytorch/pbaylies_projector.py --network={pkl} --outdir=/content/projector_output/ --target-image={target_image} --num-steps={step} --use-clip={use_clip} --use-center={center} --seed={seed} --save-video={video}

    projected_w = np.load('/content/projector_output/projected_w.npz')['w'].squeeze()
    return projected_w

def mix_w(w1, w2, content, style):
    for i in range(0,5):
        w2[i] = w1[i] * (1 - content) + w2[i] * content

    for i in range(5, 16):
        w2[i] = w1[i] * (1 - style) + w2[i] * style
    
    return w2

def display_sample_pytorch(seed=None, truncation=0.5, directions=None, distances=None, scale=1, start=0, end=14, w=None, disp=True, save=None):
    # blockPrint()
    model.truncation = truncation
    if w is None:
        w = model.sample_latent(1, seed=seed).detach().cpu().numpy()
        w = [w]*model.get_max_latents() # one per layer
    else:
        w = [np.expand_dims(x, 0) for x in w]
    
    if directions != None and distances != None:
        for l in range(start, end):
          for i in range(len(directions)):
            w[l] = w[l] + directions[i] * distances[i] * scale
    
    torch.cuda.empty_cache()
    #save image and display
    out = model.sample_np(w)
    final_im = Image.fromarray((out * 255).astype(np.uint8)).resize((500,500),Image.LANCZOS)
    
    
    if save is not None:
      if disp == False:
        print(save)
      final_im.save(f'out/{seed}_{save:05}.png')
    if disp:
      display(final_im)
    
    return final_im

def generate_mov(seed, truncation, direction_vec, scale, layers, n_frames, out_name = 'out', noise_spec = None, loop=True):
  """Generates a mov moving back and forth along the chosen direction vector"""
  # Example of reading a generated set of images, and storing as MP4.
  %mkdir out
  movieName = f'out/{out_name}.mp4'
  offset = -10
  step = 20 / n_frames
  imgs = []
  for i in log_progress(range(n_frames), name = "Generating frames"):
    print(f'\r{i} / {n_frames}', end='')
    w = model.sample_latent(1, seed=seed).cpu().numpy()

    model.truncation = truncation
    w = [w]*model.get_max_latents() # one per layer
    for l in layers:
      if l <= model.get_max_latents():
          w[l] = w[l] + direction_vec * offset * scale

    #save image and display
    out = model.sample_np(w)
    final_im = Image.fromarray((out * 255).astype(np.uint8))
    imgs.append(out)
    #increase offset
    offset += step
  if loop:
    imgs += imgs[::-1]
  with imageio.get_writer(movieName, mode='I') as writer:
    for image in log_progress(list(imgs), name = "Creating animation"):
        writer.append_data(img_as_ubyte(image))
#@title Load Model for GANSpace
selected_model = 'portrait'#@param ["portrait", "character", "model", "lookbook"]

# Load model
from IPython.utils import io
import torch
import PIL
import numpy as np
import ipywidgets as widgets
from PIL import Image
import imageio
from models import get_instrumented_model
from decomposition import get_or_compute
from config import Config
from skimage import img_as_ubyte

# Speed up computation
torch.autograd.set_grad_enabled(False)
torch.backends.cudnn.benchmark = True

# Specify model to use
config = Config(
  model='StyleGAN2',
  layer='style',
  output_class=selected_model,
  components=80,
  use_w=True,
  batch_size=5_000, # style layer quite small
)

inst = get_instrumented_model(config.model, config.output_class,
                              config.layer, torch.device('cuda'), use_w=config.use_w)

path_to_components = get_or_compute(config, inst)

model = inst.model

comps = np.load(path_to_components)
lst = comps.files
latent_dirs = []
latent_stdevs = []

load_activations = False

for item in lst:
    if load_activations:
      if item == 'act_comp':
        for i in range(comps[item].shape[0]):
          latent_dirs.append(comps[item][i])
      if item == 'act_stdev':
        for i in range(comps[item].shape[0]):
          latent_stdevs.append(comps[item][i])
    else:
      if item == 'lat_comp':
        for i in range(comps[item].shape[0]):
          latent_dirs.append(comps[item][i])
      if item == 'lat_stdev':
        for i in range(comps[item].shape[0]):
          latent_stdevs.append(comps[item][i])

def load_model(output_class):
    global config
    global inst
    global model
    config = Config(
    model='StyleGAN2',
    layer='style',
    output_class=output_class,
    components=80,
    use_w=True,
    batch_size=5_000, # style layer quite small
  )

    inst = get_instrumented_model(config.model, config.output_class,
                                  config.layer, torch.device('cuda'), use_w=config.use_w)

    path_to_components = get_or_compute(config, inst)

    model = inst.model

    comps = np.load(path_to_components)
    lst = comps.files
    latent_dirs = []
    latent_stdevs = []

    load_activations = False

    for item in lst:
        if load_activations:
          if item == 'act_comp':
            for i in range(comps[item].shape[0]):
              latent_dirs.append(comps[item][i])
          if item == 'act_stdev':
            for i in range(comps[item].shape[0]):
              latent_stdevs.append(comps[item][i])
        else:
          if item == 'lat_comp':
            for i in range(comps[item].shape[0]):
              latent_dirs.append(comps[item][i])
          if item == 'lat_stdev':
            for i in range(comps[item].shape[0]):
              latent_stdevs.append(comps[item][i])<jupyter_output>StyleGAN2: Optimized CUDA op FusedLeakyReLU not available, using native PyTorch fallback.
StyleGAN2: Optimized CUDA op UpFirDn2d not available, using native PyTorch fallback.
/content/drive/MyDrive/Colab/fyp/ganspace/models/checkpoints/stylegan2/stylegan2_portrait_512.pt<jupyter_text>GANSpace UI<jupyter_code>#@title Portrait GANSpace UI
import gradio as gr
import numpy as np

def generate_image(seed, truncation,
                  female, realism, greyhair, shorthair, shortchin, ponytail, blackhair,
                  start_layer, end_layer):

    scale = 1
    seed, start_layer, end_layer = int(seed), int(start_layer), int(end_layer)
    w = None
    params = {'female': female,
          'realism': realism,
          'greyhair': greyhair,
          'shorthair': shorthair,
          'shortchin': shortchin,
          'ponytail': ponytail,
          'blackhair': blackhair}

    param_indexes = {'female': 1,
              'realism': 4,
              'greyhair': 5,
              'shorthair': 6,
              'shortchin': 8,
              'ponytail': 9,
              'blackhair': 10}

    directions = []
    distances = []
    for k, v in params.items():
        directions.append(latent_dirs[param_indexes[k]])
        distances.append(v)

    model.truncation = truncation
    if w is None:
        w = model.sample_latent(1, seed=seed).detach().cpu().numpy()
        w = [w]*model.get_max_latents() # one per layer
    else:
        w = [np.expand_dims(x, 0) for x in w]
    

    if directions != None and distances != None:
        for l in range(start_layer, end_layer):
          for i in range(len(directions)):
            w[l] = w[l] + directions[i] * distances[i] * 1

    torch.cuda.empty_cache()
    #save image and display
    out = model.sample_np(w)
    final_im = Image.fromarray((out * 255).astype(np.uint8))
    # final_im.save('/tmp/edit_output.jpg') # save the content to temp
    
    return final_im

    #return display_sample_pytorch(int(seed), truncation, directions, distances, scale, int(start_layer), int(end_layer), disp=False)

truncation = gr.inputs.Slider(minimum=0, maximum=1, default=0.5, label="Truncation")
start_layer = gr.inputs.Number(default=0, label="Start Layer")
end_layer = gr.inputs.Number(default=14, label="End Layer")
seed = gr.inputs.Number(default=0, label="Seed")

slider_max_val = 20
slider_min_val = -20
slider_step = 1

female = gr.inputs.Slider(label="Female", minimum=slider_min_val, maximum=slider_max_val, default=0)
realism = gr.inputs.Slider(label="Realism", minimum=slider_min_val, maximum=slider_max_val, default=0)
greyhair = gr.inputs.Slider(label="Grey Hair", minimum=slider_min_val, maximum=slider_max_val, default=0)
shorthair = gr.inputs.Slider(label="Short Hair", minimum=slider_min_val, maximum=slider_max_val, default=0)
shortchin = gr.inputs.Slider(label="Short Chin", minimum=slider_min_val, maximum=slider_max_val, default=0)
ponytail = gr.inputs.Slider(label="Ponytail", minimum=slider_min_val, maximum=slider_max_val, default=0)
blackhair = gr.inputs.Slider(label="Black Hair", minimum=slider_min_val, maximum=slider_max_val, default=0)


scale = 1

inputs = [seed, truncation, female, realism, greyhair, shorthair, shortchin, ponytail, blackhair, start_layer, end_layer]

gr.Interface(generate_image, inputs, "image", live=True, title="GAN Editing").launch(debug=True)
import hashlib
seed = 'asds'

#@title Character Gradio UI
import gradio as gr
import numpy as np
import hashlib

def generate_image(seed, truncation,
                  monster, female, skimpy, light, bodysuit, bulky, human_head,
                  start_layer, end_layer):

    scale = 1
    seed = int(hashlib.sha256(seed.encode('utf-8')).hexdigest(), 16) % 10**8

    params = {'monster': monster,
          'female': female,
          'skimpy': skimpy,
          'light': light,
          'bodysuit': bodysuit,
          'bulky': bulky,
          'human_head': human_head}

    param_indexes = {'monster': 0,
              'female': 1,
              'skimpy': 2,
              'light': 4,
              'bodysuit': 5,
              'bulky': 6,
              'human_head': 8}

    directions = []
    distances = []
    for k, v in params.items():
        directions.append(latent_dirs[param_indexes[k]])
        distances.append(v)

    style = {'description_width': 'initial'}
    return display_sample_pytorch(int(seed), truncation, directions, distances, scale, int(start_layer), int(end_layer), disp=False)

truncation = gr.inputs.Slider(minimum=0, maximum=1, default=0.5, label="Truncation")
start_layer = gr.inputs.Number(default=0, label="Start Layer")
end_layer = gr.inputs.Number(default=14, label="End Layer")
seed = gr.inputs.Textbox(default='0')

slider_max_val = 20
slider_min_val = -20
slider_step = 1

monster = gr.inputs.Slider(label="Monster", minimum=slider_min_val, maximum=slider_max_val, default=0)
female = gr.inputs.Slider(label="Female", minimum=slider_min_val, maximum=slider_max_val, default=0)
skimpy = gr.inputs.Slider(label="Skimpy", minimum=slider_min_val, maximum=slider_max_val, default=0)
light = gr.inputs.Slider(label="Light", minimum=slider_min_val, maximum=slider_max_val, default=0)
bodysuit = gr.inputs.Slider(label="Bodysuit", minimum=slider_min_val, maximum=slider_max_val, default=0)
bulky = gr.inputs.Slider(label="Bulky", minimum=slider_min_val, maximum=slider_max_val, default=0)
human_head = gr.inputs.Slider(label="Human Head", minimum=slider_min_val, maximum=slider_max_val, default=0)


scale = 1

inputs = [seed, truncation, monster, female, skimpy, light, bodysuit, bulky, human_head, start_layer, end_layer]

gr.Interface(generate_image, inputs, "image", live=True, title="CharacterGAN").launch(debug=True)<jupyter_output>Colab notebook detected. This cell will run indefinitely so that you can see errors and logs. To turn off, set debug=False in launch().
This share link will expire in 24 hours. If you need a permanent link, visit: https://gradio.app/introducing-hosted (NEW!)
Running on External URL: https://53442.gradio.app
Interface loading below...<jupyter_text>FOMM<jupyter_code>#@title Load Checkpoint for FOMM
from demo import load_checkpoints
import moviepy.editor as mpe
import matplotlib.pyplot as plt
import matplotlib.animation as animation
from IPython.display import HTML
from demo import make_animation
from skimage import img_as_ubyte

#!gdown --id 1jmcn19-c3p8mf39aYNXUhdMqzqDYZhQ_ -O vox-cpk.pth.tar

generator, kp_detector = load_checkpoints(config_path='/content/drive/MyDrive/Colab/fyp/first-order-model/config/vox-256.yaml', 
                            checkpoint_path='/content/drive/MyDrive/Colab/fyp/first-order-model/vox-cpk.pth.tar')
#@title FOMM UI
import gradio as gr
import moviepy.editor as mpe
import matplotlib.pyplot as plt
import matplotlib.animation as animation
from IPython.display import HTML
from demo import make_animation
from skimage import img_as_ubyte

def predict_video(image_path, video_path, relative, show_inputs,
                  start_time=0, end_time=-1,
                  center_video_to_head=True, crop_video_to_head=True, video_crop_expansion_factor=1.5,
                  center_image_to_head=True, crop_image_to_head=True, image_crop_expansion_factor=2.5,
                  speech="", speaker='p225'):
  
    video_crop_expansion_factor = max(video_crop_expansion_factor, 1)
    image_crop_expansion_factor = max(image_crop_expansion_factor, 1)

    if end_time > start_time:
        # cut video
        print('Cutting Video...')
        ffmpeg_extract_subclip(video_path, start_time, end_time, targetname='cut.mp4')
        video_path = 'cut.mp4'

    source_image = imageio.imread(image_path.name)
    reader = imageio.get_reader(video_path)

    source_image = pad_crop_resize(source_image, *get_crop(source_image, center_face=center_image_to_head, crop_face=crop_image_to_head, expansion_factor=image_crop_expansion_factor))
    fps = reader.get_meta_data()['fps']
    print('FPS', fps)
    #fps = 8

    driving_video = []
    landmarks = None
    try:
        for i,im in enumerate(reader):
            if not crop_video_to_head:
                break
            landmarks = fa.get_landmarks_from_image(im)
            if landmarks:
                break
        x0,x1,y0,y1 = get_crop(im, center_face=center_video_to_head, crop_face=crop_video_to_head, expansion_factor=video_crop_expansion_factor, landmarks=landmarks)
        reader.set_image_index(0)
        for im in reader:
            driving_video.append(pad_crop_resize(im,x0,x1,y0,y1))
    except RuntimeError:
        pass

    # Generate animation
    predictions = make_animation(source_image, driving_video, generator, kp_detector, relative=relative)

    fig = plt.figure(figsize=(10 * show_inputs + 8 * (predictions is not None), 8))

    ims = []
    for i in range(len(driving_video)):
        cols = []
        if show_inputs and speech == "":
            cols.append(source_image)
            cols.append(driving_video[i])
        if predictions is not None:
            cols.append(predictions[i])
        im = plt.imshow(np.concatenate(cols, axis=1), animated=True)
        plt.axis('off')
        ims.append([im])

    plt.tight_layout()
    ani = animation.ArtistAnimation(fig, ims, interval=50, repeat_delay=1000)
    ani.save('output.mp4', fps=fps)
    plt.close()

    if speech != "":
        !tts --text "{speech}" --out_path speech.wav --model_name "tts_models/en/vctk/sc-glow-tts"  --speaker_idx {speaker}
        !cd Wav2Lip && python inference.py --checkpoint_path checkpoints/wav2lip_gan.pth --face "../output.mp4" --audio '../speech.wav'
        return 'Wav2Lip/results/result_voice.mp4'
    else:
        video_clip = mpe.VideoFileClip('output.mp4')
        audio_clip = mpe.AudioFileClip(video_path)
        final_clip = video_clip.set_audio(audio_clip)
        final_clip.write_videofile("result.mp4", fps=fps)
        return 'result.mp4'

image_input = gr.inputs.Image(type="file")
video_input = gr.inputs.Video(type="mp4")
relative = gr.inputs.Checkbox(default=True, label="Relative")
show_inputs = gr.inputs.Checkbox(default=True, label="Show Inputs")
start_time = gr.inputs.Number(default=0, label="Start Time")
end_time = gr.inputs.Number(default=-1, label="End Time")
center_video_to_head = gr.inputs.Checkbox(default=True, label="Center Video to Head")
crop_video_to_head = gr.inputs.Checkbox(default=True, label="Crop Video to Head")
video_crop_expansion_factor = gr.inputs.Number(default=2, label="Video Crop Expansion Factor")
center_image_to_head = gr.inputs.Checkbox(default=False, label="Center Image to Head")
crop_image_to_head = gr.inputs.Checkbox(default=False, label="Crop Image to Head")
image_crop_expansion_factor = gr.inputs.Number(default=2, label="Image Crop Expansion Factor")
speech = gr.inputs.Textbox(label="Text to Speech", default="")
speaker = gr.inputs.Dropdown(default='p225', choices=['p225', 'p226', 'p227', 'p228', 'p229', 'p230', 'p231', 'p232', 'p233', 'p234', 'p261'])

inputs = [image_input, video_input, relative, show_inputs, start_time, end_time,
          center_video_to_head, crop_video_to_head, video_crop_expansion_factor,
          center_image_to_head, crop_image_to_head, image_crop_expansion_factor,
          speech, speaker]
    
gr.Interface(predict_video, inputs, "video", live=False, title="Facial Animation").launch(debug=True)<jupyter_output>Colab notebook detected. This cell will run indefinitely so that you can see errors and logs. To turn off, set debug=False in launch().
This share link will expire in 24 hours. If you need a permanent link, visit: https://gradio.app/introducing-hosted (NEW!)
Running on External URL: https://41796.gradio.app
Interface loading below...<jupyter_text>Impersonator++<jupyter_code>#@title Impersonator++ Gradio UI
import gradio as gr
import os.path as osp
import platform
import argparse
import time
import sys
import subprocess
from IPython.display import HTML
from base64 import b64encode
from moviepy.editor import ImageSequenceClip

def imitate(im, video, bg_image, background, 
                   center_image_to_body, crop_image_to_body, image_crop_expansion_factor, keep_aspect_background, show_inputs, pose_fc, cam_fc):
    source_image = imageio.imread(im.name)
    source_image = pad_crop_resize(source_image, *get_crop_body(source_image, center_body=center_image_to_body, crop_body=crop_image_to_body, expansion_factor=image_crop_expansion_factor), new_h=800, new_w=800)
    imageio.imwrite('/content/crop.png', (source_image*255).astype(np.uint8))

    if bg_image != 'None':
        bg_image = imageio.imread(bg_image.name)
        bg_image = crop_resize(bg_image/255, source_image.shape[:2], crop=keep_aspect_background)
        imageio.imwrite('/content/bg_crop.png', (bg_image*255).astype(np.uint8))
    
    with imageio.get_reader(video, format='mp4') as reader:
        fps = reader.get_meta_data()['fps']
    
    gpu_ids = "0"
    num_source = 1
    assets_dir = "/content/drive/MyDrive/Colab/fyp/iPERCore/assets"
    output_dir = "/content/drive/MyDrive/Colab/fyp/iPERCore/results"
    shutil.rmtree(output_dir, ignore_errors=True)

    # symlink from the actual assets directory to this current directory
    work_asserts_dir = os.path.join("./assets")
    if not os.path.exists(work_asserts_dir):
        os.symlink(osp.abspath(assets_dir), osp.abspath(work_asserts_dir),
                  target_is_directory=(platform.system() == "Windows"))

    cfg_path = osp.join(work_asserts_dir, "configs", "deploy.toml")

    model_id = "mymodel"
    
    src_path = "\"path?=/content/crop.png,name?=mymodel"
    if background=='replace' and os.path.exists('/content/bg_crop.png'):
      src_path += ',bg_path?=/content/bg_crop.png'
    src_path += '"'

    ref_path = "\"path?=%s,"  \
             "name?=myoutput," \
             "pose_fc?=%d,"\
             "cam_fc?=%d,"\
             "fps?=%f\""%(video,pose_fc,cam_fc,fps)
    options = ''
    if background=='inpaint':
        options += ' --use_inpaintor'
    
    !python -m iPERCore.services.run_imitator --gpu_ids $gpu_ids --num_source $num_source --image_size $image_size --output_dir $output_dir --model_id $model_id --cfg_path $cfg_path --src_path $src_path --ref_path $ref_path $options

    if show_inputs:
        return "/content/drive/MyDrive/Colab/fyp/iPERCore/results/primitives/mymodel/synthesis/imitations/mymodel-myoutput.mp4"
    else:
        result_dir = '/content/drive/MyDrive/Colab/fyp/iPERCore/results/primitives/mymodel/synthesis/imitations/mymodel-myoutput'
        frames = [os.path.abspath(os.path.join(result_dir, p)) for p in os.listdir(result_dir) if p.endswith(('jpg', 'png'))]
        frames.sort()

        #fps = last_frame/10
        clip = ImageSequenceClip(frames, fps = fps)

        #import re
        #output_file = re.compile('\.png$').sub('.mp4', args.output)
        clip.write_videofile('impersonator_output.mp4')
        return 'impersonator_output.mp4'


im = gr.inputs.Image(type="file", label="Source Image")
bg_image = gr.inputs.Image(type="file", label="Background Image")
background = gr.inputs.Radio(choices=['None', 'replace', 'inpaint'], label="Background")
center_image_to_body = gr.inputs.Checkbox(default=True, label="Center Image to Body")
crop_image_to_body = gr.inputs.Checkbox(default=False, label="Crop Image to Body")
image_crop_expansion_factor = gr.inputs.Number(default=1.05, label="Image Crop Expansion Factor")
keep_aspect_background = gr.inputs.Checkbox(default=True, label="Keep Background Aspect Ratio")
show_inputs = gr.inputs.Checkbox(default=False, label="Show Inputs")
pose_fc = gr.inputs.Number(default=300, label="Pose Smooth Factor")
cam_fc = gr.inputs.Number(default=100, label="Camera Smooth Factor")

inputs = [im, 'video', bg_image, background, 
          center_image_to_body, crop_image_to_body, image_crop_expansion_factor, keep_aspect_background, show_inputs, pose_fc, cam_fc]



iface = gr.Interface(imitate, inputs, "video", live=False, title="Full Body Animation")
iface.launch(debug=True)<jupyter_output>Colab notebook detected. This cell will run indefinitely so that you can see errors and logs. To turn off, set debug=False in launch().
This share link will expire in 24 hours. If you need a permanent link, visit: https://gradio.app/introducing-hosted (NEW!)
Running on External URL: https://26129.gradio.app
Interface loading below...<jupyter_text>Server<jupyter_code>from IPython.display import Javascript
display(Javascript('''google.colab.output.setIframeHeight(0, true, {maxHeight: 200})'''))

!pip install fastapi nest-asyncio pyngrok uvicorn aiofiles python-multipart firebase-admin
firebase_admin.delete_app(firebase_admin.get_app())
#@title API Functions

from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import FileResponse, StreamingResponse
from fastapi import FastAPI, File, UploadFile, Form
from pydantic import BaseModel
from typing import Optional, Dict, List
from io import StringIO, BytesIO
from pydantic import BaseModel
import shutil
import datetime

import firebase_admin
from firebase_admin import credentials, firestore, storage

# Use the application default credentials
if not firebase_admin._apps:
    cred = credentials.Certificate("/content/drive/MyDrive/Colab/fyp/gancreate-firebase-adminsdk-yayqe-07b54a912d.json")
    firebase_admin.initialize_app(cred, {
        'storageBucket': 'gancreate.appspot.com'
    })

db = firestore.client()
bucket = storage.bucket()


# Log the user in
user = "YyzhczpqEzMw6uWHuikt0Icg2Sp1"


app = FastAPI()


app.add_middleware(
    CORSMiddleware,
    allow_origins=['*'],
    allow_credentials=True,
    allow_methods=['*'],
    allow_headers=['*'],
)

@app.get('/')
async def root():
    return {'hello': 'world'}

def sample(seed: int = None, truncation: float = 0.5, w=None):
    model.truncation = truncation
    if w is None:
        w = model.sample_latent(1, seed=seed).detach().cpu().numpy()
        w = [w]*model.get_max_latents() # one per layer
  
    #save image and display
    out = model.sample_np(w)
    return Image.fromarray((out * 255).astype(np.uint8))

@app.get('/generate')
async def generate(model_type: str = 'portrait', seed: int = None, truncation: float = 0.5, w=None, save=False):
    if config.output_class != model_type:
        load_model(model_type)
    model.truncation = truncation
    if w is None:
        w = model.sample_latent(1, seed=seed).detach().cpu().numpy()
        w = [w]*model.get_max_latents() # one per layer
  
    #save image and display
    out = model.sample_np(w)
    final_im = Image.fromarray((out * 255).astype(np.uint8))
    
    final_im.save('/tmp/output.jpg') # save the content to temp
    
    return FileResponse('/tmp/output.jpg', media_type="image/jpeg")

class EditConfig(BaseModel):
    model_type: Optional[str] = "portrait" 
    seed: Optional[int] = None
    truncation: Optional[float] = 0.5
    start_layer: Optional[int] = 0
    end_layer: Optional[int] = 14
    attributes: Dict[str, float]
    latent_id: Optional[str] = ""
    save: Optional[bool] = False

@app.post('/edit')
async def edit(edit_config: EditConfig):
    model_type = edit_config.model_type
    if config.output_class != model_type:
        load_model(model_type)

    seed = edit_config.seed        
    latent_id = edit_config.latent_id
    model.truncation = edit_config.truncation
    if latent_id is None or latent_id is "":
        w = model.sample_latent(1, seed=seed).detach().cpu().numpy()
        w = [w]*model.get_max_latents() # one per layer
    else:
        w = np.load(f"latents/{model_type}/{latent_id}.npy")
        w = [np.expand_dims(x, 0) for x in w]
    
    param_indexes = {
        "portrait": {
            'Gender': 1,
            'Realism': 4,
            'Gray Hair': 5,
            'Hair Length': 6,
            'Chin': 8,
            'Ponytail': 9,
            'Black Hair': 10
        },
        "model": {
            'Gender': 4,
            'Dress': 0,
            'Sleveeless': 1,
            'Short Skirt': 3,
            'Jacket': 5,
            'Darkness': 7,
            'Slimness': 9
        },
        "character": {
            'Monster': 0,
            'Gender': 1,
            'Skimpiness': 2,
            'Light': 4,
            'Bodysuit': 5,
            'Bulkiness': 6,
            'Human Head': 8
        }
    }
    
    
    

    directions = []
    distances = []
    for k, v in edit_config.attributes.items():
        directions.append(latent_dirs[param_indexes[model_type][k]])
        distances.append(v)

    if directions != None and distances != None:
        for l in range(edit_config.start_layer, edit_config.end_layer):
          for i in range(len(directions)):
            w[l] = w[l] + directions[i] * distances[i] * 1
    
    torch.cuda.empty_cache()
    #save image and display
    out = model.sample_np(w)
    final_im = Image.fromarray((out * 255).astype(np.uint8))
    final_im.save('/tmp/edit_output.jpg') # save the content to temp

    if edit_config.save:
        doc_ref = db.collection(f'{model_type}s').add({})
        doc_id = doc_ref[1].id
        
        blob = bucket.blob(f'{model_type}s/{doc_id}.jpg')
        blob.upload_from_filename('/tmp/edit_output.jpg')
        blob.make_public()

        data = {
            "url": blob.public_url,
            "user": user,
            "created_at": datetime.datetime.now()
        }
        db.collection(f'{model_type}s').document(doc_id).set(data)
        np.save(f'latents/{model_type}/{doc_id}.npy', w)

    return FileResponse('/tmp/edit_output.jpg', media_type="image/jpeg")

class AnimateFaceConfig(BaseModel):
    center_video_to_head: Optional[bool] = True
    crop_video_to_head: Optional[bool] = True
    center_image_to_head: Optional[bool] = True
    center_video_to_head: Optional[bool] = True
    video_crop_expansion_factor: Optional[float] = 1.5
    image_crop_expansion_factor: Optional[float] = 2.5
    speech: Optional[str] = ""
    speaker: Optional[str] = "p225"
    show_inputs: Optional[bool] = False



@app.post("/animate/face")
async def animate_face(
        files: List[UploadFile] = File(...),
        center_video_to_head: bool = Form(True),
        crop_video_to_head: bool = Form(True),
        video_crop_expansion_factor: float = Form(1.5),
        center_image_to_head: bool = Form(True),
        crop_image_to_head: bool = Form(True),
        image_crop_expansion_factor: float = Form(2.5),
        relative: bool = Form(True),
        speech: str = Form(""),
        speaker: str = Form("p225"),
        show_inputs: bool = Form(False),
    ):
  
    video_crop_expansion_factor = max(video_crop_expansion_factor, 1)
    image_crop_expansion_factor = max(image_crop_expansion_factor, 1)

    source_image = files[0]
    ref_video = files[1]

    with open(source_image.filename, "wb") as buffer:
        shutil.copyfileobj(source_image.file, buffer)

    video_path = "ref_video.mp4"

    with open(video_path, "wb") as buffer:
        shutil.copyfileobj(ref_video.file, buffer)

    source_image = imageio.imread(source_image.filename)
    reader = imageio.get_reader(video_path)

    source_image = pad_crop_resize(source_image, *get_crop(source_image, center_face=center_image_to_head, crop_face=crop_image_to_head, expansion_factor=image_crop_expansion_factor))
    fps = reader.get_meta_data()['fps']

    driving_video = []
    landmarks = None
    try:
        for i,im in enumerate(reader):
            if not crop_video_to_head:
                break
            landmarks = fa.get_landmarks_from_image(im)
            if landmarks:
                break
        x0,x1,y0,y1 = get_crop(im, center_face=center_video_to_head, crop_face=crop_video_to_head, expansion_factor=video_crop_expansion_factor, landmarks=landmarks)
        reader.set_image_index(0)
        for im in reader:
            driving_video.append(pad_crop_resize(im,x0,x1,y0,y1))
    except RuntimeError:
        pass

    # Generate animation
    predictions = make_animation(source_image, driving_video, generator, kp_detector, relative=relative)

    fig = plt.figure(figsize=(12 * show_inputs + 6 * (predictions is not None), 8))

    ims = []
    for i in range(len(driving_video)):
        cols = []
        if show_inputs and speech == "":
            cols.append(source_image)
            cols.append(driving_video[i])
        if predictions is not None:
            cols.append(predictions[i])
        im = plt.imshow(np.concatenate(cols, axis=1), animated=True)
        plt.axis('off')
        ims.append([im])

    plt.tight_layout()
    ani = animation.ArtistAnimation(fig, ims, interval=50, repeat_delay=1000)
    ani.save('output.mp4', fps=fps)
    plt.close()

    if speech != "":
        !tts --text "{speech}" --out_path speech.wav --model_name "tts_models/en/vctk/sc-glow-tts"  --speaker_idx {speaker}
        !cd Wav2Lip && python inference.py --checkpoint_path checkpoints/wav2lip_gan.pth --face "../output.mp4" --audio '../speech.wav'
        return FileResponse('Wav2Lip/results/result_voice.mp4', media_type="video/mp4")
    else:
        video_clip = mpe.VideoFileClip('output.mp4')
        audio_clip = mpe.AudioFileClip(video_path)
        final_clip = video_clip.set_audio(audio_clip)
        final_clip.write_videofile("result.mp4", fps=fps)
        return FileResponse('result.mp4', media_type="video/mp4")

@app.post("/animate/body")
async def animate_body(
        files: List[UploadFile] = File(...),
        background: str = Form("None"),
        center_image_to_body: bool = Form(True),
        crop_image_to_body: bool = Form(False),
        image_crop_expansion_factor: float = Form(1.05) ,
        keep_aspect_background: bool = Form(True),
        pose_fc: int = Form(300),
        cam_fc: int = Form(100),
    ):
  
    source_image = files[0]
    ref_video = files[1]
    bg_image = files[2]

    with open(source_image.filename, "wb") as buffer:
        shutil.copyfileobj(source_image.file, buffer)

    video_path = "ref_video.mp4"

    with open(video_path, "wb") as buffer:
        shutil.copyfileobj(ref_video.file, buffer)
    
    source_image = imageio.imread(source_image.filename)
    source_image = pad_crop_resize(source_image, *get_crop_body(source_image, center_body=center_image_to_body, crop_body=crop_image_to_body, expansion_factor=image_crop_expansion_factor))
    imageio.imwrite('/content/crop.png', (source_image*255).astype(np.uint8))

    if bg_image != 'None':
        with open(bg_image.filename, "wb") as buffer:
            shutil.copyfileobj(bg_image.file, buffer)
        bg_image = imageio.imread(bg_image.filename)
        bg_image = crop_resize(bg_image/255, source_image.shape[:2], crop=keep_aspect_background)
        imageio.imwrite('/content/bg_crop.png', (bg_image*255).astype(np.uint8))
    
    with imageio.get_reader(video_path, format='mp4') as reader:
        fps = reader.get_meta_data()['fps']
    
    gpu_ids = "0"
    num_source = 1
    assets_dir = "/content/drive/MyDrive/Colab/fyp/iPERCore/assets"
    output_dir = "/content/drive/MyDrive/Colab/fyp/iPERCore/results"
    shutil.rmtree(output_dir, ignore_errors=True)

    # symlink from the actual assets directory to this current directory
    work_asserts_dir = os.path.join("./assets")
    if not os.path.exists(work_asserts_dir):
        os.symlink(osp.abspath(assets_dir), osp.abspath(work_asserts_dir),
                  target_is_directory=(platform.system() == "Windows"))

    cfg_path = osp.join(work_asserts_dir, "configs", "deploy.toml")

    model_id = "mymodel"
    
    src_path = "\"path?=/content/crop.png,name?=mymodel"
    if background=='replace' and os.path.exists('/content/bg_crop.png'):
      src_path += ',bg_path?=/content/bg_crop.png'
    src_path += '"'

    ref_path = "\"path?=%s,"  \
             "name?=myoutput," \
             "pose_fc?=%d,"\
             "cam_fc?=%d,"\
             "fps?=%f\""%(video_path,pose_fc,cam_fc,fps)
    options = ''
    if background=='inpaint':
        options += ' --use_inpaintor'
    
    !python -m iPERCore.services.run_imitator --gpu_ids $gpu_ids --num_source $num_source --image_size $image_size --output_dir $output_dir --model_id $model_id --cfg_path $cfg_path --src_path $src_path --ref_path $ref_path $options

    return FileResponse('./iPERCore/results/primitives/mymodel/synthesis/imitations/mymodel-myoutput.mp4', media_type="video/mp4")
  
import nest_asyncio
from pyngrok import ngrok
import uvicorn

ngrok_tunnel = ngrok.connect(8000)
print('Public URL:', ngrok_tunnel.public_url+'/docs')
nest_asyncio.apply()
uvicorn.run(app, port=8000)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1277082021502367, 'fraction_numerical': 0.017071293640243474, 'mean_word_length': 3.2581926514399204, 'pattern_counts': {'":': 11, '<': 23, '<?xml version=': 0, '>': 20, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '49609', 'license': 'MIT', 'n_tokens_mistral': 15304, 'n_tokens_neox': 14690, 'n_words': 3016, 'path': 'gancreate.ipynb', 'repo_name': 'mfrashad/gancreate-saai', 'size': '42879'}