Upload 8 files

scripts/evaluation/ddp_wrapper.py

@@ -0,0 +1,46 @@
+import datetime
+import argparse, importlib
+from pytorch_lightning import seed_everything
+
+import torch
+import torch.distributed as dist
+
+def setup_dist(local_rank):
+    if dist.is_initialized():
+        return
+    torch.cuda.set_device(local_rank)
+    torch.distributed.init_process_group('nccl', init_method='env://')
+
+
+def get_dist_info():
+    if dist.is_available():
+        initialized = dist.is_initialized()
+    else:
+        initialized = False
+    if initialized:
+        rank = dist.get_rank()
+        world_size = dist.get_world_size()
+    else:
+        rank = 0
+        world_size = 1
+    return rank, world_size
+
+
+if __name__ == '__main__':
+    now = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--module", type=str, help="module name", default="inference")
+    parser.add_argument("--local_rank", type=int, nargs="?", help="for ddp", default=0)
+    args, unknown = parser.parse_known_args()
+    inference_api = importlib.import_module(args.module, package=None)
+
+    inference_parser = inference_api.get_parser()
+    inference_args, unknown = inference_parser.parse_known_args()
+
+    seed_everything(inference_args.seed)
+    setup_dist(args.local_rank)
+    torch.backends.cudnn.benchmark = True
+    rank, gpu_num = get_dist_info()
+
+    print("@CoLVDM Inference [rank%d]: %s"%(rank, now))
+    inference_api.run_inference(inference_args, gpu_num, rank)

scripts/evaluation/funcs.py

@@ -0,0 +1,194 @@
+import os, sys, glob
+import numpy as np
+from collections import OrderedDict
+from decord import VideoReader, cpu
+import cv2
+
+import torch
+import torchvision
+sys.path.insert(1, os.path.join(sys.path[0], '..', '..'))
+from lvdm.models.samplers.ddim import DDIMSampler
+
+
+def batch_ddim_sampling(model, cond, noise_shape, n_samples=1, ddim_steps=50, ddim_eta=1.0,\
+                        cfg_scale=1.0, temporal_cfg_scale=None, **kwargs):
+    ddim_sampler = DDIMSampler(model)
+    uncond_type = model.uncond_type
+    batch_size = noise_shape[0]
+
+    ## construct unconditional guidance
+    if cfg_scale != 1.0:
+        if uncond_type == "empty_seq":
+            prompts = batch_size * [""]
+            #prompts = N * T * [""]  ## if is_imgbatch=True
+            uc_emb = model.get_learned_conditioning(prompts)
+        elif uncond_type == "zero_embed":
+            c_emb = cond["c_crossattn"][0] if isinstance(cond, dict) else cond
+            uc_emb = torch.zeros_like(c_emb)
+                
+        ## process image embedding token
+        if hasattr(model, 'embedder'):
+            uc_img = torch.zeros(noise_shape[0],3,224,224).to(model.device)
+            ## img: b c h w >> b l c
+            uc_img = model.get_image_embeds(uc_img)
+            uc_emb = torch.cat([uc_emb, uc_img], dim=1)
+        
+        if isinstance(cond, dict):
+            uc = {key:cond[key] for key in cond.keys()}
+            uc.update({'c_crossattn': [uc_emb]})
+        else:
+            uc = uc_emb
+    else:
+        uc = None
+    
+    x_T = None
+    batch_variants = []
+    #batch_variants1, batch_variants2 = [], []
+    for _ in range(n_samples):
+        if ddim_sampler is not None:
+            kwargs.update({"clean_cond": True})
+            samples, _ = ddim_sampler.sample(S=ddim_steps,
+                                            conditioning=cond,
+                                            batch_size=noise_shape[0],
+                                            shape=noise_shape[1:],
+                                            verbose=False,
+                                            unconditional_guidance_scale=cfg_scale,
+                                            unconditional_conditioning=uc,
+                                            eta=ddim_eta,
+                                            temporal_length=noise_shape[2],
+                                            conditional_guidance_scale_temporal=temporal_cfg_scale,
+                                            x_T=x_T,
+                                            **kwargs
+                                            )
+        ## reconstruct from latent to pixel space
+        batch_images = model.decode_first_stage_2DAE(samples)
+        batch_variants.append(batch_images)
+    ## batch, <samples>, c, t, h, w
+    batch_variants = torch.stack(batch_variants, dim=1)
+    return batch_variants
+
+
+def get_filelist(data_dir, ext='*'):
+    file_list = glob.glob(os.path.join(data_dir, '*.%s'%ext))
+    file_list.sort()
+    return file_list
+
+def get_dirlist(path):
+    list = []
+    if (os.path.exists(path)):
+        files = os.listdir(path)
+        for file in files:
+            m = os.path.join(path,file)
+            if (os.path.isdir(m)):
+                list.append(m)
+    list.sort()
+    return list
+
+
+def load_model_checkpoint(model, ckpt):
+    def load_checkpoint(model, ckpt, full_strict):
+        state_dict = torch.load(ckpt, map_location="cpu")
+        try:
+            ## deepspeed
+            new_pl_sd = OrderedDict()
+            for key in state_dict['module'].keys():
+                new_pl_sd[key[16:]]=state_dict['module'][key]
+            model.load_state_dict(new_pl_sd, strict=full_strict)
+        except:
+            if "state_dict" in list(state_dict.keys()):
+                state_dict = state_dict["state_dict"]
+            model.load_state_dict(state_dict, strict=full_strict)
+        return model
+    load_checkpoint(model, ckpt, full_strict=True)
+    print('>>> model checkpoint loaded.')
+    return model
+
+
+def load_prompts(prompt_file):
+    f = open(prompt_file, 'r')
+    prompt_list = []
+    for idx, line in enumerate(f.readlines()):
+        l = line.strip()
+        if len(l) != 0:
+            prompt_list.append(l)
+        f.close()
+    return prompt_list
+
+
+def load_video_batch(filepath_list, frame_stride, video_size=(256,256), video_frames=16):
+    '''
+    Notice about some special cases:
+    1. video_frames=-1 means to take all the frames (with fs=1)
+    2. when the total video frames is less than required, padding strategy will be used (repreated last frame)
+    '''
+    fps_list = []
+    batch_tensor = []
+    assert frame_stride > 0, "valid frame stride should be a positive interge!"
+    for filepath in filepath_list:
+        padding_num = 0
+        vidreader = VideoReader(filepath, ctx=cpu(0), width=video_size[1], height=video_size[0])
+        fps = vidreader.get_avg_fps()
+        total_frames = len(vidreader)
+        max_valid_frames = (total_frames-1) // frame_stride + 1
+        if video_frames < 0:
+            ## all frames are collected: fs=1 is a must
+            required_frames = total_frames
+            frame_stride = 1
+        else:
+            required_frames = video_frames
+        query_frames = min(required_frames, max_valid_frames)
+        frame_indices = [frame_stride*i for i in range(query_frames)]
+
+        ## [t,h,w,c] -> [c,t,h,w]
+        frames = vidreader.get_batch(frame_indices)
+        frame_tensor = torch.tensor(frames.asnumpy()).permute(3, 0, 1, 2).float()
+        frame_tensor = (frame_tensor / 255. - 0.5) * 2
+        if max_valid_frames < required_frames:
+            padding_num = required_frames - max_valid_frames
+            frame_tensor = torch.cat([frame_tensor, *([frame_tensor[:,-1:,:,:]]*padding_num)], dim=1)
+            print(f'{os.path.split(filepath)[1]} is not long enough: {padding_num} frames padded.')
+        batch_tensor.append(frame_tensor)
+        sample_fps = int(fps/frame_stride)
+        fps_list.append(sample_fps)
+    
+    return torch.stack(batch_tensor, dim=0)
+
+from PIL import Image
+def load_image_batch(filepath_list, image_size=(256,256)):
+    batch_tensor = []
+    for filepath in filepath_list:
+        _, filename = os.path.split(filepath)
+        _, ext = os.path.splitext(filename)
+        if ext == '.mp4':
+            vidreader = VideoReader(filepath, ctx=cpu(0), width=image_size[1], height=image_size[0])
+            frame = vidreader.get_batch([0])
+            img_tensor = torch.tensor(frame.asnumpy()).squeeze(0).permute(2, 0, 1).float()
+        elif ext == '.png' or ext == '.jpg':
+            img = Image.open(filepath).convert("RGB")
+            rgb_img = np.array(img, np.float32)
+            #bgr_img = cv2.imread(filepath, cv2.IMREAD_COLOR)
+            #bgr_img = cv2.cvtColor(bgr_img, cv2.COLOR_BGR2RGB)
+            rgb_img = cv2.resize(rgb_img, (image_size[1],image_size[0]), interpolation=cv2.INTER_LINEAR)
+            img_tensor = torch.from_numpy(rgb_img).permute(2, 0, 1).float()
+        else:
+            print(f'ERROR: <{ext}> image loading only support format: [mp4], [png], [jpg]')
+            raise NotImplementedError
+        img_tensor = (img_tensor / 255. - 0.5) * 2
+        batch_tensor.append(img_tensor)
+    return torch.stack(batch_tensor, dim=0)
+
+
+def save_videos(batch_tensors, savedir, filenames, fps=10):
+    # b,samples,c,t,h,w
+    n_samples = batch_tensors.shape[1]
+    for idx, vid_tensor in enumerate(batch_tensors):
+        video = vid_tensor.detach().cpu()
+        video = torch.clamp(video.float(), -1., 1.)
+        video = video.permute(2, 0, 1, 3, 4) # t,n,c,h,w
+        frame_grids = [torchvision.utils.make_grid(framesheet, nrow=int(n_samples)) for framesheet in video] #[3, 1*h, n*w]
+        grid = torch.stack(frame_grids, dim=0) # stack in temporal dim [t, 3, n*h, w]
+        grid = (grid + 1.0) / 2.0
+        grid = (grid * 255).to(torch.uint8).permute(0, 2, 3, 1)
+        savepath = os.path.join(savedir, f"{filenames[idx]}.mp4")
+        torchvision.io.write_video(savepath, grid, fps=fps, video_codec='h264', options={'crf': '10'})
+

scripts/evaluation/inference.py

@@ -0,0 +1,137 @@
+import argparse, os, sys, glob, yaml, math, random
+import datetime, time
+import numpy as np
+from omegaconf import OmegaConf
+from collections import OrderedDict
+from tqdm import trange, tqdm
+from einops import repeat
+from einops import rearrange, repeat
+from functools import partial
+import torch
+from pytorch_lightning import seed_everything
+
+from funcs import load_model_checkpoint, load_prompts, load_image_batch, get_filelist, save_videos
+from funcs import batch_ddim_sampling
+from utils.utils import instantiate_from_config
+
+
+def get_parser():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--seed", type=int, default=20230211, help="seed for seed_everything")
+    parser.add_argument("--mode", default="base", type=str, help="which kind of inference mode: {'base', 'i2v'}")
+    parser.add_argument("--ckpt_path", type=str, default=None, help="checkpoint path")
+    parser.add_argument("--config", type=str, help="config (yaml) path")
+    parser.add_argument("--prompt_file", type=str, default=None, help="a text file containing many prompts")
+    parser.add_argument("--savedir", type=str, default=None, help="results saving path")
+    parser.add_argument("--savefps", type=str, default=10, help="video fps to generate")
+    parser.add_argument("--n_samples", type=int, default=1, help="num of samples per prompt",)
+    parser.add_argument("--ddim_steps", type=int, default=50, help="steps of ddim if positive, otherwise use DDPM",)
+    parser.add_argument("--ddim_eta", type=float, default=1.0, help="eta for ddim sampling (0.0 yields deterministic sampling)",)
+    parser.add_argument("--bs", type=int, default=1, help="batch size for inference")
+    parser.add_argument("--height", type=int, default=512, help="image height, in pixel space")
+    parser.add_argument("--width", type=int, default=512, help="image width, in pixel space")
+    parser.add_argument("--frames", type=int, default=-1, help="frames num to inference")
+    parser.add_argument("--fps", type=int, default=24)
+    parser.add_argument("--unconditional_guidance_scale", type=float, default=1.0, help="prompt classifier-free guidance")
+    parser.add_argument("--unconditional_guidance_scale_temporal", type=float, default=None, help="temporal consistency guidance")
+    ## for conditional i2v only
+    parser.add_argument("--cond_input", type=str, default=None, help="data dir of conditional input")
+    return parser
+
+
+def run_inference(args, gpu_num, gpu_no, **kwargs):
+    ## step 1: model config
+    ## -----------------------------------------------------------------
+    config = OmegaConf.load(args.config)
+    #data_config = config.pop("data", OmegaConf.create())
+    model_config = config.pop("model", OmegaConf.create())
+    model = instantiate_from_config(model_config)
+    model = model.cuda(gpu_no)
+    assert os.path.exists(args.ckpt_path), f"Error: checkpoint [{args.ckpt_path}] Not Found!"
+    model = load_model_checkpoint(model, args.ckpt_path)
+    model.eval()
+
+    ## sample shape
+    assert (args.height % 16 == 0) and (args.width % 16 == 0), "Error: image size [h,w] should be multiples of 16!"
+    ## latent noise shape
+    h, w = args.height // 8, args.width // 8
+    frames = model.temporal_length if args.frames < 0 else args.frames
+    channels = model.channels
+    
+    ## saving folders
+    os.makedirs(args.savedir, exist_ok=True)
+
+    ## step 2: load data
+    ## -----------------------------------------------------------------
+    assert os.path.exists(args.prompt_file), "Error: prompt file NOT Found!"
+    prompt_list = load_prompts(args.prompt_file)
+    num_samples = len(prompt_list)
+    filename_list = [f"{id+1:04d}" for id in range(num_samples)]
+
+    samples_split = num_samples // gpu_num
+    residual_tail = num_samples % gpu_num
+    print(f'[rank:{gpu_no}] {samples_split}/{num_samples} samples loaded.')
+    indices = list(range(samples_split*gpu_no, samples_split*(gpu_no+1)))
+    if gpu_no == 0 and residual_tail != 0:
+        indices = indices + list(range(num_samples-residual_tail, num_samples))
+    prompt_list_rank = [prompt_list[i] for i in indices]
+
+    ## conditional input
+    if args.mode == "i2v":
+        ## each video or frames dir per prompt
+        cond_inputs = get_filelist(args.cond_input, ext='[mpj][pn][4gj]')   # '[mpj][pn][4gj]'
+        assert len(cond_inputs) == num_samples, f"Error: conditional input ({len(cond_inputs)}) NOT match prompt ({num_samples})!"
+        filename_list = [f"{os.path.split(cond_inputs[id])[-1][:-4]}" for id in range(num_samples)]
+        cond_inputs_rank = [cond_inputs[i] for i in indices]
+
+    filename_list_rank = [filename_list[i] for i in indices]
+
+    ## step 3: run over samples
+    ## -----------------------------------------------------------------
+    start = time.time()
+    n_rounds = len(prompt_list_rank) // args.bs
+    n_rounds = n_rounds+1 if len(prompt_list_rank) % args.bs != 0 else n_rounds
+    for idx in range(0, n_rounds):
+        print(f'[rank:{gpu_no}] batch-{idx+1} ({args.bs})x{args.n_samples} ...')
+        idx_s = idx*args.bs
+        idx_e = min(idx_s+args.bs, len(prompt_list_rank))
+        batch_size = idx_e - idx_s
+        filenames = filename_list_rank[idx_s:idx_e]
+        noise_shape = [batch_size, channels, frames, h, w]
+        fps = torch.tensor([args.fps]*batch_size).to(model.device).long()
+
+        prompts = prompt_list_rank[idx_s:idx_e]
+        if isinstance(prompts, str):
+            prompts = [prompts]
+        #prompts = batch_size * [""]
+        text_emb = model.get_learned_conditioning(prompts)
+
+        if args.mode == 'base':
+            cond = {"c_crossattn": [text_emb], "fps": fps}
+        elif args.mode == 'i2v':
+            #cond_images = torch.zeros(noise_shape[0],3,224,224).to(model.device)
+            cond_images = load_image_batch(cond_inputs_rank[idx_s:idx_e], (args.height, args.width))
+            cond_images = cond_images.to(model.device)
+            img_emb = model.get_image_embeds(cond_images)
+            imtext_cond = torch.cat([text_emb, img_emb], dim=1)
+            cond = {"c_crossattn": [imtext_cond], "fps": fps}
+        else:
+            raise NotImplementedError
+
+        ## inference
+        batch_samples = batch_ddim_sampling(model, cond, noise_shape, args.n_samples, \
+                                                args.ddim_steps, args.ddim_eta, args.unconditional_guidance_scale, **kwargs)
+        ## b,samples,c,t,h,w
+        save_videos(batch_samples, args.savedir, filenames, fps=args.savefps)
+
+    print(f"Saved in {args.savedir}. Time used: {(time.time() - start):.2f} seconds")
+
+
+if __name__ == '__main__':
+    now = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
+    print("@CoLVDM Inference: %s"%now)
+    parser = get_parser()
+    args = parser.parse_args()
+    seed_everything(args.seed)
+    rank, gpu_num = 0, 1
+    run_inference(args, gpu_num, rank)

scripts/gradio/i2v_test.py

@@ -0,0 +1,83 @@
+import os
+import time
+from omegaconf import OmegaConf
+import torch
+from scripts.evaluation.funcs import load_model_checkpoint, load_image_batch, save_videos, batch_ddim_sampling
+from utils.utils import instantiate_from_config
+from huggingface_hub import hf_hub_download
+
+class Image2Video():
+    def __init__(self,result_dir='./tmp/',gpu_num=1) -> None:
+        self.download_model()
+        self.result_dir = result_dir
+        if not os.path.exists(self.result_dir):
+            os.mkdir(self.result_dir)
+        ckpt_path='checkpoints/i2v_512_v1/model.ckpt'
+        config_file='configs/inference_i2v_512_v1.0.yaml'
+        config = OmegaConf.load(config_file)
+        model_config = config.pop("model", OmegaConf.create())
+        model_config['params']['unet_config']['params']['use_checkpoint']=False   
+        model_list = []
+        for gpu_id in range(gpu_num):
+            model = instantiate_from_config(model_config)
+            # model = model.cuda(gpu_id)
+            assert os.path.exists(ckpt_path), "Error: checkpoint Not Found!"
+            model = load_model_checkpoint(model, ckpt_path)
+            model.eval()
+            model_list.append(model)
+        self.model_list = model_list
+        self.save_fps = 8
+
+    def get_image(self, image, prompt, steps=50, cfg_scale=12.0, eta=1.0, fps=16):
+        torch.cuda.empty_cache()
+        print('start:', prompt, time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())))
+        start = time.time()
+        gpu_id=0
+        if steps > 60:
+            steps = 60 
+        model = self.model_list[gpu_id]
+        model = model.cuda()
+        batch_size=1
+        channels = model.model.diffusion_model.in_channels
+        frames = model.temporal_length
+        h, w = 320 // 8, 512 // 8
+        noise_shape = [batch_size, channels, frames, h, w]
+
+        # text cond
+        text_emb = model.get_learned_conditioning([prompt])
+
+        # img cond
+        img_tensor = torch.from_numpy(image).permute(2, 0, 1).float()
+        img_tensor = (img_tensor / 255. - 0.5) * 2
+        img_tensor = img_tensor.unsqueeze(0)
+        cond_images = img_tensor.to(model.device)
+        img_emb = model.get_image_embeds(cond_images)
+        imtext_cond = torch.cat([text_emb, img_emb], dim=1)
+        cond = {"c_crossattn": [imtext_cond], "fps": fps}
+        
+        ## inference
+        batch_samples = batch_ddim_sampling(model, cond, noise_shape, n_samples=1, ddim_steps=steps, ddim_eta=eta, cfg_scale=cfg_scale)
+        ## b,samples,c,t,h,w
+        prompt_str = prompt.replace("/", "_slash_") if "/" in prompt else prompt
+        prompt_str = prompt_str.replace(" ", "_") if " " in prompt else prompt_str
+        prompt_str=prompt_str[:30]
+
+        save_videos(batch_samples, self.result_dir, filenames=[prompt_str], fps=self.save_fps)
+        print(f"Saved in {prompt_str}. Time used: {(time.time() - start):.2f} seconds")
+        model = model.cpu()
+        return os.path.join(self.result_dir, f"{prompt_str}.mp4")
+    
+    def download_model(self):
+        REPO_ID = 'VideoCrafter/Image2Video-512'
+        filename_list = ['model.ckpt']
+        if not os.path.exists('./checkpoints/i2v_512_v1/'):
+            os.makedirs('./checkpoints/i2v_512_v1/')
+        for filename in filename_list:
+            local_file = os.path.join('./checkpoints/i2v_512_v1/', filename)
+            if not os.path.exists(local_file):
+                hf_hub_download(repo_id=REPO_ID, filename=filename, local_dir='./checkpoints/i2v_512_v1/', local_dir_use_symlinks=False)
+    
+if __name__ == '__main__':
+    i2v = Image2Video()
+    video_path = i2v.get_image('prompts/i2v_prompts/horse.png','horses are walking on the grassland')
+    print('done', video_path)

scripts/gradio/t2v_test.py

@@ -0,0 +1,77 @@
+import os
+import time
+from omegaconf import OmegaConf
+import torch
+from scripts.evaluation.funcs import load_model_checkpoint, save_videos, batch_ddim_sampling
+from utils.utils import instantiate_from_config
+from huggingface_hub import hf_hub_download
+
+class Text2Video():
+    def __init__(self,result_dir='./tmp/',gpu_num=1) -> None:
+        self.download_model()
+        self.result_dir = result_dir
+        if not os.path.exists(self.result_dir):
+            os.mkdir(self.result_dir)
+        ckpt_path='checkpoints/base_512_v2/model.ckpt'
+        config_file='configs/inference_t2v_512_v2.0.yaml'
+        config = OmegaConf.load(config_file)
+        model_config = config.pop("model", OmegaConf.create())
+        model_config['params']['unet_config']['params']['use_checkpoint']=False   
+        model_list = []
+        for gpu_id in range(gpu_num):
+            model = instantiate_from_config(model_config)
+            # model = model.cuda(gpu_id)
+            assert os.path.exists(ckpt_path), "Error: checkpoint Not Found!"
+            model = load_model_checkpoint(model, ckpt_path)
+            model.eval()
+            model_list.append(model)
+        self.model_list = model_list
+        self.save_fps = 8
+
+    def get_prompt(self, prompt, steps=50, cfg_scale=12.0, eta=1.0, fps=16):
+        torch.cuda.empty_cache()
+        print('start:', prompt, time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())))
+        start = time.time()
+        gpu_id=0
+        if steps > 60:
+            steps = 60 
+        model = self.model_list[gpu_id]
+        model = model.cuda()
+        batch_size=1
+        channels = model.model.diffusion_model.in_channels
+        frames = model.temporal_length
+        h, w = 320 // 8, 512 // 8
+        noise_shape = [batch_size, channels, frames, h, w]
+
+        # text cond
+        text_emb = model.get_learned_conditioning([prompt])
+        cond = {"c_crossattn": [text_emb], "fps": fps}
+        
+        ## inference
+        batch_samples = batch_ddim_sampling(model, cond, noise_shape, n_samples=1, ddim_steps=steps, ddim_eta=eta, cfg_scale=cfg_scale)
+        ## b,samples,c,t,h,w
+        prompt_str = prompt.replace("/", "_slash_") if "/" in prompt else prompt
+        prompt_str = prompt_str.replace(" ", "_") if " " in prompt else prompt_str
+        prompt_str=prompt_str[:30]
+
+        save_videos(batch_samples, self.result_dir, filenames=[prompt_str], fps=self.save_fps)
+        print(f"Saved in {prompt_str}. Time used: {(time.time() - start):.2f} seconds")
+        model=model.cpu()
+        return os.path.join(self.result_dir, f"{prompt_str}.mp4")
+    
+    def download_model(self):
+        REPO_ID = 'VideoCrafter/VideoCrafter2'
+        filename_list = ['model.ckpt']
+        if not os.path.exists('./checkpoints/base_512_v2/'):
+            os.makedirs('./checkpoints/base_512_v2/')
+        for filename in filename_list:
+            local_file = os.path.join('./checkpoints/base_512_v2/', filename)
+
+            if not os.path.exists(local_file):
+                hf_hub_download(repo_id=REPO_ID, filename=filename, local_dir='./checkpoints/base_512_v2/', local_dir_use_symlinks=False)
+
+    
+if __name__ == '__main__':
+    t2v = Text2Video()
+    video_path = t2v.get_prompt('a black swan swims on the pond')
+    print('done', video_path)

scripts/run_text2video.sh

@@ -0,0 +1,23 @@
+name="7videos"
+
+ckpt='checkpoints/base_512_v2/model.ckpt'
+config='configs/inference_t2v_512_v2.0.yaml'
+
+prompt_file="prompts/test_prompts.txt"
+res_dir="results"
+
+python3 scripts/evaluation/inference.py \
+--seed 123 \
+--mode 'base' \
+--ckpt_path $ckpt \
+--config $config \
+--savedir $res_dir/$name \
+--n_samples 1 \
+--bs 1 --height 320 --width 512 \
+--unconditional_guidance_scale 5.0 \
+--unconditional_guidance_scale_temporal 5.0 \
+--ddim_steps 50 \
+--ddim_eta 1.0 \
+--prompt_file $prompt_file \
+--frames 20 \
+--savefps 4