Upload 94 files

.gitattributes

@@ -33,3 +33,10 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+dataset/ucf24_demo/v_Biking_g01_c01.mp4 filter=lfs diff=lfs merge=lfs -text
+dataset/ucf24_demo/v_CliffDiving_g03_c01.mp4 filter=lfs diff=lfs merge=lfs -text
+dataset/ucf24_demo/v_Fencing_g01_c06.mp4 filter=lfs diff=lfs merge=lfs -text
+dataset/ucf24_demo/v_HorseRiding_g01_c03.mp4 filter=lfs diff=lfs merge=lfs -text
+dataset/ucf24_demo/v_IceDancing_g02_c05.mp4 filter=lfs diff=lfs merge=lfs -text
+dataset/ucf24_demo/v_SalsaSpin_g03_c01.mp4 filter=lfs diff=lfs merge=lfs -text
+dataset/ucf24_demo/v_SkateBoarding_g02_c01.mp4 filter=lfs diff=lfs merge=lfs -text

config/__init__.py

@@ -0,0 +1,21 @@
+from .dataset_config import dataset_config
+from .yowo_v2_config import yowo_v2_config
+
+
+def build_model_config(args):
+    print('==============================')
+    print('Model Config: {} '.format(args.version.upper()))
+    
+    if 'yowo_v2_' in args.version:
+        m_cfg = yowo_v2_config[args.version]
+
+    return m_cfg
+
+
+def build_dataset_config(args):
+    print('==============================')
+    print('Dataset Config: {} '.format(args.dataset.upper()))
+    
+    d_cfg = dataset_config[args.dataset]
+
+    return d_cfg

config/categories_count_32s2fpsnew.json

@@ -0,0 +1 @@
+{"fighter": 9420, "threatener": 3964, "victim": 9264, "outsider": 14812}

config/dataset_config.py

@@ -0,0 +1,94 @@
+# Dataset configuration
+
+
+dataset_config = {
+    'ucf24': {
+        # dataset
+        'gt_folder': './evaluator/groundtruths_ucf_jhmdb/groundtruths_ucf/',
+        # input size
+        'train_size': 224,
+        'test_size': 224,
+        # transform
+        'jitter': 0.2,
+        'hue': 0.1,
+        'saturation': 1.5,
+        'exposure': 1.5,
+        'sampling_rate': 1,
+        # cls label
+        'multi_hot': False,  # one hot
+        # optimizer
+        'optimizer': 'adamw',
+        'momentum': 0.9,
+        'weight_decay': 5e-4,
+        # warmup strategy
+        'warmup': 'linear',
+        'warmup_factor': 0.00066667,
+        'wp_iter': 500,
+        # class names
+        'valid_num_classes': 24,
+        'label_map': (
+                    'Basketball',     'BasketballDunk',    'Biking',            'CliffDiving',
+                    'CricketBowling', 'Diving',            'Fencing',           'FloorGymnastics', 
+                    'GolfSwing',      'HorseRiding',       'IceDancing',        'LongJump',
+                    'PoleVault',      'RopeClimbing',      'SalsaSpin',         'SkateBoarding',
+                    'Skiing',         'Skijet',            'SoccerJuggling',    'Surfing',
+                    'TennisSwing',    'TrampolineJumping', 'VolleyballSpiking', 'WalkingWithDog'
+                ),
+    },
+    
+    'ava_v2.2':{
+        # dataset
+        'frames_dir': 'frames/',
+        'frame_list': 'frame_lists/',
+        'annotation_dir': 'annotations/',
+        'train_gt_box_list': 'train.csv',
+        'val_gt_box_list': 'val.csv',
+        'train_exclusion_file': 'ava_train_excluded_timestamps_v2.2.csv',
+        'val_exclusion_file': 'ava_val_excluded_timestamps_v2.2.csv',
+        'labelmap_file': 'ava_action_list_v2.1_for_activitynet_2018.pbtxt', # 'ava_v2.2/ava_action_list_v2.2.pbtxt',
+        'class_ratio_file': 'categories_count_32s2fpsnew.json',
+        'backup_dir': 'C:/Users/Administrator/Downloads/YOWOv2/backup_dir',
+        # input size
+        'train_size': 224,
+        'test_size': 224,
+        # transform
+        'jitter': 0.2,
+        'hue': 0.1,
+        'saturation': 1.5,
+        'exposure': 1.5,
+        'sampling_rate': 1,
+        # cls label
+        'multi_hot': True,  # multi hot
+        # train config
+        'optimizer': 'adamw',
+        'momentum': 0.9,
+        'weight_decay': 5e-4,
+        # warmup strategy
+        'warmup': 'linear',
+        'warmup_factor': 0.00066667,
+        'wp_iter': 500,
+        # class names
+        'valid_num_classes': 3,
+        'label_map': ('bully', 'victim', 'outsider')
+        # 'valid_num_classes': 80,
+        # 'label_map': (
+        #             'bend/bow(at the waist)', 'crawl', 'crouch/kneel', 'dance', 'fall down',  # 1-5
+        #             'get up', 'jump/leap', 'lie/sleep', 'martial art', 'run/jog',             # 6-10
+        #             'sit', 'stand', 'swim', 'walk', 'answer phone',                           # 11-15
+        #             'brush teeth', 'carry/hold (an object)', 'catch (an object)', 'chop', 'climb (e.g. a mountain)',  # 16-20
+        #             'clink glass', 'close (e.g., a door, a box)', 'cook', 'cut', 'dig',                               # 21-25
+        #             'dress/put on clothing', 'drink', 'drive (e.g., a car, a truck)', 'eat', 'enter',                 # 26-30
+        #             'exit', 'extract', 'fishing', 'hit (an object)', 'kick (an object)',                              # 31-35
+        #             'lift/pick up', 'listen (e.g., to music)', 'open (e.g., a window, a car door)', 'paint', 'play board game',  # 36-40
+        #             'play musical instrument', 'play with pets', 'point to (an object)', 'press','pull (an object)',             # 41-45
+        #             'push (an object)', 'put down', 'read', 'ride (e.g., a bike, a car, a horse)', 'row boat',                   # 46-50
+        #             'sail boat', 'shoot', 'shovel', 'smoke', 'stir',                                                             # 51-55
+        #             'take a photo', 'text on/look at a cellphone', 'throw', 'touch (an object)', 'turn (e.g., a screwdriver)',   # 56-60
+        #             'watch (e.g., TV)', 'work on a computer', 'write', 'fight/hit (a person)', 'give/serve (an object) to (a person)',  # 61-65
+        #             'grab (a person)', 'hand clap', 'hand shake', 'hand wave', 'hug (a person)',                                        # 66-70
+        #             'kick (a person)', 'kiss (a person)', 'lift (a person)', 'listen to (a person)', 'play with kids',                  # 71-75
+        #             'push (another person)', 'sing to (e.g., self, a person, a group)', 'take (an object) from (a person)',             # 76-78
+        #             'talk to (e.g., self, a person, a group)', 'watch (a person)'                                                       # 79-80
+        #        ),
+    }
+}

config/yowo_v2_config.py

@@ -0,0 +1,84 @@
+# Model configuration
+
+
+yowo_v2_config = {
+    'yowo_v2_nano': {
+        # backbone
+        ## 2D
+        'backbone_2d': 'yolo_free_nano',
+        'pretrained_2d': True,
+        'stride': [8, 16, 32],
+        ## 3D
+        'backbone_3d': 'shufflenetv2',
+        'model_size': '1.0x',
+        'pretrained_3d': True,
+        'memory_momentum': 0.9,
+        # head
+        'head_dim': 64,
+        'head_norm': 'BN',
+        'head_act': 'lrelu',
+        'num_cls_heads': 2,
+        'num_reg_heads': 2,
+        'head_depthwise': True,
+    },
+
+    'yowo_v2_tiny': {
+        # backbone
+        ## 2D
+        'backbone_2d': 'yolo_free_tiny',
+        'pretrained_2d': True,
+        'stride': [8, 16, 32],
+        ## 3D
+        'backbone_3d': 'shufflenetv2',
+        'model_size': '2.0x',
+        'pretrained_3d': True,
+        'memory_momentum': 0.9,
+        # head
+        'head_dim': 64,
+        'head_norm': 'BN',
+        'head_act': 'lrelu',
+        'num_cls_heads': 2,
+        'num_reg_heads': 2,
+        'head_depthwise': False,
+    },
+
+    'yowo_v2_medium': {
+        # backbone
+        ## 2D
+        'backbone_2d': 'yolo_free_large',
+        'pretrained_2d': True,
+        'stride': [8, 16, 32],
+        ## 3D
+        'backbone_3d': 'shufflenetv2',
+        'model_size': '2.0x',
+        'pretrained_3d': True,
+        'memory_momentum': 0.9,
+        # head
+        'head_dim': 128,
+        'head_norm': 'BN',
+        'head_act': 'silu',
+        'num_cls_heads': 2,
+        'num_reg_heads': 2,
+        'head_depthwise': False,
+    },
+
+    'yowo_v2_large': {
+        # backbone
+        ## 2D
+        'backbone_2d': 'yolo_free_large',
+        'pretrained_2d': True,
+        'stride': [8, 16, 32],
+        ## 3D
+        'backbone_3d': 'resnext101',
+        'pretrained_3d': True,
+        'memory_momentum': 0.9,
+        # head
+        'head_dim': 256,
+        'head_norm': 'BN',
+        'head_act': 'silu',
+        'num_cls_heads': 2,
+        'num_reg_heads': 2,
+        'head_depthwise': False,
+    },
+
+}

dataset/ava.py

@@ -0,0 +1,300 @@
+#!/usr/bin/python
+# encoding: utf-8
+
+import os
+import numpy as np
+
+import torch
+from torch.utils.data import Dataset
+from PIL import Image
+
+try:
+    import ava_helper
+except:
+    from . import ava_helper
+
+
+# Dataset for AVA
+class AVA_Dataset(Dataset):
+    def __init__(self,
+                 cfg,
+                 data_root,
+                 is_train=False,
+                 img_size=224,
+                 transform=None,
+                 len_clip=8,
+                 sampling_rate=1):
+        self.num_classes = 3
+        self.data_root = data_root
+        self.frames_dir = os.path.join(data_root, cfg['frames_dir'])
+        self.frame_list = os.path.join(data_root, cfg['frame_list'])
+        self.annotation_dir = os.path.join(data_root, cfg['annotation_dir'])
+        self.labelmap_file = os.path.join(data_root, cfg['annotation_dir'], cfg['labelmap_file'])
+        if is_train:
+            self.gt_box_list = os.path.join(self.annotation_dir, cfg['train_gt_box_list'])
+            self.exclusion_file = os.path.join(self.annotation_dir, cfg['train_exclusion_file'])
+        else:
+            self.gt_box_list = os.path.join(self.annotation_dir, cfg['val_gt_box_list'])
+            self.exclusion_file = os.path.join(self.annotation_dir, cfg['val_exclusion_file'])
+
+        self.transform = transform
+        self.is_train = is_train
+        
+        self.img_size = img_size
+        self.len_clip = len_clip
+        self.sampling_rate = sampling_rate
+        self.seq_len = self.len_clip * self.sampling_rate
+
+        # load ava data
+        self._load_data()
+
+
+    def _load_data(self):
+        # Loading frame paths.
+        (
+            self._image_paths,
+            self._video_idx_to_name,
+        ) = ava_helper.load_image_lists(
+            self.frames_dir,
+            self.frame_list,
+            self.is_train
+            )
+
+        # Loading annotations for boxes and labels.
+        # boxes_and_labels: {'<video_name>': {<frame_num>: a list of [box_i, box_i_labels]} }
+        boxes_and_labels = ava_helper.load_boxes_and_labels(
+            self.gt_box_list,
+            self.exclusion_file,
+            self.is_train,
+            full_test_on_val=False
+            )
+
+        assert len(boxes_and_labels) == len(self._image_paths)
+
+        # boxes_and_labels: a list of {<frame_num>: a list of [box_i, box_i_labels]}
+        boxes_and_labels = [
+            boxes_and_labels[self._video_idx_to_name[i]]
+            for i in range(len(self._image_paths))
+        ]
+
+        # Get indices of keyframes and corresponding boxes and labels.
+        # _keyframe_indices: [video_idx, sec_idx, sec, frame_index]
+        # _keyframe_boxes_and_labels: list[list[list]], outer is video_idx, middle is sec_idx,
+        # inner is a list of [box_i, box_i_labels]
+        (
+            self._keyframe_indices,
+            self._keyframe_boxes_and_labels,
+        ) = ava_helper.get_keyframe_data(boxes_and_labels)
+
+        # Calculate the number of used boxes.
+        self._num_boxes_used = ava_helper.get_num_boxes_used(
+            self._keyframe_indices, self._keyframe_boxes_and_labels
+        )
+
+        self._max_objs = ava_helper.get_max_objs(
+            self._keyframe_indices, self._keyframe_boxes_and_labels
+        )
+
+        print("=== AVA dataset summary ===")
+        print("Train: {}".format(self.is_train))
+        print("Number of videos: {}".format(len(self._image_paths)))
+        total_frames = sum(
+            len(video_img_paths) for video_img_paths in self._image_paths
+        )
+        print("Number of frames: {}".format(total_frames))
+        print("Number of key frames: {}".format(len(self)))
+        print("Number of boxes: {}.".format(self._num_boxes_used))
+
+
+    def __len__(self):
+        return len(self._keyframe_indices)
+
+
+    def get_sequence(self, center_idx, half_len, sample_rate, num_frames):
+        """
+        Sample frames among the corresponding clip.
+
+        Args:
+            center_idx (int): center frame idx for current clip
+            half_len (int): half of the clip length
+            sample_rate (int): sampling rate for sampling frames inside of the clip
+            num_frames (int): number of expected sampled frames
+
+        Returns:
+            seq (list): list of indexes of sampled frames in this clip.
+        """
+        # seq = list(range(center_idx - half_len, center_idx + half_len, sample_rate))
+        seq = list(range(center_idx - half_len*2 + 1*sample_rate, center_idx+1*sample_rate, sample_rate))
+        
+        for seq_idx in range(len(seq)):
+            if seq[seq_idx] < 0:
+                seq[seq_idx] = 0
+            elif seq[seq_idx] >= num_frames:
+                seq[seq_idx] = num_frames - 1
+        return seq
+
+
+    def get_frame_idx(self, latest_idx, sample_length, sample_rate, num_frames):
+        """
+        Sample frames among the corresponding clip. But see keyframe as the latest frame,
+        instead of viewing it in center
+        """
+        # seq = list(range(latest_idx - sample_length + 1, latest_idx + 1, sample_rate))
+        seq = list(range(latest_idx, latest_idx - sample_length, -sample_rate))
+        seq.reverse()
+        for seq_idx in range(len(seq)):
+            if seq[seq_idx] < 0:
+                seq[seq_idx] = 0
+            elif seq[seq_idx] >= num_frames:
+                seq[seq_idx] = num_frames - 1
+
+        return seq
+
+
+    def __getitem__(self, idx):
+        # load a data
+        frame_idx, video_clip, target = self.pull_item(idx)
+
+        return frame_idx, video_clip, target
+
+
+    def pull_item(self, idx):
+        # Get the frame idxs for current clip. We can use it as center or latest
+        video_idx, sec_idx, sec, frame_idx = self._keyframe_indices[idx]
+        clip_label_list = self._keyframe_boxes_and_labels[video_idx][sec_idx]
+
+        # check label list
+        assert len(clip_label_list) > 0
+        assert len(clip_label_list) <= self._max_objs
+
+        # get a sequence
+        seq = self.get_sequence(
+            frame_idx,
+            self.seq_len // 2,
+            self.sampling_rate,
+            num_frames=len(self._image_paths[video_idx]),
+        )
+        image_paths = [self._image_paths[video_idx][frame - 1] for frame in seq]
+        #print('video_idx: ', video_idx, 'frame_idx', frame_idx)
+        keyframe_info = self._image_paths[video_idx][frame_idx - 1]
+        #print(keyframe_info)
+
+        # load a video clip
+        video_clip = []
+        for img_path in image_paths:
+            frame = Image.open(img_path).convert('RGB')
+            video_clip.append(frame)
+        ow, oh = frame.width, frame.height
+
+        # Get boxes and labels for current clip.
+        boxes = []
+        labels = []
+        for box_labels in clip_label_list:
+            bbox = box_labels[0]
+            label = box_labels[1]
+            multi_hot_label = np.zeros(1 + self.num_classes)
+            multi_hot_label[..., label] = 1.0
+
+            boxes.append(bbox)
+            labels.append(multi_hot_label[..., 1:].tolist())
+
+        boxes = np.array(boxes).reshape(-1, 4)
+        # renormalize bbox
+        boxes[..., [0, 2]] *= ow
+        boxes[..., [1, 3]] *= oh
+        labels = np.array(labels).reshape(-1, self.num_classes)
+
+        # target: [N, 4 + C]
+        target = np.concatenate([boxes, labels], axis=-1)
+
+        # transform
+        video_clip, target = self.transform(video_clip, target)
+        # List [T, 3, H, W] -> [3, T, H, W]
+        video_clip = torch.stack(video_clip, dim=1)
+
+        # reformat target
+        target = {
+            'boxes': target[:, :4].float(),  # [N, 4]
+            'labels': target[:, 4:].long(),  # [N, C]
+            'orig_size': [ow, oh],
+            'video_idx': video_idx,
+            'sec': sec,
+
+        }
+
+        return [video_idx, sec], video_clip, target
+
+
+
+if __name__ == '__main__':
+    import cv2
+    from transforms import Augmentation, BaseTransform
+
+    is_train = False
+    img_size = 224
+    len_clip = 16
+    sampling_rate = 1
+    dataset_config = {
+        'data_root': 'C:/Users/Administrator/Downloads/YOWOv2/data/clip32s_2fps_new',
+        'frames_dir': 'frames/',
+        'frame_list': 'frame_lists/',
+        'annotation_dir': 'annotations/',
+        'train_gt_box_list': 'train.csv',
+        'val_gt_box_list': 'val.csv',
+        'train_exclusion_file': 'ava_train_excluded_timestamps_v2.2.csv',
+        'val_exclusion_file': 'ava_val_excluded_timestamps_v2.2.csv',
+        'labelmap_file': 'ava_action_list_v2.1_for_activitynet_2018.pbtxt',
+    }
+    trans_config = {
+        'jitter': 0.2,
+        'hue': 0.1,
+        'saturation': 1.5,
+        'exposure': 1.5
+    }
+    train_transform = Augmentation(
+        img_size=img_size,
+        jitter=trans_config['jitter'],
+        saturation=trans_config['saturation'],
+        exposure=trans_config['exposure']
+        )
+    val_transform = BaseTransform(img_size=img_size)
+
+    train_dataset = AVA_Dataset(
+        cfg=dataset_config,
+        data_root=dataset_config['data_root'],
+        is_train=is_train,
+        img_size=img_size,
+        transform=train_transform,
+        len_clip=len_clip,
+        sampling_rate=sampling_rate
+    )
+
+    print(len(train_dataset))
+    for i in range(len(train_dataset)):
+        frame_id, video_clip, target = train_dataset[i]
+        key_frame = video_clip[:, -1, :, :]
+
+        # to numpy
+        key_frame = key_frame.permute(1, 2, 0).numpy()
+        key_frame = key_frame.astype(np.uint8)
+
+        # to BGR
+        key_frame = key_frame[..., (2, 1, 0)]
+        H, W, C = key_frame.shape
+
+        key_frame = key_frame.copy()
+        bboxes = target['boxes']
+        labels = target['labels']
+
+        for box, cls_id in zip(bboxes, labels):
+            x1, y1, x2, y2 = box
+            x1 = int(x1 * W)
+            y1 = int(y1 * H)
+            x2 = int(x2 * W)
+            y2 = int(y2 * H)
+            key_frame = cv2.rectangle(key_frame, (x1, y1), (x2, y2), (255, 0, 0))
+
+        # cv2 show
+        cv2.imshow('key frame', key_frame)
+        cv2.waitKey(0)
+        

dataset/ava_helper.py

@@ -0,0 +1,231 @@
+#!/usr/bin/env python3
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+
+import logging
+import os
+import csv
+from collections import defaultdict
+
+
+logger = logging.getLogger(__name__)
+FPS = 15
+AVA_VALID_FRAMES = range(0, 64)
+#AVA_VALID_FRAMES = range(902, 1799)
+
+
+def make_image_key(video_id, timestamp):
+    """Returns a unique identifier for a video id & timestamp."""
+    return "%s,%04d" % (video_id, int(timestamp))
+
+
+def read_exclusions(exclusions_file):
+    """Reads a CSV file of excluded timestamps.
+    Args:
+      exclusions_file: A file object containing a csv of video-id,timestamp.
+    Returns:
+      A set of strings containing excluded image keys, e.g. "aaaaaaaaaaa,0904",
+      or an empty set if exclusions file is None.
+    """
+    excluded = set()
+    if exclusions_file:
+        with open(exclusions_file, "r") as f:
+            reader = csv.reader(f)
+            for row in reader:
+                assert len(row) == 2, "Expected only 2 columns, got: " + row
+                excluded.add(make_image_key(row[0], row[1]))
+    return excluded
+
+
+def load_image_lists(frames_dir, frame_list, is_train):
+    """
+    Loading image paths from corresponding files.
+
+    Args:
+        frames_dir (str): path to frames dir.
+        frame_list (str): path to frame list.
+        is_train (bool): if it is training dataset or not.
+
+    Returns:
+        image_paths (list[list]): a list of items. Each item (also a list)
+            corresponds to one video and contains the paths of images for
+            this video.
+        video_idx_to_name (list): a list which stores video names.
+    """
+    # frame_list_dir is /data3/ava/frame_lists/
+    # contains 'train.csv' and 'val.csv'
+    if is_train:
+        list_name = "train.csv"
+    else:
+        list_name = "val.csv"
+
+    list_filename = os.path.join(frame_list, list_name)
+
+    image_paths = defaultdict(list)
+    video_name_to_idx = {}
+    video_idx_to_name = []
+    with open(list_filename, "r") as f:
+        f.readline()
+        for line in f:
+            row = line.split()
+            row = row[0].split(',')
+            # The format of each row should follow:
+            # original_vido_id video_id frame_id path labels.
+            assert len(row) == 5
+            video_name = row[0]
+
+            if video_name not in video_name_to_idx:
+                idx = len(video_name_to_idx)
+                video_name_to_idx[video_name] = idx
+                video_idx_to_name.append(video_name)
+
+            data_key = video_name_to_idx[video_name]
+
+            image_paths[data_key].append(os.path.join(frames_dir, row[3]))
+
+    image_paths = [image_paths[i] for i in range(len(image_paths))]
+
+    print("Finished loading image paths from: {}".format(list_filename))
+
+    return image_paths, video_idx_to_name
+
+
+def load_boxes_and_labels(gt_box_list, exclusion_file, is_train=False, full_test_on_val=False):
+    """
+    Loading boxes and labels from csv files.
+
+    Args:
+        cfg (CfgNode): config.
+        mode (str): 'train', 'val', or 'test' mode.
+    Returns:
+        all_boxes (dict): a dict which maps from `video_name` and
+            `frame_sec` to a list of `box`. Each `box` is a
+            [`box_coord`, `box_labels`] where `box_coord` is the
+            coordinates of box and 'box_labels` are the corresponding
+            labels for the box.
+    """
+    ann_filename = gt_box_list
+    all_boxes = {}
+    count = 0
+    unique_box_count = 0
+    excluded_keys = read_exclusions(exclusion_file)
+
+    with open(ann_filename, 'r') as f:
+        for line in f:
+            row = line.strip().split(',')
+
+            video_name, frame_sec = row[0], int(row[1])
+            key = "%s,%04d" % (video_name, frame_sec)
+            # if mode == 'train' and key in excluded_keys:
+            if key in excluded_keys:
+                print("Found {} to be excluded...".format(key))
+                continue
+
+            # Only select frame_sec % 4 = 0 samples for validation if not
+            # set FULL_TEST_ON_VAL (default False)
+            if not is_train and not full_test_on_val and frame_sec % 4 != 0:
+                continue
+            # Box with [x1, y1, x2, y2] with a range of [0, 1] as float
+            box_key = ",".join(row[2:6])
+            box = list(map(float, row[2:6]))
+            label = -1 if row[6] == "" else int(row[6])
+            if video_name not in all_boxes:
+                all_boxes[video_name] = {}
+                for sec in AVA_VALID_FRAMES:
+                    all_boxes[video_name][sec] = {}
+            if box_key not in all_boxes[video_name][frame_sec]:
+                all_boxes[video_name][frame_sec][box_key] = [box, []]
+                unique_box_count += 1
+
+            all_boxes[video_name][frame_sec][box_key][1].append(label)
+            if label != -1:
+                count += 1
+
+    for video_name in all_boxes.keys():
+        for frame_sec in all_boxes[video_name].keys():
+            # Save in format of a list of [box_i, box_i_labels].
+            all_boxes[video_name][frame_sec] = list(
+                all_boxes[video_name][frame_sec].values()
+            )
+
+    print("Finished loading annotations from: %s" % ", ".join([ann_filename]))
+    print("Number of unique boxes: %d" % unique_box_count)
+    print("Number of annotations: %d" % count)
+
+    return all_boxes
+
+
+def get_keyframe_data(boxes_and_labels):
+    """
+    Getting keyframe indices, boxes and labels in the dataset.
+
+    Args:
+        boxes_and_labels (list[dict]): a list which maps from video_idx to a dict.
+            Each dict `frame_sec` to a list of boxes and corresponding labels.
+
+    Returns:
+        keyframe_indices (list): a list of indices of the keyframes.
+        keyframe_boxes_and_labels (list[list[list]]): a list of list which maps from
+            video_idx and sec_idx to a list of boxes and corresponding labels.
+    """
+
+    def sec_to_frame(sec):
+        """
+        Convert time index (in second) to frame index.
+        0: 900
+        30: 901
+        """
+        #return (sec - 900) * FPS
+        return sec+1
+
+    keyframe_indices = []
+    keyframe_boxes_and_labels = []
+    count = 0
+    for video_idx in range(len(boxes_and_labels)):
+        sec_idx = 0
+        keyframe_boxes_and_labels.append([])
+        for sec in boxes_and_labels[video_idx].keys():
+            if sec not in AVA_VALID_FRAMES:
+                continue
+
+            if len(boxes_and_labels[video_idx][sec]) > 0:
+                keyframe_indices.append(
+                    (video_idx, sec_idx, sec, sec_to_frame(sec))
+                )
+                keyframe_boxes_and_labels[video_idx].append(
+                    boxes_and_labels[video_idx][sec]
+                )
+                sec_idx += 1
+                count += 1
+    logger.info("%d keyframes used." % count)
+
+    return keyframe_indices, keyframe_boxes_and_labels
+
+
+def get_num_boxes_used(keyframe_indices, keyframe_boxes_and_labels):
+    """
+    Get total number of used boxes.
+
+    Args:
+        keyframe_indices (list): a list of indices of the keyframes.
+        keyframe_boxes_and_labels (list[list[list]]): a list of list which maps from
+            video_idx and sec_idx to a list of boxes and corresponding labels.
+
+    Returns:
+        count (int): total number of used boxes.
+    """
+
+    count = 0
+    for video_idx, sec_idx, _, _ in keyframe_indices:
+        count += len(keyframe_boxes_and_labels[video_idx][sec_idx])
+    return count
+
+
+def get_max_objs(keyframe_indices, keyframe_boxes_and_labels):
+    # max_objs = 0
+    # for video_idx, sec_idx, _, _ in keyframe_indices:
+    #     num_boxes = len(keyframe_boxes_and_labels[video_idx][sec_idx])
+    #     if num_boxes > max_objs:
+    #         max_objs = num_boxes
+
+    # return max_objs
+    return 25 #### MODIFICATION FOR NOW! TODO: FIX LATER!

dataset/transforms.py

@@ -0,0 +1,176 @@
+import random
+import numpy as np
+import torch
+import torchvision.transforms.functional as F
+from PIL import Image
+
+
+# Augmentation for Training
+class Augmentation(object):
+    def __init__(self, img_size=224, jitter=0.2, hue=0.1, saturation=1.5, exposure=1.5):
+        self.img_size = img_size
+        self.jitter = jitter
+        self.hue = hue
+        self.saturation = saturation
+        self.exposure = exposure
+
+
+    def rand_scale(self, s):
+        scale = random.uniform(1, s)
+
+        if random.randint(0, 1): 
+            return scale
+
+        return 1./scale
+
+
+    def random_distort_image(self, video_clip):
+        dhue = random.uniform(-self.hue, self.hue)
+        dsat = self.rand_scale(self.saturation)
+        dexp = self.rand_scale(self.exposure)
+        
+        video_clip_ = []
+        for image in video_clip:
+            image = image.convert('HSV')
+            cs = list(image.split())
+            cs[1] = cs[1].point(lambda i: i * dsat)
+            cs[2] = cs[2].point(lambda i: i * dexp)
+            
+            def change_hue(x):
+                x += dhue * 255
+                if x > 255:
+                    x -= 255
+                if x < 0:
+                    x += 255
+                return x
+
+            cs[0] = cs[0].point(change_hue)
+            image = Image.merge(image.mode, tuple(cs))
+
+            image = image.convert('RGB')
+
+            video_clip_.append(image)
+
+        return video_clip_
+
+
+    def random_crop(self, video_clip, width, height):
+        dw =int(width * self.jitter)
+        dh =int(height * self.jitter)
+
+        pleft  = random.randint(-dw, dw)
+        pright = random.randint(-dw, dw)
+        ptop   = random.randint(-dh, dh)
+        pbot   = random.randint(-dh, dh)
+
+        swidth =  width - pleft - pright
+        sheight = height - ptop - pbot
+
+        sx = float(swidth)  / width
+        sy = float(sheight) / height
+        
+        dx = (float(pleft) / width)/sx
+        dy = (float(ptop) / height)/sy
+
+        # random crop
+        cropped_clip = [img.crop((pleft, ptop, pleft + swidth - 1, ptop + sheight - 1)) for img in video_clip]
+
+        return cropped_clip, dx, dy, sx, sy
+
+
+    def apply_bbox(self, target, ow, oh, dx, dy, sx, sy):
+        sx, sy = 1./sx, 1./sy
+        # apply deltas on bbox
+        target[..., 0] = np.minimum(0.999, np.maximum(0, target[..., 0] / ow * sx - dx)) 
+        target[..., 1] = np.minimum(0.999, np.maximum(0, target[..., 1] / oh * sy - dy)) 
+        target[..., 2] = np.minimum(0.999, np.maximum(0, target[..., 2] / ow * sx - dx)) 
+        target[..., 3] = np.minimum(0.999, np.maximum(0, target[..., 3] / oh * sy - dy)) 
+
+        # refine target
+        refine_target = []
+        for i in range(target.shape[0]):
+            tgt = target[i]
+            bw = (tgt[2] - tgt[0]) * ow
+            bh = (tgt[3] - tgt[1]) * oh
+
+            if bw < 1. or bh < 1.:
+                continue
+            
+            refine_target.append(tgt)
+
+        refine_target = np.array(refine_target).reshape(-1, target.shape[-1])
+
+        return refine_target
+        
+
+    def to_tensor(self, video_clip):
+        return [F.to_tensor(image) * 255. for image in video_clip]
+
+
+    def __call__(self, video_clip, target):
+        # Initialize Random Variables
+        oh = video_clip[0].height  
+        ow = video_clip[0].width
+        
+        # random crop
+        video_clip, dx, dy, sx, sy = self.random_crop(video_clip, ow, oh)
+
+        # resize
+        video_clip = [img.resize([self.img_size, self.img_size]) for img in video_clip]
+
+        # random flip
+        flip = random.randint(0, 1)
+        if flip:
+            video_clip = [img.transpose(Image.FLIP_LEFT_RIGHT) for img in video_clip]
+
+        # distort
+        video_clip = self.random_distort_image(video_clip)
+
+        # process target
+        if target is not None:
+            target = self.apply_bbox(target, ow, oh, dx, dy, sx, sy)
+            if flip:
+                target[..., [0, 2]] = 1.0 - target[..., [2, 0]]
+        else:
+            target = np.array([])
+            
+        # to tensor
+        video_clip = self.to_tensor(video_clip)
+        target = torch.as_tensor(target).float()
+
+        return video_clip, target 
+
+
+# Transform for Testing
+class BaseTransform(object):
+    def __init__(self, img_size=224, ):
+        self.img_size = img_size
+
+
+    def to_tensor(self, video_clip):
+        return [F.to_tensor(image) * 255. for image in video_clip]
+
+
+    def __call__(self, video_clip, target=None, normalize=True):
+        oh = video_clip[0].height
+        ow = video_clip[0].width
+
+        # resize
+        video_clip = [img.resize([self.img_size, self.img_size]) for img in video_clip]
+
+        # normalize target
+        # if target is not None:
+        #     if normalize:
+        #         target[..., [0, 2]] /= ow
+        #         target[..., [1, 3]] /= oh
+
+        # else:
+            # target = np.array([])
+
+        # to tensor
+        video_clip = self.to_tensor(video_clip)
+        #target = torch.as_tensor(target).float()
+
+        #return video_clip, target 
+        return video_clip 
+

dataset/ucf24_demo/v_Biking_g01_c01.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:069914984bf53b5dbf4b24fbf7d79288f3697a35e494635b9ab48e3c800aea59
+size 1703798

dataset/ucf24_demo/v_CliffDiving_g03_c01.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8975e99f1199731d8d55eacb6e0d633275618e6b2225c26e64e1e42396beb47
+size 1024051

dataset/ucf24_demo/v_Fencing_g01_c06.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:529c6738bc866a1ef2c3a14fb8e4538c91d3344298bee2d0714dca859099cc5d
+size 1403751

dataset/ucf24_demo/v_HorseRiding_g01_c03.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e33ea8d44b15086ceb7f905f9c059d97d8518dedb49eff4718f64c746d463c1e
+size 1527353

dataset/ucf24_demo/v_IceDancing_g02_c05.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bb209ed584117352e8f6d9d0cd5587aaae0f8e23b323f11ff18dbf6fe179388
+size 1503889

dataset/ucf24_demo/v_SalsaSpin_g03_c01.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:776bc11f53b18f8ec4a9b5bbbf9c01c7d95c7afce5dda13aec01dd1c9749a8e0
+size 1477281

dataset/ucf24_demo/v_SkateBoarding_g02_c01.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c428046991043c8fb4736d27356a3b2ea59a512d17fc821ed5333a7058ecfc7
+size 3293243

dataset/ucf_jhmdb.py

@@ -0,0 +1,311 @@
+#!/usr/bin/python
+# encoding: utf-8
+
+import os
+import random
+import numpy as np
+import glob
+
+import torch
+from torch.utils.data import Dataset
+from PIL import Image
+
+
+# Dataset for UCF24 & JHMDB
+class UCF_JHMDB_Dataset(Dataset):
+    def __init__(self,
+                 data_root,
+                 dataset='ucf24',
+                 img_size=224,
+                 transform=None,
+                 is_train=False,
+                 len_clip=16,
+                 sampling_rate=1):
+        self.data_root = data_root
+        self.dataset = dataset
+        self.transform = transform
+        self.is_train = is_train
+        
+        self.img_size = img_size
+        self.len_clip = len_clip
+        self.sampling_rate = sampling_rate
+            
+        if self.is_train:
+            self.split_list = 'trainlist.txt'
+        else:
+            self.split_list = 'testlist.txt'
+
+        # load data
+        with open(os.path.join(data_root, self.split_list), 'r') as file:
+            self.file_names = file.readlines()
+        self.num_samples  = len(self.file_names)
+
+        if dataset == 'ucf24':
+            self.num_classes = 24
+        elif dataset == 'jhmdb21':
+            self.num_classes = 21
+
+
+    def __len__(self):
+        return self.num_samples
+
+
+    def __getitem__(self, index):
+        # load a data
+        frame_idx, video_clip, target = self.pull_item(index)
+
+        return frame_idx, video_clip, target
+
+
+    def pull_item(self, index):
+        """ load a data """
+        assert index <= len(self), 'index range error'
+        image_path = self.file_names[index].rstrip()
+
+        img_split = image_path.split('/')  # ex. ['labels', 'Basketball', 'v_Basketball_g08_c01', '00070.txt']
+        # image name
+        img_id = int(img_split[-1][:5])
+
+        # path to label
+        label_path = os.path.join(self.data_root, img_split[0], img_split[1], img_split[2], '{:05d}.txt'.format(img_id))
+
+        # image folder
+        img_folder = os.path.join(self.data_root, 'rgb-images', img_split[1], img_split[2])
+
+        # frame numbers
+        if self.dataset == 'ucf24':
+            max_num = len(os.listdir(img_folder))
+        elif self.dataset == 'jhmdb21':
+            max_num = len(os.listdir(img_folder)) - 1
+
+        # sampling rate
+        if self.is_train:
+            d = random.randint(1, 2)
+        else:
+            d = self.sampling_rate
+
+        # load images
+        video_clip = []
+        for i in reversed(range(self.len_clip)):
+            # make it as a loop
+            img_id_temp = img_id - i * d
+            if img_id_temp < 1:
+                img_id_temp = 1
+            elif img_id_temp > max_num:
+                img_id_temp = max_num
+
+            # load a frame
+            if self.dataset == 'ucf24':
+                path_tmp = os.path.join(self.data_root, 'rgb-images', img_split[1], img_split[2] ,'{:05d}.jpg'.format(img_id_temp))
+            elif self.dataset == 'jhmdb21':
+                path_tmp = os.path.join(self.data_root, 'rgb-images', img_split[1], img_split[2] ,'{:05d}.png'.format(img_id_temp))
+            frame = Image.open(path_tmp).convert('RGB')
+            ow, oh = frame.width, frame.height
+
+            video_clip.append(frame)
+
+            frame_id = img_split[1] + '_' +img_split[2] + '_' + img_split[3]
+
+        # load an annotation
+        if os.path.getsize(label_path):
+            target = np.loadtxt(label_path)
+        else:
+            target = None
+
+        # [label, x1, y1, x2, y2] -> [x1, y1, x2, y2, label]
+        label = target[..., :1]
+        boxes = target[..., 1:]
+        target = np.concatenate([boxes, label], axis=-1).reshape(-1, 5)
+            
+        # transform
+        video_clip, target = self.transform(video_clip, target)
+        # List [T, 3, H, W] -> [3, T, H, W]
+        video_clip = torch.stack(video_clip, dim=1)
+
+        # reformat target
+        target = {
+            'boxes': target[:, :4].float(),      # [N, 4]
+            'labels': target[:, -1].long() - 1,    # [N,]
+            'orig_size': [ow, oh],
+            'video_idx':frame_id[:-10]
+        }
+
+        return frame_id, video_clip, target
+
+
+    def pull_anno(self, index):
+        """ load a data """
+        assert index <= len(self), 'index range error'
+        image_path = self.file_names[index].rstrip()
+
+        img_split = image_path.split('/')  # ex. ['labels', 'Basketball', 'v_Basketball_g08_c01', '00070.txt']
+        # image name
+        img_id = int(img_split[-1][:5])
+
+        # path to label
+        label_path = os.path.join(self.data_root, img_split[0], img_split[1], img_split[2], '{:05d}.txt'.format(img_id))
+
+        # load an annotation
+        target = np.loadtxt(label_path)
+        target = target.reshape(-1, 5)
+
+        return target
+        
+
+# Video Dataset for UCF24 & JHMDB
+class UCF_JHMDB_VIDEO_Dataset(Dataset):
+    def __init__(self,
+                 data_root,
+                 dataset='ucf24',
+                 img_size=224,
+                 transform=None,
+                 len_clip=16,
+                 sampling_rate=1):
+        self.data_root = data_root
+        self.dataset = dataset
+        self.transform = transform
+        
+        self.img_size = img_size
+        self.len_clip = len_clip
+        self.sampling_rate = sampling_rate
+            
+        if dataset == 'ucf24':
+            self.num_classes = 24
+        elif dataset == 'jhmdb21':
+            self.num_classes = 21
+
+
+    def set_video_data(self, line):
+        self.line = line
+
+        # load a video
+        self.img_folder = os.path.join(self.data_root, 'rgb-images', self.line)
+
+        if self.dataset == 'ucf24':
+            self.label_paths = sorted(glob.glob(os.path.join(self.img_folder, '*.jpg')))
+        elif self.dataset == 'jhmdb21':
+            self.label_paths = sorted(glob.glob(os.path.join(self.img_folder, '*.png')))
+
+
+    def __len__(self):
+        return len(self.label_paths)
+
+
+    def __getitem__(self, index):
+        return self.pull_item(index)
+
+
+    def pull_item(self, index):
+        image_path = self.label_paths[index]
+
+        video_split = self.line.split('/')
+        video_class = video_split[0]
+        video_file = video_split[1]
+        # for windows:
+        # img_split = image_path.split('\\')  # ex. [..., 'Basketball', 'v_Basketball_g08_c01', '00070.txt']
+        # for linux
+        img_split = image_path.split('/')  # ex. [..., 'Basketball', 'v_Basketball_g08_c01', '00070.txt']
+
+        # image name
+        img_id = int(img_split[-1][:5])
+        max_num = len(os.listdir(self.img_folder))
+        if self.dataset == 'ucf24':
+            img_name = os.path.join(video_class, video_file, '{:05d}.jpg'.format(img_id))
+        elif self.dataset == 'jhmdb21':
+            img_name = os.path.join(video_class, video_file, '{:05d}.png'.format(img_id))
+
+        # load video clip
+        video_clip = []
+        for i in reversed(range(self.len_clip)):
+            # make it as a loop
+            img_id_temp = img_id - i
+            if img_id_temp < 1:
+                img_id_temp = 1
+            elif img_id_temp > max_num:
+                img_id_temp = max_num
+
+            # load a frame
+            if self.dataset == 'ucf24':
+                path_tmp = os.path.join(self.data_root, 'rgb-images', video_class, video_file ,'{:05d}.jpg'.format(img_id_temp))
+            elif self.dataset == 'jhmdb21':
+                path_tmp = os.path.join(self.data_root, 'rgb-images', video_class, video_file ,'{:05d}.png'.format(img_id_temp))
+            frame = Image.open(path_tmp).convert('RGB')
+            ow, oh = frame.width, frame.height
+
+            video_clip.append(frame)
+
+        # transform
+        video_clip, _ = self.transform(video_clip, normalize=False)
+        # List [T, 3, H, W] -> [3, T, H, W]
+        video_clip = torch.stack(video_clip, dim=1)
+        orig_size = [ow, oh]  # width, height
+
+        target = {'orig_size': [ow, oh]}
+
+        return img_name, video_clip, target
+
+
+
+
+if __name__ == '__main__':
+    import cv2
+    from transforms import Augmentation, BaseTransform
+
+    data_root = 'D:/python_work/spatial-temporal_action_detection/dataset/ucf24'
+    dataset = 'ucf24'
+    is_train = True
+    img_size = 224
+    len_clip = 16
+    trans_config = {
+        'jitter': 0.2,
+        'hue': 0.1,
+        'saturation': 1.5,
+        'exposure': 1.5
+    }
+    train_transform = Augmentation(
+        img_size=img_size,
+        jitter=trans_config['jitter'],
+        saturation=trans_config['saturation'],
+        exposure=trans_config['exposure']
+        )
+    val_transform = BaseTransform(img_size=img_size)
+
+    train_dataset = UCF_JHMDB_Dataset(
+        data_root=data_root,
+        dataset=dataset,
+        img_size=img_size,
+        transform=train_transform,
+        is_train=is_train,
+        len_clip=len_clip,
+        sampling_rate=1
+    )
+
+    print(len(train_dataset))
+    for i in range(len(train_dataset)):
+        frame_id, video_clip, target = train_dataset[i]
+        key_frame = video_clip[:, -1, :, :]
+
+        # to numpy
+        key_frame = key_frame.permute(1, 2, 0).numpy()
+        key_frame = key_frame.astype(np.uint8)
+
+        # to BGR
+        key_frame = key_frame[..., (2, 1, 0)]
+        H, W, C = key_frame.shape
+
+        key_frame = key_frame.copy()
+        bboxes = target['boxes']
+        labels = target['labels']
+
+        for box, cls_id in zip(bboxes, labels):
+            x1, y1, x2, y2 = box
+            x1 = int(x1 * W)
+            y1 = int(y1 * H)
+            x2 = int(x2 * W)
+            y2 = int(y2 * H)
+            key_frame = cv2.rectangle(key_frame, (x1, y1), (x2, y2), (255, 0, 0))
+
+        # cv2 show
+        cv2.imshow('key frame', key_frame)
+        cv2.waitKey(0)
+