<commit_before>import json
import random
from pathlib import Path
import numpy as np
import scipy.misc
from moviepy.editor import VideoFileClip
from tqdm import tqdm


class WindowNpyGenerator(object):
    def __init__(self,
                 n_train=None,
                 n_val=None,
                 fps=1,
                 timesteps=5,
                 overlap=4,
                 target_dir=None):
        self.video_dirs = []
        self.n_train = n_train or 100
        self.n_val = n_val or 20
        self.fps = fps
        self.timesteps = timesteps
        self.overlap = overlap
        self.target_dir = target_dir or Path('./npy/')

    def extract_windows(self, video_dir):
        video = VideoFileClip(str(video_dir / 'video.mp4'))
        info = json.load((video_dir / 'info.json').open())
        n_frames = int(video.duration) * self.fps
        timesteps = self.timesteps
        overlap = self.overlap

        print(video_dir, n_frames, timesteps, overlap)

        label = np.zeros(n_frames, dtype=np.uint8)
        for s, e in zip(info['starts'], info['ends']):
            fs = round(s * self.fps)
            fe = round(e * self.fps)
            label[fs:fe + 1] = 1

        windows = [(video, f - timesteps, f, label[f - 1])
                   for f in range(timesteps, n_frames, n_frames - overlap)]
        return windows

    def gen_npz(self, windows, chunk_size=10000):
        for idx in tqdm(range(0, len(windows), chunk_size), desc='chunk'):
            chunk_s, chunk_e = idx, min(idx + chunk_size, len(windows))
            chunk = windows[chunk_s:chunk_e]

            n = len(chunk)
            xs = np.zeros((n, self.timesteps, 224, 224, 3), dtype=np.float32)
            ys = np.zeros(n, dtype=np.uint8)
            for i, (video, s, e, y) in enumerate(tqdm(chunk), desc='data'):
                for j in range(e - s):
                    img = video.get_frame((s + j) / self.fps)
                    xs[i][j] = scipy.misc.imresize(img, (224, 224))
                ys[i] = y

            npz_path = self.target_dir / '{:04d}.npy'.format(idx // chunk_size)
            np.savez(npz_path, xs=xs, ys=ys)

    def fit(self, video_dirs):
        train, val = [], []
        for video_dir in video_dirs:
            windows = self.extract_windows(video_dir)
            random.shuffle(windows)
            pivot = round((self.n_train) / (self.n_train + self.n_val) * len(windows))
            train.extend(windows[:pivot])
            val.extend(windows[pivot:])

        train = random.sample(train, k=self.n_train)<commit_msg>Update at 2017-07-21 16-11-13<commit_after>import json
import random
from pathlib import Path
import numpy as np
import scipy.misc
from moviepy.editor import VideoFileClip
from tqdm import tqdm


class WindowNpyGenerator(object):
    def __init__(self,
                 n_train=None,
                 n_val=None,
                 fps=1,
                 timesteps=5,
                 overlap=4,
                 target_dir=None):
        self.video_dirs = []
        self.n_train = n_train or 100
        self.n_val = n_val or 20
        self.fps = fps
        self.timesteps = timesteps
        self.overlap = overlap
        self.target_dir = target_dir or Path('./npy/')

    def extract_windows(self, video_dir):
        video = VideoFileClip(str(video_dir / 'video.mp4'))
        info = json.load((video_dir / 'info.json').open())
        n_frames = int(video.duration) * self.fps
        timesteps = self.timesteps
        overlap = self.overlap

        label = np.zeros(n_frames, dtype=np.uint8)
        for s, e in zip(info['starts'], info['ends']):
            fs = round(s * self.fps)
            fe = round(e * self.fps)
            label[fs:fe + 1] = 1

        windows = [(video, f - timesteps, f, label[f - 1])
                   for f in range(timesteps, n_frames, n_frames - overlap)]
        print(video_dir, len(windows))
        return windows

    def gen_npz(self, windows, chunk_size=10000):
        for idx in tqdm(range(0, len(windows), chunk_size), desc='chunk'):
            chunk_s, chunk_e = idx, min(idx + chunk_size, len(windows))
            chunk = windows[chunk_s:chunk_e]

            n = len(chunk)
            xs = np.zeros((n, self.timesteps, 224, 224, 3), dtype=np.float32)
            ys = np.zeros(n, dtype=np.uint8)
            for i, (video, s, e, y) in enumerate(tqdm(chunk), desc='data'):
                for j in range(e - s):
                    img = video.get_frame((s + j) / self.fps)
                    xs[i][j] = scipy.misc.imresize(img, (224, 224))
                ys[i] = y

            npz_path = self.target_dir / '{:04d}.npy'.format(idx // chunk_size)
            np.savez(npz_path, xs=xs, ys=ys)

    def fit(self, video_dirs):
        train, val = [], []
        for video_dir in video_dirs:
            windows = self.extract_windows(video_dir)
            random.shuffle(windows)
            pivot = round((self.n_train) / (self.n_train + self.n_val) * len(windows))
            train.extend(windows[:pivot])
            val.extend(windows[pivot:])

        train = random.sample(train, k=self.n_train)
{'fraction_non_alphanumeric': 0.12283096120101385, 'fraction_numerical': 0.01715734061220511, 'mean_word_length': 1.9197495731360272, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5332060', 'n_tokens_mistral': 1775, 'n_tokens_neox': 1668, 'n_words': 378}