<commit_before>from django.core.management.base import BaseCommand, CommandError
from query.models import *
import subprocess
import cv2
import os


def get_dimensions(path):
    cmd = 'ffprobe -v error -show_entries stream=width,height -of default=noprint_wrappers=1 "{}"'
    s = subprocess.check_output(cmd.format(path), shell=True).split("\n")
    if s[0].split("=")[1] == "N/A":
        width = int(s[2].split("=")[1])
        height = int(s[3].split("=")[1])
    else:
        width = int(s[0].split("=")[1])
        height = int(s[1].split("=")[1])
    return width, height


def get_fps(path):
    cmd = 'ffmpeg -i "{}" 2>&1 | sed -n "s/.*, \\(.*\\) fp.*/\\1/p"'
    return float(subprocess.check_output(cmd.format(path), shell=True))


def get_num_frames(path):
    cmd = '''
    ffprobe -v error -count_frames -select_streams v:0 \
      -show_entries stream=nb_read_frames -of default=nokey=1:noprint_wrappers=1 \
      "{}"'''
    return int(subprocess.check_output(cmd.format(path), shell=True))


def make_thumbnail(video):
    subprocess.check_call(['mkdir', '-p', 'assets/thumbnails'])
    subprocess.check_call(['cp '+'assets/thumbnails/{}_frame_000001.png '.format(video.id)+'assets/thumbnails/{}.png'.format(video.id)], shell=True)

def extract_audio(video):
    subprocess.check_call(['mkdir', '-p', 'assets/audio'])
    cmd = 'ffmpeg -y -i "{}" -c:a copy "assets/audio/{}.aac"'.format(video.path, video.id)
    subprocess.check_call(cmd, shell=True)


def extract_frames(video):
    subprocess.check_call(['mkdir', '-p', 'assets/thumbnails'])
    print video.path
    cmd = 'ffmpeg -y -i "{}" assets/thumbnails/{}_frame_%06d.png' \
          .format(video.path, video.id)
    subprocess.check_call(cmd, shell=True)

def delete_between_frames(video):
    stride = video.get_stride()
    for frame_num in range(video.num_frames):
        if (frame_num % stride == 0):
            continue
        filename = ("assets/thumbnails/{}_frame_{:>06}.png".format(video.id, frame_num+1))
        if os.path.isfile(filename):
            os.remove(filename)


class Command(BaseCommand):
    help = 'Ingest videos'

    def add_arguments(self, parser):
        parser.add_argument('path')

    def handle(self, *args, **options):
        with open(options['path']) as f:
            paths = set([s.strip() for s in f.readlines()])

        # Save ingested videos into SQL database
        for path in paths:
            print path
            videos = Video.objects.filter(path=path)
            video = None
            if len(videos) == 0:
                video = Video()
                video.path = path
                video.fps = get_fps(path)
                video.num_frames = get_num_frames(path)
                width, height = get_dimensions(path)
                video.width = width
                video.height = height
                video.save()
            else:
                video = videos.get()

            # Create corresponding label sets<commit_msg>fixes for ingest<commit_after>from django.core.management.base import BaseCommand, CommandError
from query.models import *
import subprocess
import cv2
import os


def get_dimensions(path):
    cmd = 'ffprobe -v error -show_entries stream=width,height -of default=noprint_wrappers=1 "{}"'
    s = subprocess.check_output(cmd.format(path), shell=True).split("\n")
    if s[0].split("=")[1] == "N/A":
        width = int(s[2].split("=")[1])
        height = int(s[3].split("=")[1])
    else:
        width = int(s[0].split("=")[1])
        height = int(s[1].split("=")[1])
    return width, height


def get_fps(path):
    cmd = 'ffmpeg -i "{}" 2>&1 | sed -n "s/.*, \\(.*\\) fp.*/\\1/p"'
    return float(subprocess.check_output(cmd.format(path), shell=True))


def get_num_frames(path):
    cmd = '''
    ffprobe -v error -count_frames -select_streams v:0 \
      -show_entries stream=nb_read_frames -of default=nokey=1:noprint_wrappers=1 \
      "{}"'''
    return int(subprocess.check_output(cmd.format(path), shell=True))


def make_thumbnail(video):
    subprocess.check_call(['mkdir', '-p', 'assets/thumbnails'])
    subprocess.check_call(['cp '+'assets/thumbnails/{}_frame_000001.png '.format(video.id)+'assets/thumbnails/{}.png'.format(video.id)], shell=True)

def extract_audio(video):
    subprocess.check_call(['mkdir', '-p', 'assets/audio'])
    cmd = 'ffmpeg -y -i "{}" -c:a copy "assets/audio/{}.aac"'.format(video.path, video.id)
    subprocess.check_call(cmd, shell=True)


def extract_frames(video):
    subprocess.check_call(['mkdir', '-p', 'assets/thumbnails'])
    print video.path
    cmd = 'ffmpeg -ss 00:00:00 -t 01:00:00 -y -i "{}" assets/thumbnails/{}_frame_%06d.png' \
          .format(video.path, video.id)
    subprocess.check_call(cmd, shell=True)

def delete_between_frames(video):
    stride = video.get_stride()
    for frame_num in range(video.num_frames):
        if (frame_num % stride == 0):
            continue
        filename = ("assets/thumbnails/{}_frame_{:>06}.png".format(video.id, frame_num+1))
        if os.path.isfile(filename):
            os.remove(filename)


class Command(BaseCommand):
    help = 'Ingest videos'

    def add_arguments(self, parser):
        parser.add_argument('path')

    def handle(self, *args, **options):
        with open(options['path']) as f:
            paths = set([s.strip() for s in f.readlines()])

        # Save ingested videos into SQL database
        for path in paths:
            print path
            videos = Video.objects.filter(path=path)
            video = None
            if len(videos) == 0:
                video = Video()
                video.path = path
                video.fps = get_fps(path)
                video.num_frames = min(get_num_frames(path), int(video.fps*60*60))
                width, height = get_dimensions(path)
                video.width = width
                video.height = height
                video.save()
            else:
                video = videos.get()

            # Create corresponding label sets
{'fraction_non_alphanumeric': 0.15734439834024896, 'fraction_numerical': 0.012946058091286307, 'mean_word_length': 2.6632218844984803, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1879200', 'n_tokens_mistral': 2034, 'n_tokens_neox': 1945, 'n_words': 396}