<commit_before>import os
import skvideo.io
import numpy as np
from skimage.io import *


def pad_batch(batch, batch_size, frame, pad=True):
    """
    DESCRIPTION:
        Take in a batch, pad it with 0s if necessary and return appended batch

    INPUTS:
        frame: a numpy array extracted from an MPEG
                (length x width x channel)
        batch: list of frames
        batch_size: number of frames per batch (integer >= 1)

    OUTPUT:
        batch: list of a batch, batch has batch_size frames, each an ndarray:
        (L x W x C)
    """

    if len(batch) > batch_size:
        raise ValueError('len(batch) should be <= batch_size!')
    elif batch_size < 1:
        raise ValueError('batch_size must be >= 1')
    elif len(batch) == batch_size:
        return batch
    elif len(batch) < batch_size:
        if pad is True:
            pad_list = [np.zeros(frame.shape)] * (batch_size - len(batch))
            batch += pad_list
        return batch
    else:
        raise ValueError('Something is wrong with the pad_batch function')


def decode_mpeg(v_path, batch_size=1, stride=1, start_idx=0, end_idx=-1,
                pad=True):
    """
    DESCRIPTION:
        Creates a list of batches of frames from an MPEG file.

    INPUTS:
        v_path: Path to MPEG video (i.e. include the video's name & 
                extension)
        batch_size: Number of frames in each batch
        stride: Stride indicates beginning of batches, i.e. every stride'th
                frame (integer > 1)
        start_idx: Index of first frame for first batch (integer >= 0)
        end_idx: Index of last frame in the range of interest (integer >= 0)
        pad: Boolean value indicating whether the last batch should be
             padded if it is not full after decoding the mpeg

    OUTPUTS:
        LIST of NUMPY batches of frames (length x width x channels), and
        if the last batch is not full, it is padded with frames of:
        np.zeros((frame.shape))

    #TODO create optional saving, incase someone wants to visualize the frames<commit_msg>minor comment changes<commit_after>import os
import skvideo.io
import numpy as np
from skimage.io import *


def pad_batch(batch, batch_size, frame, pad=True):
    """
    DESCRIPTION:
        Take in a batch, pad it with 0s if necessary and return appended batch

    ARGS:
        frame: a numpy array extracted from an MPEG
                (length x width x channel)
        batch: list of frames
        batch_size: number of frames per batch (integer >= 1)

    RETURN:
        batch: list of a batch, batch has batch_size frames, each an ndarray:
        (L x W x C)
    """

    if len(batch) > batch_size:
        raise ValueError('len(batch) should be <= batch_size!')
    elif batch_size < 1:
        raise ValueError('batch_size must be >= 1')
    elif len(batch) == batch_size:
        return batch
    elif len(batch) < batch_size:
        if pad is True:
            pad_list = [np.zeros(frame.shape)] * (batch_size - len(batch))
            batch += pad_list
        return batch
    else:
        raise ValueError('Something is wrong with the pad_batch function')


def decode_mpeg(v_path, batch_size=1, stride=1, start_idx=0, end_idx=-1,
                pad=True):
    """
    DESCRIPTION:
        Creates a list of batches of frames from an MPEG file.

    ARGS:
        v_path: Path to MPEG video (i.e. include the video's name & 
                extension)
        batch_size: Number of frames in each batch
        stride: Stride indicates beginning of batches, i.e. every stride'th
                frame (integer > 1)
        start_idx: Index of first frame for first batch (integer >= 0)
        end_idx: Index of last frame in the range of interest (integer >= 0)
        pad: Boolean value indicating whether the last batch should be
             padded if it is not full after decoding the mpeg

    RETURNS:
        LIST of NUMPY batches of frames (length x width x channels), and
        if the last batch is not full, it is padded with frames of:
        np.zeros((frame.shape))

    #TODO create optional saving, incase someone wants to visualize the frames
{'fraction_non_alphanumeric': 0.07194244604316546, 'fraction_numerical': 0.005275779376498801, 'mean_word_length': 2.3076923076923075, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2872274', 'n_tokens_mistral': 1214, 'n_tokens_neox': 1147, 'n_words': 501}