<jupyter_start><jupyter_text>Preprocessing  Train, Eval, Test Split ![](https://cdn-images-1.medium.com/max/2000/1*H5CUZ0lQqRk6xbzNRa2M9A.png)  - 3 TRAINING videos follow the trajectory above. - 1 EVAL video uses a different trajectory and is colored differently. - 1 TEST video uses a different trajectory.<jupyter_code>import cv2
import matplotlib.pyplot as plt
from skvideo.io import FFmpegReader
filenames = [
    './data/6zXXZvVvTFs',
    './data/2AYaxTiWKoY',
    './data/sYbBgkP9aMo',
    './data/pF6xCZA72o0',
    './data/SuZVyOlgVek',
]
trims = [
    (960, 1403),
    (550, 1515),
    (1, 331),
    (1465, 2201),
    (15, 437),
]
crops = [
    (35, 50, 445, 300),
    (0, 13, 640, 335),
    (5, 22, 475, 341),
    (20, 3, 620, 360),
    (79, 18, 560, 360),
]
labels = [
    'Train',
    'Train',
    'Train',
    'Eval',
    'Test',
]
first_frames = []
last_frames = []
for filename, trim, crop in zip(filenames, trims, crops):
    # Get video frames with scikit-video
    reader = FFmpegReader(filename + '.mp4',
                          inputdict={'-r': '15'},
                          outputdict={'-r': '15'}
                          )

    for frame_idx, frame in enumerate(reader.nextFrame()):
        # Trim video (time)
        if frame_idx < trim[0]:
            continue
        elif frame_idx == trim[0]:
            # Crop frames (space)
            frame = frame[crop[1]:crop[3], crop[0]:crop[2], :]
            frame = cv2.resize(frame, (140, 140))
            first_frames.append(frame)
        elif frame_idx < trim[1] - 1:
            continue
        elif frame_idx == trim[1] - 1:
            frame = frame[crop[1]:crop[3], crop[0]:crop[2], :]
            frame = cv2.resize(frame, (140, 140))
            last_frames.append(frame)
            break
        else:
            break<jupyter_output><empty_output><jupyter_text>First Frame<jupyter_code>fig = plt.figure(figsize=(16, 8))
for i, frame in enumerate(first_frames):
    fig.add_subplot(1, len(first_frames), i+1)
    plt.imshow(frame)
    plt.axis('off')
    plt.title(labels[i] + '\n' + filenames[i].split('/')[-1])
plt.axis('off')
plt.show()<jupyter_output><empty_output><jupyter_text>Last Frame in First Room<jupyter_code>fig = plt.figure(figsize=(16, 8))
for i, frame in enumerate(last_frames):
    fig.add_subplot(1, len(last_frames), i+1)
    plt.imshow(frame)
    plt.axis('off')
    plt.title(labels[i] + '\n' + filenames[i].split('/')[-1])
plt.axis('off')
plt.show()<jupyter_output><empty_output><jupyter_text>First and Last<jupyter_code>fig = plt.figure(figsize=(16, 8))
fig.tight_layout()
for i, frame in enumerate(first_frames):
    fig.add_subplot(2, len(first_frames), i+1)
    plt.imshow(frame)
    plt.axis('off')
    plt.title(labels[i] + '\n' + filenames[i].split('/')[-1])
for i, frame in enumerate(last_frames):
    fig.add_subplot(2, len(last_frames), len(last_frames)+i+1)
    plt.imshow(frame)
    plt.axis('off')
plt.axis('off')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1659959758551308, 'fraction_numerical': 0.05130784708249497, 'mean_word_length': 2.919842312746386, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 17, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '330693', 'license': 'MIT', 'n_tokens_mistral': 1236, 'n_tokens_neox': 1123, 'n_words': 221, 'path': 'PREPROCESS.ipynb', 'repo_name': 'seungjaeryanlee/playing-hard-exploration-games-by-watching-youtube', 'size': '2982'}