<jupyter_start><jupyter_text>Example: Predict body part positions from an MP4 fileThis notebook presents an example pipeline for applying a trained LEAP network to ~360k frames read from an MP4 video.You can download the data to reproduce the benchmarking results below.**Input data:** [072212_163153.mp4](https://1drv.ms/v/s!AnmpIqqfwz3zgcgekCxNp-MN76p1UQ) (254 MiB)**Output data:** [072212_163153.preds.h5](https://1drv.ms/u/s!AnmpIqqfwz3zgcgdDhQrKRsBaxvCXQ) (46.9 MiB)The trained network can be [downloaded here](https://1drv.ms/u/s!AnmpIqqfwz3zgdpIOzsqojhEmr0J0w) or from the links in the [repository data folder](https://github.com/talmo/leap/tree/master/data).<jupyter_code>import os
import numpy as np
import cv2
import h5py
from time import time

import keras
import keras.models
from leap.predict_box import convert_to_peak_outputs
from leap.utils import versions

versions(list_devices=True)<jupyter_output>C:\Anaconda3\lib\site-packages\h5py\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
  from ._conv import register_converters as _register_converters
Using TensorFlow backend.<jupyter_text>Parameters<jupyter_code># Media file path
video_path = "D:/tmp/072212_163153.mp4"

# Trained network path
model_path = "D:/OneDrive/code/leap/data/BermanFlies/models/180615_025354-n=1500/final_model.h5"

# Predictions output path
save_path = "D:/tmp/072212_163153.preds.h5"

# Number of frames to read before predicting (higher = faster, but limited by RAM)
chunk_size = 10000

# Number of frames to evaluate at once on the GPU (higher = faster, but limited by GPU memory)
batch_size = 64<jupyter_output><empty_output><jupyter_text>Processing<jupyter_code>t0_all = time()

# Load model and convert to peak-coordinate output
model = convert_to_peak_outputs(keras.models.load_model(model_path))
print("Model:", model_path)
print("    Input:", str(model.input_shape))
print("    Output:", str(model.output_shape))

# model = keras.utils.multi_gpu_model(model, gpus=2)

# Open video for reading
reader = cv2.VideoCapture(video_path)
num_samples = int(reader.get(cv2.CAP_PROP_FRAME_COUNT))

# Initialize
positions_pred = []
conf_pred = []
buffer = []
samples_predicted = 0
reading_runtime = 0
prediction_runtime = 0
done = False

# Process video chunk-by-chunk
while not done:
    t0_reading = time()
    # Read and finish if no frame was retrieved
    returned_frame, I = reader.read()
    done = not returned_frame
    reading_runtime += time() - t0_reading
    
    # Add current frame to buffer
    if not done:
        buffer.append(I[...,0])
    
    # Do we have anything to predict?
    if len(buffer) >= chunk_size or (done and len(buffer) > 0):
        t0_prediction = time()
        
        # Predict on buffer
        Y = model.predict(np.stack(buffer, axis=0)[...,None], batch_size=batch_size)
        
        # Save
        positions_pred.append(Y[:,:2,:].astype("int32"))
        conf_pred.append(Y[:,2,:].squeeze())
        
        # Empty out buffer container
        buffer = []
        
        # Performance stats
        samples_predicted += len(Y)
        prediction_runtime += time() - t0_prediction
        elapsed = time() - t0_all
        fps = samples_predicted / elapsed
        print("Predicted: %d/%d frames | Elapsed: %.1f min / %.1f FPS / ETA: %.1f min" %
              (samples_predicted, num_samples, elapsed / 60, fps, (num_samples - samples_predicted) / fps / 60))
        
# Close video reader
reader.release()

# Merge arrays
positions_pred = np.concatenate(positions_pred, axis=0)
conf_pred = np.concatenate(conf_pred, axis=0)

# Report performance stats
print("Finished predicting %d frames." % samples_predicted)
print("    Prediction | Runtime: %.2f min / %.3f FPS" % (prediction_runtime / 60, samples_predicted / prediction_runtime))
print("    Reading    | Runtime: %.2f min / %.3f FPS" % (reading_runtime / 60, samples_predicted / reading_runtime))

# Save
if os.path.exists(save_path):
    os.remove(save_path)
with h5py.File(save_path, "w") as f:
        f.attrs["num_samples"] = num_samples
        f.attrs["video_path"] = video_path
        f.attrs["model_path"] = model_path

        ds_pos = f.create_dataset("positions_pred", data=positions_pred, compression="gzip", compression_opts=1)
        ds_pos.attrs["description"] = "coordinate of peak at each sample"
        ds_pos.attrs["dims"] = "(sample, [x, y], joint) === (sample, [column, row], joint)"

        ds_conf = f.create_dataset("conf_pred", data=conf_pred, compression="gzip", compression_opts=1)
        ds_conf.attrs["description"] = "confidence map value in [0, 1.0] at peak"
        ds_conf.attrs["dims"] = "(sample, joint)"

        total_runtime = time() - t0_all
        f.attrs["reading_runtime_secs"] = reading_runtime
        f.attrs["prediction_runtime_secs"] = prediction_runtime
        f.attrs["total_runtime_secs"] = total_runtime
        
    
print("Saved:", save_path)

print("Total runtime: %.1f mins" % (total_runtime / 60))
print("Total performance: %.3f FPS" % (samples_predicted / total_runtime))<jupyter_output>Model: D:/OneDrive/code/leap/data/BermanFlies/models/180615_025354-n=1500/final_model.h5
    Input: (None, 192, 192, 1)
    Output: (None, 3, 32)
Predicted: 10000/361000 frames | Elapsed: 0.4 min / 391.1 FPS / ETA: 15.0 min
Predicted: 20000/361000 frames | Elapsed: 0.8 min / 435.0 FPS / ETA: 13.1 min
Predicted: 30000/361000 frames | Elapsed: 1.1 min / 451.0 FPS / ETA: 12.2 min
Predicted: 40000/361000 frames | Elapsed: 1.5 min / 459.0 FPS / ETA: 11.7 min
Predicted: 50000/361000 frames | Elapsed: 1.8 min / 464.7 FPS / ETA: 11.2 min
Predicted: 60000/361000 frames | Elapsed: 2.1 min / 468.7 FPS / ETA: 10.7 min
Predicted: 70000/361000 frames | Elapsed: 2.5 min / 471.4 FPS / ETA: 10.3 min
Predicted: 80000/361000 frames | Elapsed: 2.8 min / 473.5 FPS / ETA: 9.9 min
Predicted: 90000/361000 frames | Elapsed: 3.2 min / 474.8 FPS / ETA: 9.5 min
Predicted: 100000/361000 frames | Elapsed: 3.5 min / 476.0 FPS / ETA: 9.1 min
Predicted: 110000/361000 frames | Elapsed: 3.8 min / 477.4 FPS / ETA: 8.8 mi[...]
{'fraction_non_alphanumeric': 0.12123166209898437, 'fraction_numerical': 0.06416250201515396, 'mean_word_length': 3.7686395080707147, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '626675', 'license': 'Apache-2.0', 'n_tokens_mistral': 2409, 'n_tokens_neox': 2125, 'n_words': 556, 'path': 'examples/batch_process_video.ipynb', 'repo_name': 'holmosaint/leap', 'size': '6203'}