<jupyter_start><jupyter_text>Simple example working with video data<jupyter_code>import findspark
findspark.init()

from pyspark.sql import DataFrame, SparkSession
from pyspark.sql.functions import udf, size, col
from pyspark.sql.types import (
    ArrayType,
    FloatType,
    IntegerType,
    LongType,
    StringType,
    StructField,
    StructType,
)

spark = (
    SparkSession
    .builder
    .appName('rikai-quickstart')
    .config('spark.jars.packages', 'ai.eto:rikai_2.12:0.0.3-SNAPSHOT')
    .master('local[*]')
    .getOrCreate()
)<jupyter_output><empty_output><jupyter_text>We'll work with a small set of music videos from The Hu<jupyter_code># Youtube video ids
the_hu = {
    'Sad But True': 'QpxA_ZxGX_M',
    'S<NAME>': 'YwutOqv4cGo',
    'The Great Chingghis Khaan': 'pD1gDSao1eA',
    'The Gereg': 'pu5jxc2pZtE',
    '<NAME>': 'jM8dCGIm6yc',
    'Yu<NAME>': 'v4xZUr0BEfE'
}
from rikai.spark.types import YouTubeVideoType
from rikai.types import YouTubeVideo
import pandas as pd

pdf = (pd.DataFrame({'vid': the_hu})
      .reset_index()
      .rename(columns={'index':'name'}))

pdf['youtube_video'] = pdf.vid.apply(YouTubeVideo)

schema = StructType(
    [
        StructField("name", StringType(), False),
        StructField("vid", StringType(), False),        
        StructField("youtube_video", YouTubeVideoType(), False)
    ]
)

df = spark.createDataFrame(pdf, schema=schema)<jupyter_output><empty_output><jupyter_text>We can read/write Rikai format with native video types<jupyter_code>df.coalesce(1).write.mode('overwrite').format('rikai').save('/tmp/thehu')
df = spark.read.format("rikai").load("/tmp/thehu/")
df.printSchema()
df.show()
yt = df.first()['youtube_video']
yt
v = yt.get_stream()
v<jupyter_output><empty_output><jupyter_text>We make it easy to sample the frames for deep learning<jupyter_code>from rikai.torch import DataLoader

data_loader = DataLoader(
    '/tmp/thehu',
    batch_size=1,
    shuffle=True
)
max_frames = 1 #
for batch in data_loader:
    yt = batch[0]['youtube_video']
    print(yt)
    for fno, frame in enumerate(yt.get_stream()):
        if fno >= max_frames:
            break
        print('fno: ', fno)
        print('frame: ', frame)<jupyter_output>2021-01-27 20:03:23,510 INFO Rikai (dataset.py:108): Loading parquet files: ['file:///tmp/thehu/part-00000-26b28f78-0cfb-43ae-be9d-0c97fe72878c-c000.snappy.parquet']
{'fraction_non_alphanumeric': 0.14285714285714285, 'fraction_numerical': 0.027230833682446585, 'mean_word_length': 4.225382932166302, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '359195', 'license': 'Apache-2.0', 'n_tokens_mistral': 934, 'n_tokens_neox': 889, 'n_words': 170, 'path': 'notebooks/VideoData.ipynb', 'repo_name': 'gitter-badger/rikai', 'size': '2401'}