<jupyter_start><jupyter_text>CC_WEB VIDEO EDA<jupyter_code>import pandas as pd
from glob import glob
from encodings.aliases import aliases
glob('../data/external/CC_WEB_VIDEO/**.txt')
SHOT_INFO = '../data/external/CC_WEB_VIDEO/Shot_Info.txt'
VIDEO_COMPLETE = '../data/external/CC_WEB_VIDEO/Video_Complete.txt'
VIDEO_LIST = '../data/external/CC_WEB_VIDEO/Video_List.txt'
SEED = '../data/external/CC_WEB_VIDEO/Seed.txt'
GROUND_TRUTH = '../data/external/CC_WEB_VIDEO/GT/GT_*.st'
video_df = pd.read_csv(VIDEO_LIST,sep='\t',header=None)
video_df.head()
video_complete_df = pd.read_csv(VIDEO_COMPLETE,error_bad_lines=False,sep='\t')
video_complete_df.shape
video_complete_df[video_complete_df['Category'].isnull()]
video_complete_df.head(1)
video_complete_df['TopicID'].value_counts()
video_complete_df['VideoName'].value_counts().sort_values(ascending=False)<jupyter_output><empty_output><jupyter_text>Ground Truth Labels<jupyter_code>gt_files = glob('../data/external/CC_WEB_VIDEO/GT/GT_*.rst')
gt_files
pd.read_csv(gt_files[0],sep='\t',he)
ground_truth_df.shape
ground_truth_df = pd.concat([pd.read_csv(x,sep='\t',header=None) for x in gt_files],)
ground_truth_df.dropna(inplace=True,axis=1)
ground_truth_df[1].value_counts()
ground_truth_df.columns = ['ID','Label']
video_complete_df.head()
video_complete_df = video_complete_df.merge(ground_truth_df,on='ID')
video_complete_df.shape
video_complete_df.to_csv('cc_video_labels.csv')<jupyter_output><empty_output><jupyter_text>Downloading the files http://vireo.cs.cityu.edu.hk/webvideo/videos/QueryID/VideoName<jupyter_code>import requests
import shutil
import os
THIS_FOLDER = os.path.dirname(os.path.abspath(''))
SAVE_FOLDER = os.path.join(THIS_FOLDER, 'data/external/CC_WEB_VIDEO/videos')



def download_file(local_filename,url):
    r = requests.get(url, stream=True)
    with open(local_filename, 'wb') as f:
        shutil.copyfileobj(r.raw, f)

    return local_filename
SAVE_FOLDER
for row in video_complete_df.iterrows():
    _,data = row
    total = len(video_complete_df)
    download_url = "http://vireo.cs.cityu.edu.hk/webvideo/videos/{}/{}".format(data['TopicID'],data['VideoName'])
    local_path = os.path.join(SAVE_FOLDER, data['VideoName'])
    
    print('Downloading file {} of {}'.format(_,total))
    download_file(local_path,download_url)<jupyter_output>Downloading file 0 of 13129
Downloading file 1 of 13129
Downloading file 2 of 13129
Downloading file 3 of 13129
Downloading file 4 of 13129
Downloading file 5 of 13129
Downloading file 6 of 13129
Downloading file 7 of 13129
Downloading file 8 of 13129
Downloading file 9 of 13129
Downloading file 10 of 13129
Downloading file 11 of 13129
Downloading file 12 of 13129
Downloading file 13 of 13129
Downloading file 14 of 13129
Downloading file 15 of 13129
Downloading file 16 of 13129
Downloading file 17 of 13129
Downloading file 18 of 13129
Downloading file 19 of 13129
Downloading file 20 of 13129
Downloading file 21 of 13129
Downloading file 22 of 13129
Downloading file 23 of 13129
Downloading file 24 of 13129
Downloading file 25 of 13129
Downloading file 26 of 13129
Downloading file 27 of 13129
Downloading file 28 of 13129
Downloading file 29 of 13129
Downloading file 30 of 13129
Downloading file 31 of 13129
Downloading file 32 of 13129
Downloading file 33 of 13129
Downloading file 34 of 1[...]
{'fraction_non_alphanumeric': 0.10510510510510511, 'fraction_numerical': 0.07057057057057058, 'mean_word_length': 6.163440860215053, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 41, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '66679', 'license': 'MIT', 'n_tokens_mistral': 1390, 'n_tokens_neox': 1183, 'n_words': 276, 'path': 'notebooks/CC_WEB_VIDEO EDA-Copy1.ipynb', 'repo_name': 'fsbatista/VideoDeduplication', 'size': '3330'}