<jupyter_start><jupyter_text><jupyter_code>!wget -P /root/input/ -c "https://www.rocq.inria.fr/cluster-willow/amiech/howto100m/HowTo100M.zip"
!unzip '/root/input/HowTo100M.zip'
import pandas as pd 
df= pd.read_csv('HowTo100M_v1.csv')
tasks_df=pd.read_csv('task_ids.csv',sep='\t')
df[df['category_1']=='Cars & Other Vehicles'].category_2.unique()
df['category_1'].unique()
#!/usr/bin/python
# -*- coding: utf-8 -*-
 
import urllib.request
import urllib.parse
import urllib.error
from bs4 import BeautifulSoup
import ssl
import json
import ast
import json
import os
from urllib.request import Request, urlopen
from datetime import date
#importing the libraries
from urllib.request import urlopen
from bs4 import BeautifulSoup

# For ignoring SSL certificate errors
 
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
 
 
def get_data(url):

    req = Request(url, headers={'User-Agent': 'Mozilla/5.0'})
    webpage = urlopen(req).read()
 
    # Creating a BeautifulSoup object of the html page for easy extraction of data.
 
    soup = BeautifulSoup(webpage, 'html.parser')
    html = soup.prettify('utf-8')
    video_details = {}
    other_details = {}
    for span in soup.findAll('span',attrs={'class': 'watch-title'}):
        video_details['TITLE'] = span.text.strip()
    for script in soup.findAll('script',attrs={'type': 'application/ld+json'}):
            channelDesctiption = json.loads(script.text.strip())
            video_details['CHANNEL_NAME'] = channelDesctiption['itemListElement'][0]['item']['name']
    for div in soup.findAll('div',attrs={'class': 'watch-view-count'}):
        video_details['NUMBER_OF_VIEWS'] = div.text.strip()
    for button in soup.findAll('button',attrs={'title': 'I like this'}):
        video_details['LIKES'] = button.text.strip()
    for button in soup.findAll('button',attrs={'title': 'I dislike this'}):
        video_details['DISLIKES'] = button.text.strip()
    for span in soup.findAll('span',attrs={'class': 'yt-subscription-button-subscriber-count-branded-horizontal yt-subscriber-count'}):
        video_details['NUMBER_OF_SUBSCRIPTIONS'] = span.text.strip()
    hashtags = []
    for span in soup.findAll('span',attrs={'class': 'standalone-collection-badge-renderer-text'}):
        for a in span.findAll('a',attrs={'class': 'yt-uix-sessionlink'}):
            hashtags.append(a.text.strip())
    video_details['HASH_TAGS'] = hashtags
    return video_details
#importing the libraries
from urllib.request import urlopen
from bs4 import BeautifulSoup

def get_tags(url):    
    webpage = urlopen(url).read()
    soup = BeautifulSoup(webpage, "lxml")
    for tag in soup.find_all("article") :
        id = tag.get('id')
    tag = soup.find_all("meta",  property="og:video:tag")
    description =soup.find_all("meta",property="og:description")
    if (len(description)>0 and "content" in description[0]):
      data = description[0]["content"]
    else:
      data=""
    return[x['content'] for x in tag],data 

get_tags('https://www.youtube.com/watch?v=r0tsAYv67FU')
video_df=pd.DataFrame(columns=['id','Title','url','channel','likes','description'])
keywords=pd.DataFrame()
video_df.columns

def getlikes(video_details):
  if 'Likes' in video_details:
    return video_details['LIKES']
  else:
    return 0

for video in df[(df['category_1']=='Cars & Other Vehicles') & (df['category_2'].isin(['Motorcycles','Cars', 'Bicycles','Trucks' ]))].sort_values(['category_1','category_2','rank'],ascending=[True,True,True]).reset_index().head(100).itertuples():
  url="https://www.youtube.com/watch?v={0}".format(video.video_id)
  video_details = get_data(url)
  if ('TITLE' in video_details and video.video_id not in video_df['id']):
    key_words,description = get_tags(url)
    print(video.video_id)
    row = {'id':video.video_id,'Title':video_details['TITLE'],'url':url,'channel':video_details['CHANNEL_NAME'],'likes':getlikes(video_details),'description':description,'category':video.category_1,'views':video_details['NUMBER_OF_VIEWS'],'channelid':''}
    video_df.loc[len(video_df)]=row;
    kw=pd.DataFrame()
    kw['Tag']=key_words
    kw['video_id']=video.video_id
    keywords=pd.concat([keywords,kw],axis=0,ignore_index=True)
  else:
    video_df.loc[video_df['id']==video.video_id]['category']=video.category_1

video_df.shape
video_df.to_csv('video.csv')
keywords.to_csv('video_keywords.csv')
!pip install youtube-data-api
# AIzaSyBLH5XsMarM0HurtVTxSmk-XUiDFwLYNW8
from youtube_api import YouTubeDataAPI

api_key = '<KEY>'
yt = YouTubeDataAPI(api_key)
video_df['views']=''
video_df['category']=''
video_df['channelid']=''
for video in video_df.itertuples():
  try:
    metadata = yt.get_video_metadata(video.id)
    print(metadata['channel_id'],metadata['video_category'])
    video_df.loc[video_df['id']==video.id,'likes']=metadata['video_like_count']
    video_df.loc[video_df['id']==video.id,'views']=metadata['video_view_count']
    video_df.loc[video_df['id']==video.id,'category']=metadata['video_category']
    video_df.loc[video_df['id']==video.id,'channelid']=metadata['channel_id']
    video_df.loc[video_df['id']==video.id,'description']=metadata['video_description']
  except:
    print('error')

video_df[video_df['category']=='22'].count()
json_data=[{
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/Xy1mB4_yLrHy_BmKmPBggty2mZQ\"",
      "id": "1",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "Film & Animation" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/UZ1oLIIz2dxIhO45ZTFR3a3NyTA\"",
      "id": "2",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "Autos & Vehicles" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/nqRIq97-xe5XRZTxbknKFVe5Lmg\"",
      "id": "10",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "Music" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/HwXKamM1Q20q9BN-oBJavSGkfDI\"",
      "id": "15",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "Pets & Animals" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/9GQMSRjrZdHeb1OEM1XVQ9zbGec\"",
      "id": "17",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "Sports" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/FJwVpGCVZ1yiJrqZbpqe68Sy_OE\"",
      "id": "18",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "Short Movies" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/M-3iD9dwK7YJCafRf_DkLN8CouA\"",
      "id": "19",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "Travel & Events" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/WmA0qYEfjWsAoyJFSw2zinhn2wM\"",
      "id": "20",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "Gaming" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/EapFaGYG7K0StIXVf8aba249tdM\"",
      "id": "21",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "Videoblogging" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/xId8RX7vRN8rqkbYZbNIytUQDRo\"",
      "id": "22",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "People & Blogs" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/G9LHzQmx44rX2S5yaga_Aqtwz8M\"",
      "id": "23",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "Comedy" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/UVB9oxX2Bvqa_w_y3vXSLVK5E_s\"",
      "id": "24",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "Entertainment" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/QiLK0ZIrFoORdk_g2l_XR_ECjDc\"",
      "id": "25",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "News & Politics" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/r6Ck6Z0_L0rG37VJQR200SGNA_w\"",
      "id": "26",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "Howto & Style" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/EoYkczo9I3RCf96RveKTOgOPkUM\"",
      "id": "27",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "Education" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/w5HjcTD82G_XA3xBctS30zS-JpQ\"",
      "id": "28",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "Science & Technology" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/SalkJoBWq_smSEqiAx_qyri6Wa8\"",
      "id": "29",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "Nonprofits & Activism" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/lL7uWDr_071CHxifjYG1tJrp4Uo\"",
      "id": "30",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "Movies" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/WnuVfjO-PyFLO7NTRQIbrGE62nk\"",
      "id": "31",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "Anime/Animation" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/ctpH2hGA_UZ3volJT_FTlOg9M00\"",
      "id": "32",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "Action/Adventure" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/L0kR3-g1BAo5UD1PLVbQ7LkkDtQ\"",
      "id": "33",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "Classics" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/pUZOAC_s9sfiwar639qr_wAB-aI\"",
      "id": "34",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "Comedy" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/Xb5JLhtyNRN3AQq021Ds-OV50Jk\"",
      "id": "35",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "Documentary" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/u8WXzF4HIhtEi805__sqjuA4lEk\"",
      "id": "36",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "Drama" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/D04PP4Gr7wc4IV_O9G66Z4A8KWQ\"",
      "id": "37",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "Family" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/i5-_AceGXQCEEMWU0V8CcQm_vLQ\"",
      "id": "38",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "Foreign" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/rtlxd0zOixA9QHdIZB26-St5qgQ\"",
      "id": "39",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "Horror" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/N1TrDFLRppxZgBowCJfJCvh0Dpg\"",
      "id": "40",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "Sci-Fi/Fantasy" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/7UMGi6zRySqXopr_rv4sZq6Za2E\"",
      "id": "41",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "Thriller" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/RScXhi324h8usyIetreAVb-uKeM\"",
      "id": "42",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "Shorts" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/0n9MJVCDLpA8q7aiGVrFsuFsd0A\"",
      "id": "43",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "Shows" 
      }
    },
    {
      "kind": "youtube#videoCategory",
      "etag": "\"Dn5xIderbhAnUk5TAW0qkFFir0M/x5NxSf5fz8hn4loSN4rvhwzD_pY\"",
      "id": "44",
      "snippet": {
        "channelId": "UCBR8-60-B28hp2BmDPdntcQ",
        "title": "Trailers" 
      }
    }]
categories_df=pd.DataFrame(json_data)
categories_df['category_name']=categories_df['snippet'].apply(lambda x: x['title'])
video_df[]
video_result=pd.merge(video_df, categories_df, how='inner', left_on='category', right_on='id',
         validate=None)
video_result.to_csv('video.csv')

# video_result=video_result.drop(columns=['category_x','category_y','snippet','kind','etag'])
video_result=video_result.rename(columns={'id_y':'cateogry_id'})
!pip install google-api-python-client<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17585810713772798, 'fraction_numerical': 0.04150509837713629, 'mean_word_length': 3.247331503507167, 'pattern_counts': {'":': 192, '<': 6, '<?xml version=': 0, '>': 7, 'https://': 3, 'lorem ipsum': 0, 'www.': 3, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '196150', 'license': 'Apache-2.0', 'n_tokens_mistral': 6158, 'n_tokens_neox': 5842, 'n_words': 671, 'path': 'video_curation.ipynb', 'repo_name': 'mariamadalina/pytrends', 'size': '13960'}