<jupyter_start><jupyter_text>Main code starts here<jupyter_code>from PyDictionary import PyDictionary

dictionary=PyDictionary()
from bs4 import BeautifulSoup #for scraping
import requests               #required for reading the file
from selenium import webdriver 
import pandas as pd 
from selenium.webdriver.support.ui import WebDriverWait
import warnings
import time, random
warnings.filterwarnings("ignore")
options = webdriver.ChromeOptions()
options.add_argument("headless") #to stop opening new chrome browser on every hit
driver=webdriver.Chrome('D:\chromedriver.exe', chrome_options=options) #change path location to your path
from selenium.webdriver.common.keys import Keys
import json, pickle
from textblob import TextBlob
from textblob import Word
import mysql.connector as db
import numpy as np
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
# options = webdriver.ChromeOptions()
# options.add_argument("start-maximized")
# options.add_argument("headless") #to stop opening new chrome browser on every hit
# options.add_experimental_option("excludeSwitches", ["enable-automation"])
# options.add_experimental_option('useAutomationExtension', False)
# driver = webdriver.Chrome(chrome_options=options, executable_path=r'D:\chromedriver.exe')
# driver.get("https://sslproxies.org/")
# driver.execute_script("return arguments[0].scrollIntoView(true);", WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.XPATH, "//table[@class='table table-striped table-bordered dataTable']//th[contains(., 'IP Address')]"))))
# ips = [my_elem.get_attribute("innerHTML") for my_elem in WebDriverWait(driver, 5).until(EC.visibility_of_all_elements_located((By.XPATH, "//table[@class='table table-striped table-bordered dataTable']//tbody//tr[@role='row']/td[position() = 1]")))]
# ports = [my_elem.get_attribute("innerHTML") for my_elem in WebDriverWait(driver, 5).until(EC.visibility_of_all_elements_located((By.XPATH, "//table[@class='table table-striped table-bordered dataTable']//tbody//tr[@role='row']/td[position() = 2]")))]
# driver.quit()
# proxies = []
# for i in range(0, len(ips)):
#     proxies.append(ips[i]+':'+ports[i])
mydb = db.connect(host="video-scrapping-db.cnmkmpcubdot.us-east-2.rds.amazonaws.com",
  user="asmi",
  password="<PASSWORD>")

mycursor = mydb.cursor()
try:
    mycursor.execute("CREATE DATABASE YOUTUBE")
except:
    pass
# Create table as per requirement
sql = """CREATE TABLE IF NOT EXISTS nissan_car (
    id INT(80) primary key NOT NULL AUTO_INCREMENT,
    object_name VARCHAR(100),
    timestamp VARCHAR(100),
    video_ID VARCHAR(80),
    link VARCHAR(100),
    title VARCHAR(200),
    description VARCHAR(200),
    channel_id VARCHAR(100),
    publishedAt VARCHAR(100),
    thumbnailUrl VARCHAR(100),
    channelTitle VARCHAR(200),
    tags VARCHAR(100),
    categoryId VARCHAR(100),
    viewCount VARCHAR(100),
    likeCount VARCHAR(100),
    dislikeCount VARCHAR(100),
    favoriteCount VARCHAR(100),
    commentCount VARCHAR(100),
    uploadStatus VARCHAR(70),
    privacyStatus VARCHAR(100),
    license VARCHAR(100),
    embeddable VARCHAR(100),
    publicStatsviewable VARCHAR(100)
    )"""

# Create table as per requirement
conn = db.connect(host="video-scrapping-db.cnmkmpcubdot.us-east-2.rds.amazonaws.com",
  user="asmi",
  password="<PASSWORD>", database='METADATA')
cur = conn.cursor()
cur.execute(sql)
cur.close()
conn.commit()
conn.close()
all_links = []
links = []
ID = []
title = []
description = []
channel_id = []
publishedAt = []
thumbnailUrl = []
channelTitle = []
tags = []
categoryId = []
viewCount = []
likeCount = []
dislikeCount = []
favoriteCount = []
commentCount = []
uploadStatus = []
privacyStatus = []
license = []
embededdable = []
publicStatsViewable = []
i_key = 0 
#unique_ids = pickle.load(open('unique_ids.pickle', 'rb'))
#df = pd.read_csv('df.csv')
unique_ids_nissan = set()

df_nissan = pd.DataFrame(columns = ['object_name','timestamp','video_ID', 'link', 'title', 'description', 'channel_id', 'publishedAt', 'thumbnailUrl', 'channelTitle', 'tags', 'categoryId', 'viewCount', 'likeCount', 'dislikeCount', 'favoriteCount', 'commentCount', 'uploadStatus', 'privacyStatus', 'license', 'embeddable', 'publicStatsviewable'])
df_nissan.shape
def get_scrapper(ids, query):
    print(query)
    print(ids)
    global i_key
    categories = ['snippet', 'statistics', 'status']
    keys = [
        '<KEY>',
        '<KEY>',
        '<KEY>',
        '<KEY>',
        '<KEY>',
        '<KEY>',
        '<KEY>'
    ]
    print(keys[i_key])
    for cat in categories:
        driver.get("https://www.googleapis.com/youtube/v3/videos?part=" + cat + "&id=" + ids + "&key="+ keys[i_key])
       
        soup = BeautifulSoup(driver.page_source, 'html.parser')
        user_data = soup.find('pre').contents[0]
        user_data = user_data.replace('\n','')

        user_data = json.loads(user_data)
        
        if user_data.get('items'):
            items = user_data['items'][0]
            if cat == 'snippet':
                #for snippet
                local_ID = items['id']
                local_title = items['snippet']['title']
                local_description = items.get('snippet').get('description', "Not Found")
                local_channel_id = items['snippet']['channelId']
                local_publishedAt = items['snippet']['publishedAt']
                local_thumbnailUrl = items['snippet']['thumbnails']['default']['url']
                local_channelTitle = items['snippet']['channelTitle']
                local_tag = items.get('snippet').get('tags',["Not found"])
                local_tags = ','.join(local_tag)
                local_categoryId = items['snippet']['categoryId']

            elif cat == 'statistics':
                #for statistics
                local_Count = items.get('statistics').get('viewCount',0) 
                local_likeCount = items.get('statistics').get('likeCount',0) 
                local_dislikeCount = items.get('statistics').get('dislikeCount',0)
                local_favoriteCount = items.get('statistics').get('favoriteCount',0)
                local_commentCount = items.get('statistics').get('commentCount',0) 

            else:
                #for status
                local_uploadStatus = items['status']['uploadStatus']
                local_privacyStatus = items['status']['privacyStatus']
                local_license = items['status']['license']
                local_embededdable = items['status']['embeddable']
                local_publicStatsViewable = items['status']['publicStatsViewable']
                
        else:
            i_key+=1
            if i_key>=len(keys):
                return "All API key visited for today"
            return
            
        
    total_data = [local_ID, local_title, local_description, local_channel_id, local_publishedAt, local_thumbnailUrl, 
                  local_channelTitle, local_tags, local_categoryId, 
                  local_Count, local_likeCount, local_dislikeCount, local_favoriteCount, 
                  local_commentCount, local_uploadStatus, local_privacyStatus,
                  local_license, local_embededdable, local_publicStatsViewable
                 ]
    return total_data
from selenium.webdriver.common.keys import Keys

def scrapper(t, query):

    driver.get(f"https://www.youtube.com/results?search_query={query}")
    
    for i in range(1,400):  #Scroll 400 times
        html = driver.find_element_by_tag_name('html')
        html.send_keys(Keys.END)

    user_data = driver.find_elements_by_xpath('//*[@id="video-title"]')

    for i1 in user_data:
        links.append(i1.get_attribute('href'))
    
    new_links = [link for link in set(links) if link is not None]
    IDs = [link.split('=')[1] for link in set(links) if link is not None]
    
    for i, link in zip(IDs, new_links):
        global unique_ids_nissan
        if i in unique_ids_nissan:
            continue
        unique_ids_nissan.add(i)
        total_data = get_scrapper(i, query)
        if total_data is None:
            total_data = get_scrapper(i, query)
        if total_data == 'All API key visited for today':
            return 'All API key visited for today'
        
        # Connecting to MYSQL database
        conn = db.connect(host="video-scrapping-db.cnmkmpcubdot.us-east-2.rds.amazonaws.com",
          user="asmi",
          password="<PASSWORD>", database='METADATA')
        cur = conn.cursor()
        mySql_insert_query = """INSERT INTO nissan_car(object_name, timestamp, video_ID, link, title, description, channel_id, publishedAt,
                        thumbnailUrl, channelTitle, tags, categoryId, viewCount, likeCount, dislikeCount, 
                        favoriteCount, commentCount, uploadStatus, privacyStatus, license, embeddable, publicStatsviewable) 
                        VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s ,%s, %s, %s, %s,%s, %s, %s, %s) """

        meta_datas = (query, t, total_data[0], link, total_data[1], total_data[2], total_data[3],
                     total_data[4], total_data[5], total_data[6], total_data[7], total_data[8], total_data[9],
                    total_data[10], total_data[11], total_data[12], total_data[13], total_data[14], total_data[15],
                    total_data[16], total_data[17], total_data[18])
        cur.execute(mySql_insert_query, meta_datas)
        cur.close()
        conn.commit()
        conn.close()
        
        global df_nissan
        df_nissan = df_nissan.append({'object_name':query, 'timestamp':t, 'video_ID': total_data[0], 'link':link,'title':total_data[1], 
                        'description':total_data[2], 'channel_id':total_data[3], 'publishedAt':total_data[4], 
                        'thumbnailUrl':total_data[5], 'channelTitle':total_data[6], 'tags':total_data[7], 
                        'categoryId':total_data[8], 'viewCount':total_data[9], 'likeCount':total_data[10],
                        'dislikeCount':total_data[11], 'favoriteCount':total_data[12], 'commentCount':total_data[13], 
                        'uploadStatus':total_data[14], 'privacyStatus':total_data[15], 'license':total_data[16], 
                        'embeddable':total_data[17], 'publicStatsviewable':total_data[18]}, ignore_index=True)
i_key
timestamp = ['0:01', '0:25','2:02', '2:03','2:04', '2:04', '2:05','2:06','2:06']

final_keywords = ['nissan sentra 2020', 'white tshirt','nissan altima', 'kia forte 5', 'subaru outback', 'golf alltrack',
                 'prius prime', 'hyundai sonata', 'toyota corolla']

for t,word in zip(timestamp,final_keywords):
    a = scrapper(t,word)
    if a == 'All API key visited for today':
        print("All API key visited for today")
        break
import pickle
with open('unique_ids_nissan.pickle', 'wb') as f:
    pickle.dump(unique_ids_nissan, f)
    
df_nissan.to_csv("df_nissan.csv", index=False)
print(len(unique_ids))
df.head()
df.shape
conn = db.connect(host="video-scrapping-db.cnmkmpcubdot.us-east-2.rds.amazonaws.com",
  user="asmi",
  password="<PASSWORD>", database='METADATA')
cur = conn.cursor()
df = pd.read_sql('SELECT * FROM nissan_car', con=conn)
df
df.to_csv("df_nissan_car.csv", index=False)
print(len(list(set(links))))
print(len(new_links))
links[-1]
def get_scrapper(ids):
    categories = ['snippet', 'statistics', 'status']
    for cat in categories:
        driver.get("https://www.googleapis.com/youtube/v3/videos?part=" + cat + "&id=" + ids + "&key=<KEY>")

        soup = BeautifulSoup(driver.page_source, 'html.parser')
        user_data = soup.find('pre').contents[0]
        user_data = user_data.replace('\n','')

        user_data = json.loads(user_data)

        items = user_data['items'][0]

        if cat == 'snippet':
            #for snippet
            local_ID = items['id']
            local_title = items['snippet']['title']
            local_description = items['snippet']['description']
            local_channel_id = items['snippet']['channelId']
            local_publishedAt = items['snippet']['publishedAt']
            local_thumbnailUrl = items['snippet']['thumbnails']['default']['url']
            local_channelTitle = items['snippet']['channelTitle']
            local_tags = items['snippet']['tags']
            local_categoryId = items['snippet']['categoryId']

        elif cat == 'statistics':
            #for statistics
            local_viewCount = items['statistics']['viewCount']
            local_likeCount = items['statistics']['likeCount']
            local_dislikeCount = items['statistics']['dislikeCount']
            local_favoriteCount = items['statistics']['favoriteCount']
            local_commentCount = items['statistics']['commentCount']

        else:
            #for status
            local_uploadStatus = items['status']['uploadStatus']
            local_privacyStatus = items['status']['privacyStatus']
            local_license = items['status']['license']
            local_embededdable = items['status']['embeddable']
            local_publicStatsViewable = items['status']['publicStatsViewable']
            
        total_data = [local_ID, local_title, local_description, local_channel_id, local_publishedAt, local_thumbnailUrl, 
                      local_channelTitle, local_tags, local_categoryId, 
                      local_viewCount, local_likeCount, local_dislikeCount, local_favoriteCount, 
                      local_commentCount, local_uploadStatus, local_privacyStatus,
                      local_license, local_embededdable, local_publicStatsViewable
                     ]
        return total_data
print(len(head))
print(len(links))
print(len(set(head)))
print(head[1])
print(head[-20:-1])
df['ids'] = ids
df.head()
# wait = WebDriverWait(driver, 10)
# v_category = "CATEGORY_NAME"
# for x in links:
#     driver.get(x)
#     v_id = x.strip('https://www.youtube.com/watch?v=')
#     v_title = wait.until(EC.presence_of_element_located(
#                (By.CSS_SELECTOR,"h1.title yt-formatted-string"))).text
#     v_description =  wait.until(EC.presence_of_element_located(
#                              (By.CSS_SELECTOR,"div#description 
#                               yt-formatted-string"))).text
#     df.loc[len(df)] = [v_id, v_title, v_description, v_category]
driver.get("about:config");<jupyter_output><empty_output><jupyter_text>https://www.googleapis.com/youtube/v3/videos?part=snippet&id=XbM6PyRsJc4&key=AIzaSyAKkUeSAkyieoS9Ebe6HEBkjQkkae1yK4M<jupyter_code>cats = ['snippet', 'statistics', 'contentDetails','fileDetails', 'player', 
        'processingDetails', 'recordingDetails','status','suggestions', 'topicDetails']
driver.get("https://www.googleapis.com/youtube/v3/videos?part=" + cats[9] + "&id=" + id + "&key=<KEY>")
soup = BeautifulSoup(driver.page_source, 'html.parser')
user_data = soup.find('pre').contents[0]
user_data = user_data.replace('\n','')

import json
user_data = json.loads(user_data)
user_data
items = user_data['items'][0]
items
#snippet ko lagi
ID = items['id']
title = items['snippet']['title']
description = items['snippet']['description']
channel_id = items['snippet']['channelId']
publishedAt = items['snippet']['publishedAt']
thumbnailUrl = items['snippet']['thumbnails']['default']['url']
channelTitle = items['snippet']['channelTitle']
tags = items['snippet']['tags']
categoryId = items['snippet']['categoryId']

#statistics ko lagi
viewCount = items['statistics']['viewCount']
likeCount = items['statistics']['likeCount']
dislikeCount = items['statistics']['dislikeCount']
favoriteCount = items['statistics']['favoriteCount']
commentCount = items['statistics']['commentCount']

#status ko lagi
uploadStatus = items['status']['uploadStatus']
privacyStatus = items['status']['privacyStatus']
license = items['status']['license']
embededdable = items['status']['embeddable']
publicStatsViewable = items['status']['embeddable']
from PyDictionary import PyDictionary
from textblob import TextBlob
from textblob import Word
 
keywords_list = []
a ='ronaldo'
text_word = Word(a)
 
synonyms = set()
for synset in text_word.synsets:
    for lemma in synset.lemmas():
        synonyms.add(lemma.name())
         
if len(list(synonyms)) == 0:
    keywords_list.append(a)
else:
    keywords_list.append(a)
    keywords_list.extend(list(synonyms))

dictionary=PyDictionary()

answer = dictionary.synonym(a)
if answer is None:
    keywords_list.append(a)
else:
    keywords_list.append(a)
    keywords_list.extend(answer)
    
print(list(set(keywords_list)))
from textblob import TextBlob
from textblob import Word
 
synm = []
a = 'car'
text_word = Word(a)
 
synonyms = set()
for synset in text_word.synsets:
    for lemma in synset.lemmas():
        synonyms.add(lemma.name())
         
if len(list(synonyms)) == 0:
    synm.append(a)
else:
    synm.append(a)
    synm.extend(list(synonyms))

print(list(set(synm)))
a = [1,2,3,4]
b = [1,2,3,4]
a.extend(b)

print(a)
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException

options = webdriver.ChromeOptions()
options.add_argument("start-maximized")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
driver = webdriver.Chrome(chrome_options=options, executable_path=r'D:\chromedriver.exe')
driver.get("https://sslproxies.org/")
driver.execute_script("return arguments[0].scrollIntoView(true);", WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.XPATH, "//table[@class='table table-striped table-bordered dataTable']//th[contains(., 'IP Address')]"))))
ips = [my_elem.get_attribute("innerHTML") for my_elem in WebDriverWait(driver, 5).until(EC.visibility_of_all_elements_located((By.XPATH, "//table[@class='table table-striped table-bordered dataTable']//tbody//tr[@role='row']/td[position() = 1]")))]
ports = [my_elem.get_attribute("innerHTML") for my_elem in WebDriverWait(driver, 5).until(EC.visibility_of_all_elements_located((By.XPATH, "//table[@class='table table-striped table-bordered dataTable']//tbody//tr[@role='row']/td[position() = 2]")))]
driver.quit()
proxies = []
for i in range(0, len(ips)):
    proxies.append(ips[i]+':'+ports[i])
print(proxies)
for i in range(0, len(proxies)):
    try:
        print("Proxy selected: {}".format(proxies[i]))
        options = webdriver.ChromeOptions()
        options.add_argument('--proxy-server={}'.format(proxies[i]))
        options.add_argument("headless") #to stop opening new chrome browser on every hit
        driver = webdriver.Chrome(options=options, executable_path=r'D:\chromedriver.exe')
        driver.get('https://www.youtube.com/results?search_query=ronaldo')
        if "Proxy Type" in WebDriverWait(driver, 5).until(EC.visibility_of_element_located((By.CSS_SELECTOR, "p.card-text"))):
            break
    except Exception:
        driver.quit()
import random

a = [1,2,3,4,5,6,7]

print(random.choice(a))
import requests
from bs4 import BeautifulSoup
import urllib
import os
from random import choice


def get_proxy():
	proxy_url='https://www.sslproxies.org/'
	r = requests.get(proxy_url)
	soup = BeautifulSoup(r.content, 'html5lib')
	proxy = {'https': choice(list(map(lambda x:x[0]+':'+x[1], list(zip(map(lambda x:x.text, soup.findAll('td')[::8]),
		   									   map(lambda x:x.text, soup.findAll('td')[1::8]))))))}
	
	#print(proxy)
	return proxy


''' As we get a proxy lets do arequest using proxy '''
while True:
	try:
		proxy = get_proxy()
		url ="https://www.expressvpn.com/what-is-my-ip"		
		print('Using Proxy:{}'.format(proxy))
		r = requests.get(url, proxies=proxy, timeout=5)
		print(r.json())
		html = r.content
		soup = BeautifulSoup(html, 'html5lib')
		ip = soup.find('p', {'class': 'ip-address'})		
		break
	except:
		pass
print("Your ip is:",ip)<jupyter_output>Using Proxy:{'https': '172.16.17.32:4145'}
Using Proxy:{'https': '192.168.3.11:3128'}
Using Proxy:{'https': '192.168.3.11:3128'}
Using Proxy:{'https': '192.168.127.12:8080'}
Using Proxy:{'https': '172.16.31.10:6666'}
Using Proxy:{'https': '172.16.17.32:3128'}
Using Proxy:{'https': '172.16.58.3:42119'}
Using Proxy:{'https': '192.168.3.11:8080'}
Using Proxy:{'https': '172.16.58.3:80'}
Using Proxy:{'https': '172.16.17.32:3129'}
{'fraction_non_alphanumeric': 0.14947501458292825, 'fraction_numerical': 0.018180050554151273, 'mean_word_length': 3.798926988570096, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 22, 'https://': 11, 'lorem ipsum': 0, 'www.': 9, 'xml': 0}, 'pii_count': 10, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '91016', 'license': 'Apache-2.0', 'n_tokens_mistral': 6875, 'n_tokens_neox': 6504, 'n_words': 1234, 'path': 'Nissan Car Video.ipynb', 'repo_name': 'PDahal2871/Web-Scrapping', 'size': '20901'}