<jupyter_start><jupyter_text>Web Scraping & Data Analysis with Selenium and Python Author: <NAME>Github: https://github.com/min2bro/WebScrapingwithSeleniumTwitter: @min2bro   IPython Notebook* Write, Edit, Replay python scripts* Interactive Data Visualization and report Presentation* Notebook can be saved and shared* Run Selenium Python Scripts Pandas Matplotlib   Analysis of the Filmfare Awards for Best Picture from 1955-2015   Web Scraping: Extracting Data from the Web<jupyter_code>%matplotlib inline 
from selenium import webdriver
import os,time,json
import pandas as pd
from collections import defaultdict,Counter
import matplotlib.pyplot as plt
url = "http://www.imdb.com/list/ls061683439/"
with open('./filmfare.json',encoding="utf-8") as f:
    datatbl = json.load(f)
driver = webdriver.Chrome(datatbl['data']['chromedriver'])
driver.get(url)<jupyter_output><empty_output><jupyter_text>Getting Data<jupyter_code>def ExtractText(Xpath):
    textlist=[]
    if(Xpath=="Movies_Runtime_Xpath"):
        [textlist.append(item.text[-10:-7]) for item in driver.find_elements_by_xpath(datatbl['data'][Xpath])]
    else:    
        [textlist.append(item.text) for item in driver.find_elements_by_xpath(datatbl['data'][Xpath])]
    return textlist
#Extracting Data from Web
Movies_Votes,Movies_Name,Movies_Ratings,Movies_RunTime=[[] for i in range(4)]
datarepo = [[]]*4
Xpath_list = ['Movies_Name_Xpath','Movies_Rate_Xpath','Movies_Runtime_Xpath','Movies_Votes_Xpath']

for i in range(4):
    if(i==3):
        driver.find_element_by_xpath(datatbl['data']['listview']).click()
    datarepo[i] = ExtractText(Xpath_list[i])
    
driver.quit()<jupyter_output><empty_output><jupyter_text>Store Data in a  Python Dictionary<jupyter_code># Result in a Python Dictionary
Years=range(2015,1954,-1)
result = defaultdict(dict)
for i in range(0,len(datarepo[0])):
    result[i]['Movie Name']= datarepo[0][i]
    result[i]['Year']= Years[i]
    result[i]['Rating']= datarepo[1][i]
    result[i]['Votes']= datarepo[3][i]
    result[i]['RunTime']= datarepo[2][i]<jupyter_output><empty_output><jupyter_text>Data before Clean Up<jupyter_code># Dictionary Result
print(json.dumps(result[59], indent=2))<jupyter_output>{
  "RunTime": "149",
  "Movie Name": "Boot Polish",
  "Votes": "498",
  "Rating": "8.1",
  "Year": 1956
}<jupyter_text>Clean Data<jupyter_code>for key,values in result.items():
    values['Votes'] = int(values['Votes'].replace(",",""))
    values['Rating']= float(values['Rating'])
    try:
        values['RunTime'] = int(values['RunTime'])
    except ValueError:
        values['RunTime'] = 0
        
    
print(json.dumps(result[0], indent=2))<jupyter_output>{
  "RunTime": 158,
  "Movie Name": "<NAME>",
  "Votes": 17265,
  "Rating": 7.2,
  "Year": 2015
}<jupyter_text>How does data in Dictionary looks like<jupyter_code># Dictionary Result
print(json.dumps(result[0], indent=2))<jupyter_output>{
  "RunTime": 158,
  "Movie Name": "<NAME>",
  "Votes": 17265,
  "Rating": 7.2,
  "Year": 2015
}<jupyter_text>Data in Pandas Dataframe<jupyter_code># create dataframe
df = pd.DataFrame.from_dict(result,orient='index')
df.sort_values(by='Year',ascending=True,inplace=True)
df = df[['Year', 'Movie Name', 'Rating', 'Votes','RunTime']]
df<jupyter_output><empty_output><jupyter_text>Movies with Highest Ratings<jupyter_code>#Highest Rating Movies
df.sort_values('Rating',ascending=[False]).head(5)<jupyter_output><empty_output><jupyter_text>Movies with Maximum Run time<jupyter_code>#Movies with maximum Run Time
df.sort_values(['RunTime'],ascending=[False]).head(10)<jupyter_output><empty_output><jupyter_text>Best Picture Run time<jupyter_code>df.plot(x=df.Year,y=['RunTime']);<jupyter_output><empty_output><jupyter_text>Best Picture Ratings<jupyter_code>#Rating Greater than 7
df[(df['Rating']>=7)]['Rating'].count()
#Create Rating Graph
Rating_Hist = defaultdict(dict)

Rating_Hist['Btwn 6&7'] = df[(df['Rating']>=6)&(df['Rating']<7)]['Rating'].count()
Rating_Hist['GTEQ 8'] = df[(df['Rating']>=8)]['Rating'].count()
Rating_Hist['Btwn 7 & 8'] = df[(df['Rating']>=7)&(df['Rating']<8)]['Rating'].count()

plt.bar(range(len(Rating_Hist)), Rating_Hist.values(), align='center',color='brown',width=0.4)
plt.xticks(range(len(Rating_Hist)), Rating_Hist.keys(), rotation=25);
#Histogram for average movie run timr
df['RunTime'].mean()<jupyter_output><empty_output><jupyter_text>Best Picture by Genre<jupyter_code># Movies by Genre
Category=Counter(datatbl['data']['Genre'])
df1 = pd.DataFrame.from_dict(Category,orient='index')
df1 = df1.sort_values([0],ascending=[False]).head(5)
df1.plot(kind='barh',color=['g','c','m']);
# Vote vs rating
%matplotlib inline 
import matplotlib.pyplot as plt
df.plot(x=df.Votes,y=['Rating'],kind='scatter');
# plt.scatter(df.Votes, df.Rating, s=df.Rating)
import numpy as np
df = pd.DataFrame(np.random.randint(100000, size=(10000, 2)), 
                  columns=['Votes', 'Rating'])

df.plot(kind='scatter', x='Votes', y='Rating', logx=True, alpha=0.5, color='purple', edgecolor='')
plt.ylabel('IMDB Rating')
plt.xlabel('Number of Votes')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1724677356276887, 'fraction_numerical': 0.02522487289792726, 'mean_word_length': 7.131955484896661, 'pattern_counts': {'":': 15, '<': 51, '<?xml version=': 0, '>': 53, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '282084', 'license': 'MIT', 'n_tokens_mistral': 2007, 'n_tokens_neox': 1866, 'n_words': 337, 'path': '.ipynb_checkpoints/Filmfare Data Analysis-checkpoint.ipynb', 'repo_name': 'pleabargain/WebScraping', 'size': '5136'}