<jupyter_start><jupyter_text>Scraping IMDB website using BeautifulSoup<jupyter_code>from requests import get
url = "http://www.imdb.com/search/title?release_date=2017&sort=num_votes,desc&page=1"<jupyter_output><empty_output><jupyter_text>Inspecting the linkThe search is detailed in the link http://www.imdb.com/search/title?release_date=2017&sort=num_votes,desc&page=1<jupyter_code>response = get(url)
response<jupyter_output><empty_output><jupyter_text>Response 200 means that we are successfull in obtaining data from the website<jupyter_code>from bs4 import BeautifulSoup as bs ## importing Beautiful Soup
html_soup = bs(response.text,'html.parser')## python's in built library HTML parser
type(html_soup)
id_check = html_soup.find(id ="main")
movie_container = id_check.find_all(class_ ="lister-item mode-advanced")
len(movie_container)
container = movie_container[0]
meta = container.find("span", class_ = "metascore").text
print(meta)<jupyter_output>77<jupyter_text>findfind_all() returned a ResultSet object with the length of 50 movies we are interested in.<jupyter_code>#list to store scraped value data in:
movie_names = []
year_release = []
imdb_ratings = []
metascores = []
votes = []
movie_description = []
certificate = []
runtime = []
genre = []
director_name = []
star_cast = []
gross_value = []

#extract data from individual movie container
for container in movie_container:
    
   
    #if movie has Metascore, then extract:
    if container.find("div", class_ = "ratings-metascore") is not None:
        
        
        #the movie_name
        name = container.h3.a.text
        movie_names.append(name)
        
        #the year od release
        release = container.find("span", class_ = "lister-item-year text-muted unbold").text
        year_release.append(release)
        
        #the ratings for the movies
        ratings = float(container.strong.text)
        imdb_ratings.append(ratings)
        
        #the metascores
        meta = container.find("span", class_ = "metascore").text
        metascores.append(int(meta))
        
        #the votes
        vote = container.find("span", attrs = {"name":"nv"})['data-value']
        votes.append(int(vote))
        
        #the certificate
        certi = container.find("span", class_ = "certificate").text
        certificate.append(certi)
        
        #the runtime
        run = container.find("span", class_ ="runtime").text
        runtime.append(run)
        
        #the genre
        gen = container.find("span", class_ ="genre").text
        genre.append(gen)
        
        #fetching all <p> tags
        content = container.find_all("p")
        
        #the description
        desc = content[1].text
        movie_description.append(desc)
        
        #subsetting all the <a> tags in 3rd <p> tag
        content_2 = content[2].find_all("a")
        
        #the director
        director = content_2[0].text
        director_name.append(director)
        
        #the gross value
        if len(container.find_all("span", attrs = {"name":"nv"})) >= 2:
            gross = container.find_all("span", attrs = {"name":"nv"})[1]['data-value']
            gross_value.append(gross)
        else:
            gross_value.append(None)
        #extracting artists names
        temp = []
        for i in range(len(content_2)-1):
            temp.append(content_2[i].text)
        star_cast.append(temp)

import pandas as pd

test_df = pd.DataFrame({"movie_names":movie_names,
                        "year_release":year_release,
                        "imdb_ratings":imdb_ratings,
                        "metscores":metascores,
                        "votes":votes,
                        "movie_description":movie_description,
                        "certificate":certificate,
                        "runtime":runtime,
                        "genre":genre,
                        "director_name": director_name,
                        "star_cast": star_cast,
                        "gross_value":gross_value
                       })
test_df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1109753116597409, 'fraction_numerical': 0.007577609386458079, 'mean_word_length': 2.4531645569620255, 'pattern_counts': {'":': 15, '<': 19, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '296088', 'license': 'MIT', 'n_tokens_mistral': 1256, 'n_tokens_neox': 1213, 'n_words': 251, 'path': 'Week_2/NBB_IMDB_Web_Scraper.ipynb', 'repo_name': 'nikbearbrown/INFO_6210', 'size': '4091'}