<jupyter_start><jupyter_text>1. Main page Extracting Movie title, url, rank and rating of movies on the page<jupyter_code>site = "http://www.imdb.com/chart/moviemeter?ref_=nv_mv_mpm_8"
hdr = {'User-Agent': 'Mozilla/5.0'}
bookpage = requests.get(site)
soup = BeautifulSoup(bookpage.text, "html.parser")
#print(soup.prettify())
titles = soup.find_all('td', class_ ='titleColumn')
###unhash below if you want to see the output
#titles
#first row in the dataframe
titles[0]
#number of rows in the dataframe
len(titles)
#getting the text of the first row
titles[0].text.strip()
#getting the text before the \n 
titles[0].text.strip().split('\n')
titles[0].text.strip().split('\n')[0]
#getting the url of the first movie
titles[0].a['href']
title_db = []
url_db = []
for i in titles:
    name = i.text.strip().split('\n')[0]
    url = i.a['href']
    title_db.append(name)
    url_db.append(url)
title_db
url_db[0]
#getting the rank of the first movie
titles[0].text.strip().split('\n')[2]
#getting the rating of the movies
ratings = soup.find_all('td', class_ ='ratingColumn imdbRating')
#see rating of first movie
ratings[0].text.strip()<jupyter_output><empty_output><jupyter_text>2. Single page Extracting Movie advisory category, duration, genre, date<jupyter_code>nextsite = 'http://www.imdb.com/title/tt1825683/?pf_rd_m=A2FGELUUNOQJNL&pf_rd_p=ea4e08e1-c8a3-47b5-ac3a-75026647c16e&pf_rd_r=0P0MZYWFF77N2XF1Z2Y9&pf_rd_s=center-1&pf_rd_t=15506&pf_rd_i=moviemeter&ref_=chtmvm_tt_1'
hdr = {'User-Agent': 'Mozilla/5.0'}
onemoviepage = requests.get(nextsite)
onesoup = BeautifulSoup(onemoviepage.text, "html.parser")
j = onesoup.find('div', class_ = "subtext")
j
j.find(itemprop="contentRating").get("content")
j.find(itemprop="duration").get_text().strip()
j.find(itemprop="genre").get_text().strip()
k = j.find_all('a')[-1]
k.find(itemprop="datePublished").get("content")<jupyter_output><empty_output><jupyter_text>3. Putting everything together<jupyter_code>import time
data = []
site = "http://www.imdb.com/chart/moviemeter?ref_=nv_mv_mpm_8"
hdr = {'User-Agent': 'Mozilla/5.0'}
bookpage = requests.get(site)
soup = BeautifulSoup(bookpage.text, "html.parser")
#print(soup.prettify())
titles = soup.find_all('td', class_ ='titleColumn')
ratings = soup.find_all('td', class_ ='ratingColumn imdbRating')
title_db = []
url_db = []
for i in titles:
    name = i.text.strip().split('\n')[0]
    url = i.a['href']
    title_db.append(name)
    url_db.append(url)
for i in range(0,100): 
#for i in range(0,27): ###you can test a smaller range first
    name = title_db[i]
    rank = titles[i].text.strip().split('\n')[2]
    rating = ratings[i].text.strip()
    nextsite = 'http://www.imdb.com' + url_db[i]
    hdr = {'User-Agent': 'Mozilla/5.0'}
    onebookpage = requests.get(nextsite)
    onesoup = BeautifulSoup(onebookpage.text, "html.parser")
    #print(onesoup.prettify())
    j = onesoup.find('div', class_ = "subtext")
    try:
        advisory = j.find(itemprop="contentRating").get("content")
    except: 
        advisory = "NA"
    try:
        duration = j.find(itemprop="duration").get_text().strip()
    except:
        duration = "NA"
    try:
        genre = j.find(itemprop="genre").get_text().strip()
    except:
        genre = "NA"
    k = j.find_all('a')[-1]
    try:
        date = k.find(itemprop="datePublished").get("content")
    except:
        date = "NA"
    data.append((name,rank,rating,advisory,genre,duration,date))
    print(i)
    time.sleep(2)
#data
len(data)
df = pd.DataFrame(np.array(data))
df.columns = ['name','rank','rating','advisory','genre','duration','date']
df.head()
df.to_csv('imdbpopular.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1758601856908793, 'fraction_numerical': 0.021572910977607863, 'mean_word_length': 4.8514376996805115, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '66519', 'license': 'MIT', 'n_tokens_mistral': 1452, 'n_tokens_neox': 1421, 'n_words': 255, 'path': 'notebooks/imdb.ipynb', 'repo_name': 'alvations/datadoubleconfirm', 'size': '3662'}