<jupyter_start><jupyter_text>IMDB Scraper from Wayback Machine<jupyter_code>import sys
import requests as rq
from bs4 import BeautifulSoup as bs
import json
import pandas as pd
import csv
import re<jupyter_output><empty_output><jupyter_text>filmID input Ex: tt0120338<jupyter_code>filmId= input("filmID: ")
film='https://www.imdb.com/title/'+filmId
filePath = input("File path: "+"r'")
with open(filePath+"/"+filmId+".csv", mode='w', newline='') as newFolder:
    newWriter = csv.writer(newFolder, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) 
    newWriter.writerow(['Year'+";"+'Month'+";"+'Day'+";"+'Hours'+";"+'Rating'+";"+'Toplam Oy'])
    newFolder.close()<jupyter_output><empty_output><jupyter_text>Part1<jupyter_code>startTime='20070228'
finishTime='20070331'
url = 'http://web.archive.org/cdx/search/cdx?url='+film+'/&collapse=digest&from='+startTime+'&to='+finishTime+'&output=json'
urls = rq.get(url).text
parse_url = json.loads(urls)
url_list = []
for i in range(1,len(parse_url)):
    orig_url = parse_url[i][2]
    tstamp = parse_url[i][1]
    waylink = 'https://web.archive.org/web/'+tstamp+'/'+orig_url
    url_list.append(waylink)    
for i in range(len(url_list)):
    try:
        final_url=url_list[i]
        req = rq.get(final_url).text
        print(req)
        break
        soup = bs(req,'html.parser')
        classScraper = soup.find(class_='rating')
        tagScraper1 = classScraper.find_all('b')
        tagScraper2 = classScraper.find_all('a')
        print(final_url[28:42]+";"+tagScraper1[1].contents[0].replace('/10','')+";"+tagScraper2[2].contents[0].replace('votes','').replace(',',''))
        with open(filePath+"/"+filmId+".csv", mode='a', newline='') as newFolder:
            newWriter = csv.writer(newFolder, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) 
            newWriter.writerow([final_url[28:32]+";"+final_url[32:34]+";"+final_url[34:36]+";"+(final_url[36:38]+":"+final_url[38:40])+";"+tagScraper1[1].contents[0].replace('/10','')+";"+tagScraper2[2].contents[0].replace('votes','').replace(',','')])
            newFolder.close()
    except: 
        pass<jupyter_output><empty_output><jupyter_text>Part2<jupyter_code>startTime='20070401'
finishTime='20080531'
url = 'http://web.archive.org/cdx/search/cdx?url='+film+'/&collapse=digest&from='+startTime+'&to='+finishTime+'&output=json'
urls = rq.get(url).text
parse_url = json.loads(urls)
url_list = []
for i in range(1,len(parse_url)):
    orig_url = parse_url[i][2]
    tstamp = parse_url[i][1]
    waylink = 'https://web.archive.org/web/'+tstamp+'/'+orig_url
    url_list.append(waylink)    
for i in range(len(url_list)):
    try:
        final_url=url_list[i]
        req = rq.get(final_url).text
        soup = bs(req,'html.parser')
        classScraper = soup.find(class_='general rating')
        tagScraper1 = classScraper.find_all('b')
        tagScraper2 = classScraper.find_all('a')
        print(final_url[28:42]+";"+tagScraper1[1].contents[0].replace('/10','')+";"+tagScraper2[0].contents[0].replace('votes','').replace(',',''))
        with open(filePath+"/"+filmId+".csv", mode='a', newline='') as newFolder:
            newWriter = csv.writer(newFolder, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) 
            newWriter.writerow([final_url[28:32]+";"+final_url[32:34]+";"+final_url[34:36]+";"+(final_url[36:38]+":"+final_url[38:40])+";"+tagScraper1[1].contents[0].replace('/10','')+";"+tagScraper2[0].contents[0].replace('votes','').replace(',','')])
            newFolder.close()
    except: 
        pass<jupyter_output><empty_output><jupyter_text>Part3<jupyter_code>startTime='20080601'
finishTime='20091231'
url = 'http://web.archive.org/cdx/search/cdx?url='+film+'/&collapse=digest&from='+startTime+'&to='+finishTime+'&output=json'
urls = rq.get(url).text
parse_url = json.loads(urls)
url_list = []
for i in range(1,len(parse_url)):
    orig_url = parse_url[i][2]
    tstamp = parse_url[i][1]
    waylink = 'https://web.archive.org/web/'+tstamp+'/'+orig_url
    url_list.append(waylink)    
for i in range(len(url_list)):
    try:
        final_url=url_list[i]
        req = rq.get(final_url).text
        soup = bs(req,'html.parser')
        classScraper = soup.find(class_='meta')
        tagScraper1 = classScraper.find_all('b')
        tagScraper2 = classScraper.find_all('a')
        print(final_url[28:42]+";"+tagScraper1[0].contents[0].replace('/10','')+";"+tagScraper2[0].contents[0].replace('votes','').replace(',',''))
        with open(filePath+"/"+filmId+".csv", mode='a', newline='') as newFolder:
            newWriter = csv.writer(newFolder, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) 
            newWriter.writerow([final_url[28:32]+";"+final_url[32:34]+";"+final_url[34:36]+";"+(final_url[36:38]+":"+final_url[38:40])+";"+tagScraper1[0].contents[0].replace('/10','')+";"+tagScraper2[0].contents[0].replace('votes','').replace(',','')])
            newFolder.close()
    except: 
        pass<jupyter_output><empty_output><jupyter_text>Part4<jupyter_code>startTime='20100101'
finishTime='20100930'
url = 'http://web.archive.org/cdx/search/cdx?url='+film+'/&collapse=digest&from='+startTime+'&to='+finishTime+'&output=json'
urls = rq.get(url).text
parse_url = json.loads(urls)
url_list = []
for i in range(1,len(parse_url)):
    orig_url = parse_url[i][2]
    tstamp = parse_url[i][1]
    waylink = 'https://web.archive.org/web/'+tstamp+'/'+orig_url
    url_list.append(waylink)    
for i in range(len(url_list)):
    try:
        final_url=url_list[i]
        req = rq.get(final_url).text
        soup = bs(req,'html.parser')
        classScraper = soup.find(class_='starbar-meta')
        tagScraper1 = classScraper.find_all('b')
        tagScraper2 = classScraper.find_all('a')
        print(final_url[28:42]+";"+tagScraper1[0].contents[0].replace('/10','')+";"+tagScraper2[0].contents[0].replace('votes','').replace(',',''))
        with open(filePath+"/"+filmId+".csv", mode='a', newline='') as newFolder:
            newWriter = csv.writer(newFolder, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) 
            newWriter.writerow([final_url[28:32]+";"+final_url[32:34]+";"+final_url[34:36]+";"+(final_url[36:38]+":"+final_url[38:40])+";"+tagScraper1[0].contents[0].replace('/10','')+";"+tagScraper2[0].contents[0].replace('votes','').replace(',','')])
            newFolder.close()
    except: 
        pass<jupyter_output><empty_output><jupyter_text>Part5<jupyter_code>startTime='20101001'
finishTime='20110731'
url = 'http://web.archive.org/cdx/search/cdx?url='+film+'/&collapse=digest&from='+startTime+'&to='+finishTime+'&output=json'
urls = rq.get(url).text
parse_url = json.loads(urls)
url_list = []
for i in range(1,len(parse_url)):
    orig_url = parse_url[i][2]
    tstamp = parse_url[i][1]
    waylink = 'https://web.archive.org/web/'+tstamp+'/'+orig_url
    url_list.append(waylink)    
for i in range(len(url_list)):
    try:
        final_url=url_list[i]
        req = rq.get(final_url).text
        soup = bs(req,'html.parser')
        classScraper = soup.find(class_='star-box')
        tagScraper1 = classScraper.find_all('a')
        tagScraper2 = classScraper.find_all('span')
        print(final_url[28:42]+";"+tagScraper2[13].contents[0]+";"+tagScraper1[11].contents[0].replace(',','').replace('votes',''))
        with open(filePath+"/"+filmId+".csv", mode='a', newline='') as newFolder:
            newWriter = csv.writer(newFolder, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) 
            newWriter.writerow([final_url[28:32]+";"+final_url[32:34]+";"+final_url[34:36]+";"+(final_url[36:38]+":"+final_url[38:40])+";"+tagScraper2[13].contents[0]+";"+tagScraper1[11].contents[0].replace(',','').replace('votes','')])
            newFolder.close()
    except:
        pass<jupyter_output><empty_output><jupyter_text>Part6<jupyter_code>startTime='20110801'
finishTime='20151213'
url = 'http://web.archive.org/cdx/search/cdx?url='+film+'/&collapse=digest&from='+startTime+'&to='+finishTime+'&output=json'
urls = rq.get(url).text
parse_url = json.loads(urls)
url_list = []
for i in range(1,len(parse_url)):
    orig_url = parse_url[i][2]
    tstamp = parse_url[i][1]
    waylink = 'https://web.archive.org/web/'+tstamp+'/'+orig_url
    url_list.append(waylink)    
for i in range(len(url_list)):
    try:
        final_url=url_list[i]
        req = rq.get(final_url).text
        soup = bs(req,'html.parser')
        classScraper = soup.find(class_='star-box-details')
        tagScraper1 = classScraper.find_all('span')
        print(final_url[28:42]+";"+tagScraper1[0].contents[0]+";"+tagScraper1[3].contents[0].replace(',',''))
        with open(filePath+"/"+filmId+".csv", mode='a', newline='') as newFolder:
            newWriter = csv.writer(newFolder, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) 
            newWriter.writerow([final_url[28:32]+";"+final_url[32:34]+";"+final_url[34:36]+";"+(final_url[36:38]+":"+final_url[38:40])+";"+tagScraper1[0].contents[0]+";"+tagScraper1[3].contents[0].replace(',','')])
            newFolder.close()
    except:
        pass<jupyter_output><empty_output><jupyter_text>Part7<jupyter_code>startTime='20151214'
finishTime='20160106'
url = 'http://web.archive.org/cdx/search/cdx?url='+film+'/&collapse=digest&from='+startTime+'&to='+finishTime+'&output=json'
urls = rq.get(url).text
parse_url = json.loads(urls)
url_list = []
for i in range(1,len(parse_url)):
    orig_url = parse_url[i][2]
    tstamp = parse_url[i][1]
    waylink = 'https://web.archive.org/web/'+tstamp+'/'+orig_url
    url_list.append(waylink)    
for i in range(len(url_list)):
    try:
        final_url=url_list[i]
        req = rq.get(final_url).text
        soup = bs(req,'html.parser')
        classScraper = soup.find(class_='imdbRating')
        tagScraper1 = classScraper.find_all('span')
        print(final_url[28:42]+";"+tagScraper1[0].contents[0]+";"+tagScraper1[3].contents[0].replace(',',''))
        with open(filePath+"/"+filmId+".csv", mode='a', newline='') as newFolder:
            newWriter = csv.writer(newFolder, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) 
            newWriter.writerow([final_url[28:32]+";"+final_url[32:34]+";"+final_url[34:36]+";"+(final_url[36:38]+":"+final_url[38:40])+";"+tagScraper1[0].contents[0]+";"+tagScraper1[3].contents[0].replace(',','')])
            newFolder.close()
    except:
        pass<jupyter_output><empty_output><jupyter_text>Part8<jupyter_code>startTime='20160107'
finishTime='20160126'
url = 'http://web.archive.org/cdx/search/cdx?url='+film+'/&collapse=digest&from='+startTime+'&to='+finishTime+'&output=json'
urls = rq.get(url).text
parse_url = json.loads(urls)
url_list = []
for i in range(1,len(parse_url)):
    orig_url = parse_url[i][2]
    tstamp = parse_url[i][1]
    waylink = 'https://web.archive.org/web/'+tstamp+'/'+orig_url
    url_list.append(waylink)    
for i in range(len(url_list)):
    try:
        final_url=url_list[i]
        req = rq.get(final_url).text
        soup = bs(req,'html.parser')
        classScraper = soup.find(class_='star-box-details')
        tagScraper1 = classScraper.find_all('span')
        print(final_url[28:42]+";"+tagScraper1[0].contents[0]+";"+tagScraper1[3].contents[0].replace(',',''))
        with open(filePath+"/"+filmId+".csv", mode='a', newline='') as newFolder:
            newWriter = csv.writer(newFolder, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) 
            newWriter.writerow([final_url[28:32]+";"+final_url[32:34]+";"+final_url[34:36]+";"+(final_url[36:38]+":"+final_url[38:40])+";"+tagScraper1[0].contents[0]+";"+tagScraper1[3].contents[0].replace(',','')])
            newFolder.close()
    except:
        pass<jupyter_output><empty_output><jupyter_text>Part9<jupyter_code>startTime='20160127'
finishTime='2022'
url = 'http://web.archive.org/cdx/search/cdx?url='+film+'/&collapse=digest&from='+startTime+'&to='+finishTime+'&output=json'
urls = rq.get(url).text
parse_url = json.loads(urls)
url_list = []
for i in range(1,len(parse_url)):
    orig_url = parse_url[i][2]
    tstamp = parse_url[i][1]
    waylink = 'https://web.archive.org/web/'+tstamp+'/'+orig_url
    url_list.append(waylink)    
for i in range(len(url_list)):
    try:
        final_url=url_list[i]
        req = rq.get(final_url).text
        soup = bs(req,'html.parser')
        classScraper = soup.find(class_='imdbRating')
        tagScraper1 = classScraper.find_all('span')
        print(final_url[28:42]+";"+tagScraper1[0].contents[0]+";"+tagScraper1[3].contents[0].replace(',',''))
        with open(filePath+"/"+filmId+".csv", mode='a', newline='') as newFolder:
            newWriter = csv.writer(newFolder, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) 
            newWriter.writerow([final_url[28:32]+";"+final_url[32:34]+";"+final_url[34:36]+";"+(final_url[36:38]+":"+final_url[38:40])+";"+tagScraper1[0].contents[0]+";"+tagScraper1[3].contents[0].replace(',','')])
            newFolder.close()
    except:
        pass<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.21372819100091828, 'fraction_numerical': 0.041475359657177836, 'mean_word_length': 4.847427293064877, 'pattern_counts': {'":': 9, '<': 45, '<?xml version=': 0, '>': 45, 'https://': 10, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '246269', 'license': 'MIT', 'n_tokens_mistral': 5295, 'n_tokens_neox': 5079, 'n_words': 503, 'path': 'waybackMachineIMDBScraper.ipynb', 'repo_name': 'batucimenn/waybackMachineIMDBScraper', 'size': '13068'}