<jupyter_start><jupyter_text>Module 1: Dataset  Import<jupyter_code># not all libraries are used
!pip install imdbpy
from bs4 import BeautifulSoup
import urllib.request
import urllib.parse
import re
import csv
import time
import datetime
import imdb
import ast
from tqdm import tnrange, tqdm_notebook
import sys
from urllib.request import HTTPError
import warnings
import html
import json
import os
import math
import inspect<jupyter_output>Collecting imdbpy
[?25l  Downloading https://files.pythonhosted.org/packages/95/23/60a7244a2d3473c80f71ae8dad832047058dc346525b3a6eacad78bc8d62/IMDbPY-6.7-py3-none-any.whl (285kB)
[K     |████████████████████████████████| 286kB 5.2MB/s 
[?25hRequirement already satisfied: lxml in /usr/local/lib/python3.6/dist-packages (from imdbpy) (4.2.6)
Requirement already satisfied: SQLAlchemy in /usr/local/lib/python3.6/dist-packages (from imdbpy) (1.3.5)
Installing collected packages: imdbpy
Successfully installed imdbpy-6.7<jupyter_text>Setup google drive<jupyter_code>root_dir = '/root/aml/'
drive_dir = root_dir + 'My Drive/AML/'
git_rep = 'Git'
git_dir = drive_dir + git_rep+'/'
dataset_dir = git_dir + 'datasets'
# Run this cell to mount your Google Drive.
from google.colab import drive
drive.mount(root_dir, force_remount=True) # run this line every time you have changed something in you drive
os.chdir(drive_dir)<jupyter_output>Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code

Enter your authorization code:
··········
Mounted at /root/aml/<jupyter_text>Utility functions<jupyter_code>_WARNINGS = False

def urlopen(url, mobile = False):
    try:
        if mobile:
            urlheader =  {'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46' ,
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
            'Accept-Encoding': 'none',
            'Accept-Language': 'en-US,en;q=0.8',
            'Connection': 'keep-alive'}
        else:
            urlheader = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) ' 
                          'AppleWebKit/537.11 (KHTML, like Gecko) '
                          'Chrome/23.0.1271.64 Safari/537.11',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
            'Accept-Encoding': 'none',
            'Accept-Language': 'en-US,en;q=0.8',
            'Connection': 'keep-alive'
                        }
        #header2 = 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.27 Safari/537.17'
        return urllib.request.urlopen(urllib.request.Request(url=url, data=None, headers=urlheader)).read().decode('utf-8')
    except HTTPError as e:
        if (_WARNINGS):
            time.sleep(5);
            warnings.warn(str(e))
            return urlopen(url)
        else:
            raise e

def wrap_error(func):
    def func_wrapper(*args, **kwargs):
        if (_WARNINGS):
            try:
                return func(*args, **kwargs)
            except Exception as e:
                warnings.warn(datetime.datetime.now().strftime("%d-%m-%Y %H:%M")+" - "+"Function "+ func.__name__ + " "+str(e))
        else:
            return func(*args, **kwargs)
    return func_wrapper<jupyter_output><empty_output><jupyter_text>MoviesDataset class  Definition of the class<jupyter_code>class MoviesDataset:
    def __init__(self):
        self.Youtube_urlroot = "https://www.youtube.com"
        self.Imdb_urlroot = "https://www.imdb.com"
        self.TheNumbers_urlroot = "https://www.the-numbers.com"
        self.max_number_movies = 16439
        self.number_movies_per_page = 100
        
        self.IMDb = imdb.IMDb()
        self.data = {}
        self.filename = "MoviesDataset"
            
    def NewJSON(self):
        with open(self.filename+'.json', 'w') as jsonFile:
            load = {}
            json.dump(load, jsonFile)
            jsonFile.close()
            self.estimatedSize = 0
    
    # append to the created json file
    @wrap_error
    def AppendJSON(self, movie):
        with open(self.filename+'.json', "r+") as jsonFile:
            load = json.load(jsonFile)
            load[movie['boxOffice']['id']-1] = movie
            jsonFile.seek(0)  # rewind
            json.dump(load, jsonFile)
            jsonFile.truncate()
            return os.path.getsize(self.filename+'.json')/1000000, len(load)
            #jsonFile.close()
            
    # load dataset
    def Load(self, filename = None):
        if not filename:
            filename = self.filename
        else:
            self.filename = filename
        with open(filename+'.json', "r+") as jsonFile:
            self.data = json.load(jsonFile)
            jsonFile.seek(0)  # rewind
            json.dump(self.data, jsonFile)
            jsonFile.truncate()
            
    #retrieve data from TheNumbers
    @wrap_error
    def BoxOfficeRetrieve(self, item):
        data = {}
        data['id']=int(item[0].text.replace(',',''))
        data['name']=item[2].text
        data['year']=int(item[1].text)
        data['url']=item[2].find_all('a')[0]['href']
        data['revenue_total']= int(item[3].text.replace(',','').replace('$',''))
        # retrieve first week revenue
        url = urllib.parse.urljoin("http://www.the-numbers.com", data['url'])
        html = urlopen(url, False)
        soup = BeautifulSoup(html, 'html.parser')
        div = soup.findAll(attrs={'id':'box_office_chart'})
        if len(div) >0:
            div = div[0]
            tables = div.findAll("table")
            data['revenue_week1']= int(div.findAll("td")[2].text.replace(',','').replace('$',''))
        else:
            return None, data['name']
        # retrieve country
        url = url[:url.index('#')]+'#tab=summary'
        html = urlopen(url, False)
        soup = BeautifulSoup(html, 'html.parser')
        table = soup.findAll("table")[3]
        data['country'] = [i.text for i in table.findAll("tr")[-1].findAll("td")[1].findAll("a")]
        return data, data['name']
        
    
    #search imdb id
    @wrap_error
    def IMDbSearch(self, movie_name, movie_year):
        try:
            result = self.IMDb.search_movie(movie_name)
            #print(result)
            score = 0
            for item in result:
                try:
                    if (item['kind'] == 'movie' and item['year'] == movie_year):
                        if (len(set(list(str(item).lower().split(" "))).intersection(list(movie_name.lower().split(" "))))>score):
                            return item.movieID
                except KeyError:
                    if (item['kind'] == 'movie'):
                        if (len(set(list(str(item).lower().split(" "))).intersection(list(movie_name.lower().split(" "))))>score):
                            return item.movieID
            for item in result:
                try:
                    if (item['kind'] == 'episode' and item['year'] == movie_year):
                        if (len(set(list(str(item).lower().split(" "))).intersection(list(movie_name.lower().split(" "))))>score):
                            return item.movieID
                except KeyError:
                    if (item['kind'] == 'episode'):
                        if (len(set(list(str(item).lower().split(" "))).intersection(list(movie_name.lower().split(" "))))>score):
                            return item.movieID
            return None
        except:
            print('Movie:' + movie_name + ' - year:' + str(movie_year) + ' could not be found in IMDb')
            return None
    
    @wrap_error
    def IMDbRetrieve(self, movie_name, movie_year):
        id = self.IMDbSearch(movie_name, movie_year)
        data = {}
        if id:
            url = 'https://www.imdb.com/title/tt'+str(id)
            html = urlopen(url)
            soup = BeautifulSoup(html, 'html.parser')
            load = json.loads(soup.find('script', type='application/ld+json').text)
            data.update(load)
            if 'embedUrl' in urlopen(urllib.parse.urljoin(self.Imdb_urlroot, data['url'])):
                url = urllib.parse.urljoin(self.Imdb_urlroot, data['trailer']['embedUrl'])
                html = urlopen(url)
                script = BeautifulSoup(html, 'html.parser').find_all('script')[-3].text
                load = json.loads(script[script.index('push(')+len('push('):script.index(');')])
                data['video'] = load
        return data
        
    
    #retrieve data from Youtube (also for Mobile device, defined by the url header information)
    @wrap_error
    def YoutubeRetrieve(self, movie_name, movie_year):
        data = {}
        query = urllib.parse.quote(movie_name+' '+str(movie_year)+' official trailer')
        url = 'https://www.google.com/search?biw=1620&bih=889&tbs=srcf%3AH4sIAAAAAAAAANOuzC8tKU1K1UvOz1UrSM0vyIEwSzJSy4sSC8DsssSizNSSSoiSnMTK5NS8kqLEHL2UVLX0zPREEA0AcHJbJEcAAAA&tbm=vid&q='+query
        #print(url)
        html = urlopen(url, mobile=True)
        soup = BeautifulSoup(html, 'html.parser')
        div = soup.findAll(attrs={'class':'mnr-c Tyz4ad'})
        if len(div):
            try:
                pos = 0
                while not('watch?v=' in str(div[pos])):
                    pos += 1
                div = div[pos]

                href = div.find_all('a')[0]['href']
                #print(href)
                data['name'] = soup.findAll(attrs={'class':'lCT7Rc Rqb6rf'})[0].text
                data['url'] = '/watch?v='+str(href[href.index('watch?v=')+len('watch?v='):])
                return data
            except IndexError:
                return None
        else:
            return None
                        
    #retrieve is based on the list of TheNumbers
    def Generate(self, movies_id, filename = None, save = True, new = True):
        def getMoviesIDList(movies_id):
            if isinstance(movies_id, list):
                if len(movies_id) >= 2:
                    if isinstance(movies_id[0], str) or isinstance(movies_id[-1], str):
                        if (movies_id[0] is 'start'):
                            start_id = 1
                            movies_id = list(range(start_id, movies_id[-1]+1))
                        if (movies_id[-1] is 'end'):
                            end_id = self.max_number_movies
                            movies_id = list(range(movies_id[0], end_id+1))
                    else:
                        movies_id = list(set(movies_id) & set(list(range(1,self.max_number_movies+1))))
                        movies_id.sort()
                else:
                    raise Exception("movies_id arg must be a list of the at least 2 ids")
            return list(movies_id)
        def id2page(id):
            return math.floor((id-1)/self.number_movies_per_page)*100+1
        def getMoviesList(page, ids):
            url = urllib.parse.urljoin(self.TheNumbers_urlroot, "/box-office-records/domestic/all-movies/cumulative/all-time/") + str(page)
            html = urlopen(url)
            soup = BeautifulSoup(html, 'html.parser')
            tables = soup.findAll("table")
            if tables:
                first_table = tables[0]
                first_table = first_table.find_all('tr')[1:]
                return [i for i in first_table if int(i.find_all('td')[0].text.replace(',', '')) in ids]
            else:
                return None
        def getOneMovie (page_movies, id):
            for movie in page_movies:
                if int(movie.find_all('td')[0].text.replace(',', '')) == id:
                    return movie.find_all('td')
            raise
            return None
                           
        
        if filename:
            self.filename = filename
        if save:
            if new:
                self.NewJSON()
        else:
            self.data = []
            
        #regroup
        movies_id = getMoviesIDList(movies_id)
        
        pbar_pages = tqdm_notebook(list(set([id2page(id) for id in movies_id])), file=sys.stdout, ncols = 800)
        pbar_movies = tqdm_notebook(movies_id, file=sys.stdout, ncols = 800)
        current_page = id2page(movies_id[0])
        page_movies = getMoviesList(current_page, list(set(movies_id) & set(range(current_page, current_page+100))))
        one_movie = getOneMovie(page_movies, movies_id[0])
        pbar_pages.set_description(('Page %d: ') % (current_page))
        for id in pbar_movies:
            if id2page(id) != current_page:
                current_page = id2page(id)
                page_movies = getMoviesList(current_page, list(set(movies_id) & set(range(current_page, current_page+100))))
                pbar_pages.update()
            
            boxoffice_data, imdb_data, youtube_data = None, None, None
            #get the movie line
            one_movie = getOneMovie(page_movies, id)
            if (one_movie):
                #retrieve box office
                boxoffice_data, movie_name = self.BoxOfficeRetrieve(item = one_movie)
                if boxoffice_data:
                    #retrieve IMDb
                    imdb_data = self.IMDbRetrieve(movie_name = boxoffice_data['name'], movie_year = boxoffice_data['year'])
                    if imdb_data:
                        #retrieve youtube
                        #print(boxoffice_data['name'])
                        youtube_data = self.YoutubeRetrieve(movie_name = imdb_data['name'], movie_year = boxoffice_data['year'])
                        if youtube_data:
                            #all data retrieved and ready to be stored
                            movie = {'boxOffice' : boxoffice_data}
                            movie.update(imdb_data)
                            if not 'video' in movie: #trailer in imdb was not found
                                movie['video'] = {'videos': {'videoMetadata':{}}}
                            movie['video']['videos']['videoMetadata'].update({'youtube': youtube_data})
                            print(str(id)+': ', movie['name'], ' stored')
                            #save in json file and update bar
                            if save:
                                current_size, json_length = self.AppendJSON(movie)
                                self.estimatedSize = current_size + current_size/json_length*(len(movies_id))
                                pbar_pages.set_description(('Page %d estimated total size %d.3MB: ') % (current_page, self.estimatedSize))
                            else:
                                self.data.append(movie)
                pbar_movies.set_description(str(id)+': '+movie_name)<jupyter_output><empty_output><jupyter_text>creation of movies object<jupyter_code>movies = MoviesDataset()<jupyter_output><empty_output><jupyter_text>dataset generation **movies.Generate**:* **interval** : list(range(id_start, id_end+1)), [4, 32, 501], [id_start, 'end'], ['start', id_end]* **filename**: "MoviesDataset" (the name of the generated json file)* **save**: True, False (if True, save the retrieved movies in the json file, if False, the data will be stored in the variable movies.data)* **new**: True, False (create a new json file, be careful, this will overwrite the existing json file named $filename)<jupyter_code>os.chdir(dataset_dir)
movies.Generate(movies_id = [16364, 'end'], filename='Dataset', save = True, new = False)<jupyter_output><empty_output><jupyter_text>load dataset **movies.Load**:* **filename**: "MoviesDataset" (the name of the json file to be loaded, the result will be stored in the instance movies.data), no value means the file movies.filename will be used, which is "MoviesDataset" per default<jupyter_code>movies.Load(filename = 'Dataset')
print(len(movies.data))<jupyter_output>9178<jupyter_text>filter Download the software HugeDataViewer and open your dataset to see how the dataset is structured.There are the most important instructions of using dict in Python for our project: (for more, see python wiki)*   Get the movie of id=id```movies.data[str(id)]```*   Get list of movies given a condition```[item for id, item in movies.data.items() if item['boxOffice']['country'][0] == 'United States']```<jupyter_code>movies.data.get(str(4)).get('actor')[0].get('name')
[item for id, item in movies.data.items() if item.get('boxOffice').get('name') == 'Minions']<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13387411924921547, 'fraction_numerical': 0.020782758008171, 'mean_word_length': 2.248076923076923, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 36, 'https://': 7, 'lorem ipsum': 0, 'www.': 10, 'xml': 5}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '9565', 'license': 'MIT', 'n_tokens_mistral': 5569, 'n_tokens_neox': 5248, 'n_words': 1058, 'path': 'models/dataset/dataset_generator_json.ipynb', 'repo_name': 'ussaema/Box_Office_Sales_Estimation', 'size': '16889'}