<jupyter_start><jupyter_text>Combine Movie Functions<jupyter_code>## Basic stuff
%load_ext autoreload
%autoreload
from IPython.core.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))
display(HTML("""<style>div.output_area{max-height:10000px;overflow:scroll;}</style>"""))

## Python Version
import sys
print("Python: {0}".format(sys.version))

from wikipedia import wikipedia
from wikifilm import wikifilm
from oscar import oscars
from razzies import razzies
from rollingstone import rollingstone
from BAFTA import BAFTA
from SAG import SAG
from filmsite import filmsite
from films101 import films101
from AACTA import AACTA
from flops import flops
from canada import Canada
from boxofficemojo import boxofficemojo
from rottentomatoes import rottentomatoes
from ultimatemovierankings import ultimatemovierankings
from combine import combine


import datetime as dt
start = dt.datetime.now()
print("Notebook Last Run Initiated: "+str(start))<jupyter_output><empty_output><jupyter_text>Get/Parse/Merge/Process Rotten Tomatoes Data<jupyter_code>%load_ext autoreload
%autoreload

comb = combine()
comb.setOscarData(oscars(wikipedia()), 10)
comb.setWikiFilmData(wikifilm(), 1)
comb.setRazziesData(razzies(), 1)
comb.setBAFTAData(BAFTA(), 1)
comb.setSAGData(SAG(), 1)
comb.setAACTAData(AACTA(), 1)
comb.setFilmsiteData(filmsite(), 1)
comb.setFilms101Data(films101(), 1)
comb.setFlopsData(flops(), 1)
comb.setCanadaData(Canada(), 1)
comb.setRollingStoneData(rollingstone(), 1)
comb.setRottenTomatoesData(rottentomatoes(), 90)
comb.setBoxOfficeMojoData(boxofficemojo(), 5e6)
comb.setUltimateMovieRankingsData(ultimatemovierankings(), 90)
comb.getData()
comb.saveCorrections(debug=True)
comb.mergeMovies(debug=True)<jupyter_output>The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload
Found 61 Years of Canada Movies
Found 50 Years of AACTA Movies
Found 105 Years of Filmsite Movies
Found 84 Years of Oscar Movies
Found 32 Years of Flops Movies
Found 94 Years of Rotten Tomatoes Movies
Found 38 Years of Razzies Movies
Found 39 Years of Box Office Mojo Movies
Found 98 Years of Wiki Film Movies
Found 47 Years of BAFTA Movies
Found 67 Years of Ultimate Movie Rankings Movies
Found 119 Years of Films101 Movies
Found 23 Years of Rolling Stone Movies
Found 25 Years of SAG Movies
Found Data Between 1900 and 2018
There is no corrections data.
Found 17748 movies
	 ('Films101', 8276)
	 ('Box Office Mojo', 3935)
	 ('Oscar', 2268)
	 ('Rotten Tomatoes', 987)
	 ('AACTA', 672)
	 ('Filmsite', 670)
	 ('Razzies', 349)
	 ('Canada', 225)
	 ('Wiki Film', 155)
	 ('BAFTA', 153)
	 ('Ultimate Movie Rankings', 29)
	 ('SAG', 13)
	 ('Rolling Stone', 11)
	 ('Flops', 5)
Could not remove Amarcord [1973]
Could not r[...]<jupyter_text>Find Corrections<jupyter_code>from ioUtils import getFile
data = getFile("/Users/tgadfort/Documents/code/movies/combine/results/movies.json")
for movie,name in data.items():
    year  = movie[-5:-1]
    movie = movie[:-7]
    
    pos = movie.rfind("(")
    if pos == -1:
        continue        

    if pos > len(movie) - 15 and movie.find(",") != -1:
        fix = movie[:pos].strip()
        print("{0}:".format(fix))
        print(" - {0}".format(movie))
from searchUtils import findNearest
from fsUtils import setFile
from ioUtils import saveFile

savename = setFile(comb.getDataDir(), "saved.yaml")
savedData = getFile(savename)

movies = comb.getCombinedMovies()

repData = {}
for corrMovie,corrs in savedData.items():
    for corr in corrs:
        try:
            repData[corr] = corrMovie
        except:
            raise ValueError(corr,corrMovie)
            
print("Found {0} movies".format(len(movies)))
print("Found {0} corrections".format(len(repData)))
movielist = {}
for moviename in movies.keys():
    name = moviename[:-7]
    year = moviename[-5:-1]
    try:
        year = int(year)
    except:
        raise ValueError(moviename)

    if repData.get(name) is not None:
        name = repData[name]

    if name.find("IMAX") != -1:
        continue
    if name.find("re-issue") != -1:
        continue        
        
    for y in range(year-2,year+3):
        if movielist.get(y) is None:
            movielist[y] = []
        movielist[y].append(name)

for year in movielist.keys():
    movielist[year] = set(movielist[year])
data   = {}
cutoff = 0.75

for year,movienames in movielist.items():
    print(year,len(movienames))
    
    for im,movie in enumerate(list(movienames)):
        if movie.endswith(' '):        
            if data.get(movie[:-1]) is None:
                print('\t',len(data),'\t',movie[:-1])
                data[movie[:-1]] = set()
                data[movie[:-1]].add(movie)
            tmpList = list(filter(lambda a: a != movie[:-1], list(movienames)))
            results = findNearest(movie[:-1], tmpList, 5, cutoff)
            if len(results) > 0:
                for value in results:
                    data[movie[:-1]].add(value)
        else:
            tmpList = list(filter(lambda a: a != movie, list(movienames)))    
            results = findNearest(movie, tmpList, 5, cutoff)
            if len(results) > 0:
                if data.get(movie) is None:
                    print('\t',len(data),'\t',movie)
                    data[movie] = set()
                for value in results:
                    data[movie].add(value)

        if len(data) > 300:
            break
        
for k in data.keys():
    data[k] = list(data[k])

            

savename = setFile(comb.getDataDir(), "corr.yaml")
saveFile(idata=data, ifile=savename, debug=True)<jupyter_output>1898 8
1899 19
	 0 	 Why Mrs. Jones Got A Divorce
1900 25
1901 33
1902 42
1903 41
1904 38
1905 42
1906 41
1907 45
1908 64
1909 75
1910 93
1911 109
1912 130
1913 129
1914 153
	 1 	 Shoes
	 2 	 She
1915 159
	 3 	 A Man There Was
	 4 	 A Fool There Was
1916 165
	 5 	 A Woman
	 6 	 Woman
1917 163
	 7 	 The Silent Man
	 8 	 Behind The Screen
	 9 	 Behind The Door
	 10 	 The Miracle Man
1918 181
	 11 	 The Return Of Tarzan
	 12 	 The Return Of Draw Egan
1919 179
	 13 	 The Boat
	 14 	 The Bond
	 15 	 The Goat
	 16 	 A Tale Of Two Cities
	 17 	 A Tale Of Two Worlds
1920 196
1921 209
	 18 	 The Spoilers
	 19 	 The Spiders
1922 220
	 20 	 White Shadows
	 21 	 Whispering Shadows
1923 235
	 22 	 The Circle
	 23 	 The White Desert
	 24 	 The White Sister
	 25 	 The Cradle
1924 258
	 26 	 What Price Goofy?
	 27 	 What Price Glory?
	 28 	 The Toll Of The Sea
	 29 	 The Son Of The Sheik
1925 275
	 30 	 The King Of Kings
	 31 	 The Sting Of Stings
	 32 	 7th Heaven
	 33 	 Seventh Heaven
1926 314
	 34 [...]
{'fraction_non_alphanumeric': 0.10583446404341927, 'fraction_numerical': 0.06286748077792854, 'mean_word_length': 3.090012330456227, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '357596', 'license': 'MIT', 'n_tokens_mistral': 2667, 'n_tokens_neox': 2173, 'n_words': 692, 'path': 'Combine.ipynb', 'repo_name': 'tgadf/movies', 'size': '6633'}