<jupyter_start><jupyter_text>Import netflix shows dataset<jupyter_code>df = pd.read_csv("data/netflix_shows.csv", index_col='show_id')
df.head()<jupyter_output><empty_output><jupyter_text>Separate duration into two columns, duration for movie and number of seasons for TV Show<jupyter_code>seasons = []
movie_duration = []
for i in range(len(df)):
    tmp = df.iloc[i]['duration']
    number = tmp.split(" ")[0]
    if 'Season' in tmp:
        seasons.append(number)
        movie_duration.append(np.nan)
    else:
        seasons.append(np.nan)
        movie_duration.append(number)
df['seasons']=seasons
df['duration']=movie_duration
df.head()<jupyter_output><empty_output><jupyter_text>Data scrappingscrapping IMDB rating for every movie using the HTML parser beautiful soup.We first get the IMDB ID of the netflix show attributes, then visit IMDB.com/title/ID to get the score.<jupyter_code>def scrap_ID(title,year,media_type):
    encoding = 'utf-8'
    find_url = 'https://www.imdb.com/find' 
    values = {'q':title}
    query = urllib.parse.urlencode(values).encode(encoding)
    req = urllib.request.Request(find_url, query)
    resp = urllib.request.urlopen(req)
    html = BeautifulSoup(resp,"html")
    search = html.findAll("td", {"class": "result_text"})[:3]
    if len(search) > 0:
        for item in search: 
            if  str(item.text).find(str(year)) > 0:
                if str(media_type) == "TV Show":
                    if str(item.text).find("Series") > 0:
                        title_names = str(item.text)
                        title_IDs = str(item.a["href"])
                        break
                    else:
                        title_names = str(item.text)
                        title_IDs = str(item.a["href"])
                        break
                else:
                    title_names = str(item.text)
                    title_IDs = str(item.a["href"])
                    break

            else:
                if str(media_type) == "TV Show":
                    if str(item.text).find("Series") > 0:
                        title_names = str(item.text)
                        title_IDs = str(item.a["href"])
                        break
                    else:
                        title_names = str(item.text)
                        title_IDs = str(item.a["href"])
                        break
                else:
                    title_IDs = ''
                    break
    else:
        title_IDs = ''

    return title_IDs
def scrap_rating(title_ID):
    imdb_url = "https://www.imdb.com"
    if title_ID == '':
        rating = ''
    else:
        target_url = imdb_url + title_ID
        target_url = target_url
        resp = urllib.request.urlopen(target_url)
        html = BeautifulSoup(resp,"html")
        search = html.findAll("div", {"class": "ratingValue"})
        if len(search) > 0:
            for item in search:
                rating = str(item.strong['title'])
        else:
            rating = ''

    return rating<jupyter_output><empty_output><jupyter_text>We use fibonacci great property of taking time to build up while scalling exponentially.<jupyter_code>IMDB_rating = []
IMDB_titleID = []
i,j,c = 1,1,1
for index, row in df.iterrows():
    show_ID = scrap_ID(row.title, row.release_year, row.type)
    IMDB_titleID.append(show_ID)
    rating = scrap_rating(show_ID)
    IMDB_rating.append(rating)
    c+=1
    if c == i+j:
        print('scrapped {} IMDB rating {} left'.format(c, len(df)-c))
        i, j = j, i+j
print('done!')
IMDB_rating[:30]<jupyter_output><empty_output><jupyter_text>We can now add a rating and popularity column to our database<jupyter_code>score = []
popularity = []
for rating in IMDB_rating:
    if rating == '':
        score.append('')
        popularity.append('')
    else:
        tmp = rating.split(' ')
        score.append(tmp[0])
        popularity.append(tmp[3].replace(',',''))
df.rename(columns={'rating':'PG','listed_in':'genres'}, inplace=True)
df['rating'] = score
df['popularity'] = popularity
df['IMDB_id'] = ['' if tmp == '' else tmp[7:16] for tmp in IMDB_titleID]
df.head(5)<jupyter_output><empty_output><jupyter_text>reorder for readability<jupyter_code>df = df[['IMDB_id', 'type', 'title', 'release_year', 'rating', 'popularity', 'genres', 'PG', 'duration', 'seasons', 'director','date_added', 'country', 'cast', 'description']]
df.head(10)
df.to_csv("data/netflix_shows_with_IMDB_rating.csv", sep='|')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13557435440783616, 'fraction_numerical': 0.004897595725734639, 'mean_word_length': 2.401211203633611, 'pattern_counts': {'":': 4, '<': 25, '<?xml version=': 0, '>': 30, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '259070', 'license': 'MIT', 'n_tokens_mistral': 1464, 'n_tokens_neox': 1422, 'n_words': 290, 'path': 'notebook.ipynb', 'repo_name': 'Ismail-Maj/PostgreSQL-vs-Neo4j', 'size': '4492'}