<jupyter_start><jupyter_text>Load original data from movieLens<jupyter_code>data_directory = "../../Downloads/ml-latest/"
movieFile = "movies.csv"
ratingFile = "ratings.csv"
df_rating = pd.read_csv(data_directory + ratingFile, dtype = {'timestamp': 'Int64'})
df_movie = pd.read_csv(data_directory + movieFile)
df_rating.head()
df_movie.head()<jupyter_output><empty_output><jupyter_text>Target dataset movies table movie_id, imdb_id, title, genres, imdb_url, poster_url ratings table rater_id, movie_id, rating, timestamp  Scrape info from IMDB<jupyter_code>movieDatasetFile = 'movie_full_dataset.csv'
with open(data_directory + movieDatasetFile, 'w', newline='') as out_csv:
    writer = csv.writer(out_csv, delimiter=',')
    writer.writerow(['movie_id', 'imdb_id', 'title', 'genres', 'imdb_url', 'poster_url'])
endPoint = -1
%%time
total = len(df_movie)
count = endingPoint
with open(data_directory + movieDatasetFile, 'a', newline='') as out_csv:
    writer = csv.writer(out_csv, delimiter=',')
    for i in range(len(df_movie))[endingPoint + 1 :]:
        row = df_movie.iloc[i]
        movieId = row['movieId']
        title = row['title']
        genres = row['genres']
        domain = 'http://www.imdb.com'
        search_url = domain + '/find?q=' + urllib.parse.quote_plus(title)
        count += 1
        if (count % 100 == 0):
            print(f'{count} processing, {total - count} left')
        with urllib.request.urlopen(search_url) as response:
            html = response.read()
            soup = BeautifulSoup(html, 'html.parser')
            # Get url of 1st search result
            try:
                foundTitle = soup.find('table', class_='findList').tr.a['href']
                movieUrl = domain + foundTitle
                imdbId = int(foundTitle.split('/')[2][2:])
                with urllib.request.urlopen(movieUrl) as response:
                    html = response.read()
                    soup = BeautifulSoup(html, 'html.parser')
                    # Get url of poster image
                    try:
                        posterUrl = soup.find('div', class_='poster').a.img['src']
                        extension = '.jpg'
                        posterUrl = ''.join(posterUrl.partition('_')[0]) + extension
                        writer.writerow([movieId, imdbId, title, genres, movieUrl, posterUrl])
                    # Ignore cases where search returns no results
                    except AttributeError:
                        pass
            # Ignore cases where search returns no results
            except AttributeError:
                pass<jupyter_output><empty_output><jupyter_text>In case connection closed by IMDB host, get breakpoint, re-run the upper cell to keep scraping.<jupyter_code># URLError: <urlopen error [Errno 54] Connection reset by peer>
movieDatasetFile = 'movie_full_dataset.csv'
df_movieDataset = pd.read_csv(data_directory + movieDatasetFile)
endingId = df_movieDataset.iloc[len(df_movieDataset) - 1]['movie_id']
for i in range(len(df_movie)):
    row = df_movie.iloc[i]
    movieId = row['movieId']
    if movieId == endingId:
        print(f"found ending point {i}th row")
        break
endingPoint = i
movieDatasetFile = 'movie_full_dataset.csv'
df_movieDataset = pd.read_csv(data_directory + movieDatasetFile)
print(f'yield {len(df_movieDataset)} rows with poster urls from {len(df_movie)} rows in original dataset')<jupyter_output>yield 53806 rows with poster urls from 58098 rows in original dataset<jupyter_text>Normalize ratings using Decoupling Normalization<jupyter_code>allRaters = np.unique(df_rating['userId'])
ratingsPerRater = {rater: [] for rater in allRaters}
for i in range(len(df_rating)):
    row = df_rating.iloc[i]
    ratingsPerRater[row['userId']].append(row['rating'])
POSSIBLE_RATINGS = [0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 4.5, 5]
def getProb(ratings, r):
    p1 = np.sum([rating < r for rating in ratings]) / len(ratings)
    p2 = np.sum([rating == r for rating in ratings]) / len(ratings)
    return p1 + p2 / 2

def getDecoupledRatings(ratings):
    prob = {r : getProb(ratings, r) for r in POSSIBLE_RATINGS}
    decoupledRatings = [prob[rating] for rating in ratings]
    return decoupledRatings

def getGaussianNormalizedRatings(ratings):
    mean = np.mean(ratings)
    std = np.std(ratings)
    ratings = np.array(ratings)
    return (ratings - mean) / std
        
decoupledRatingPerrater = {rater: getDecoupledRatings(ratingsPerRater[rater]) for rater in allRaters}
otherInfosPerRater = {rater: [] for rater in allRaters}
for i in range(len(df_rating)):
    row = df_rating.iloc[i]
    otherInfosPerRater[row['userId']].append((row['movieId'], row['timestamp']))
ratingDatasetFile = 'rating_full_dataset_normalized.csv'
with open(data_directory + ratingDatasetFile, 'w', newline='') as out_csv:
    writer = csv.writer(out_csv, delimiter=',')
    writer.writerow(['rater_id', 'movie_id', 'rating', 'timestamp'])
%%time
total = len(allRaters)
count = 0
with open(data_directory + ratingDatasetFile, 'a', newline='') as out_csv:
    writer = csv.writer(out_csv, delimiter=',')
    for rater in allRaters:
        if len(otherInfosPerRater[rater]) != len(decoupledRatingPerrater[rater]):
            print(f"error! rater: {rater} len_info {len(otherInfosPerRater[rater])} len_decoupled {len(decoupledRatingPerrater[rater])}")

        for i in range(len(otherInfosPerRater[rater])):
            rating = decoupledRatingPerrater[rater][i]
            movieId = otherInfosPerRater[rater][i][0]
            timestamp = otherInfosPerRater[rater][i][1]
            writer.writerow([rater, movieId, rating - 0.5, timestamp])
    
        count += 1
        if count % 100 == 0:
            print(f'processed {count} raters, {total - count} left')
ratingDatasetFile = 'rating_full_dataset_normalized.csv'
df_ratingDataset = pd.read_csv(data_directory + ratingDatasetFile)<jupyter_output><empty_output><jupyter_text>Only preserve ratings that are associated with a movie present in movie database<jupyter_code>movieIds = list(df_movieDataset['movie_id'])
print(len(movieIds), len(np.unique(movieIds)))
df_ratingDataset_filtered = df_ratingDataset[df_ratingDataset['movie_id'].isin(movieIds)]
print(f'yield {len(df_ratingDataset_filtered)} rows from {len(df_rating)} rows in original dataset')
ratingDatasetFile_filtered = 'rating_full_dataset_normalized_filtered.csv'
df_ratingDataset_filtered.to_csv(data_directory + ratingDatasetFile_filtered, index = False)
print(len(np.unique(df_ratingDataset_filtered['movie_id'])))<jupyter_output>49800<jupyter_text>Only preserve movies that have ratings in rating database<jupyter_code>moviesWithRatings = list(np.unique(df_ratingDataset_filtered['movie_id']))
df_movieDataset_filtered = df_movieDataset[df_movieDataset['movie_id'].isin(moviesWithRatings)]
print(f'{len(df_movieDataset_filtered)} out of {len(df_movieDataset)} movies have ratings in rating dataset')
movieDatasetFile_filtered = 'movie_full_dataset_filtered.csv'
df_movieDataset_filtered.to_csv(data_directory + movieDatasetFile_filtered, index = False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11953763744009022, 'fraction_numerical': 0.008598815900761206, 'mean_word_length': 4.0, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '184000', 'license': 'MIT', 'n_tokens_mistral': 2403, 'n_tokens_neox': 2296, 'n_words': 496, 'path': 'movieLens-data-processing.ipynb', 'repo_name': 'yunxiaoli2017/27M-movieLens-dataset-processing', 'size': '7094'}