<jupyter_start><jupyter_text>Recommender model comparisons<jupyter_code>from pyspark.ml import recommendation
mPath =  "als_recommender"
persisted_ALS_model = recommendation.ALSModel.load(mPath)

pickle_in = open("lightfm-no-features.pkl","rb")
persisted_light_FM = pickle.load(pickle_in)
pickle_in_dataset = open("dataset-lightfm-no-features.pkl","rb")
persisted_dataset_light_FM = pickle.load(pickle_in_dataset)

validata = pd.read_csv('data/test_ratings_df.csv')
def calc_prediction_scores(holdout_set):
    count = 0
    user_ground_truth, movie_mean_rating, lfm_rating, als_rating = [], [], [], []
    ho_users = holdout_set.groupby('userId').count().reset_index()['userId']
    for user in ho_users:
        count += 1
        if count < 4000:
            sampled_user_movies = holdout_set[holdout_set['userId'] == user][['movieId','rating','mean']]#'movie Id numbers, not indices'
            sampled_user_movies['userId'] = user

            movie_samps = []
            for movie in sampled_user_movies['movieId']:
                movie_samps.append(persisted_dataset_light_FM.mapping()[2][movie])
                
            # ALS Model predictions
            df2=spark.createDataFrame(sampled_user_movies)
            ALS_pred=persisted_ALS_model.transform(df2).toPandas()
#             print(ALS_pred.head(3))
            
            #LightFM model predictions
            prediction = persisted_light_FM.predict(user_ids = persisted_dataset_light_FM.mapping()[0][user], item_ids = movie_samps, item_features=None, user_features=None)
            sampled_user_movies['lfm_predict'] = prediction
#             print(sampled_user_movies.sort_values(by='lfm_predict',ascending=False).head(3))
            
            sampled_user_movies=sampled_user_movies.drop(['rating', 'mean','userId'], axis=1)
            sampled_user_movies=sampled_user_movies.merge(ALS_pred, how='inner', left_on='movieId', right_on='movieId')
            sampled_user_movies=sampled_user_movies.rename(columns = {'prediction':'als_predict'})
            
            user_ground_truth.append(sampled_user_movies.sort_values('rating', ascending=False).head(5).sum()['rating'] / 5)
            movie_mean_rating.append(sampled_user_movies.sort_values('mean', ascending=False).head(5).sum()['rating'] / 5)
            lfm_rating.append(sampled_user_movies.sort_values('lfm_predict', ascending=False).head(5).sum()['rating'] / 5)
            als_rating.append(sampled_user_movies.sort_values('als_predict', ascending=False).head(5).sum()['rating'] / 5)
    
    return user_ground_truth, movie_mean_rating, lfm_rating, als_rating, np.mean(user_ground_truth), np.std(user_ground_truth), np.mean(movie_mean_rating), np.std(movie_mean_rating), np.mean(lfm_rating), np.std(lfm_rating), np.mean(als_rating), np.std(als_rating)
        
ground, pop_rtg, lfm_rtg, als_rtg, ground_mean, ground_std, pop_best, pop_std, lfm_best, lfm_std, als_best, als_std = calc_prediction_scores(validata)
print(ground_mean, pop_best, lfm_best, als_best)
bench_test = pd.DataFrame(ground, columns=(['ground']))
bench_test['pop_rating'] = pd.DataFrame(pop_rtg)
bench_test['lfm_rating'] = pd.DataFrame(lfm_rtg)
bench_test['als_rating'] = pd.DataFrame(als_rtg)
bench_test.head(20)
bench_test.to_csv('data/bench_test.csv', index=False)<jupyter_output><empty_output><jupyter_text>Statistical comparisons- How do the results of the bench test compare? - Develop a hypothesis, and test the hypothesis.- Hypothesis: The mean score on the bench test data for the ALS model is 0.2 points higher than the score for the LFM model.- test statistic, p, will give us the probability of seeing observed results that are at least as extreme as what was measured.<jupyter_code>def welch_test_statistic(sample_1, sample_2):
    numerator = np.mean(sample_1) - np.mean(sample_2)
    denominator_sq = (np.var(sample_1) / len(sample_1)) + (np.var(sample_2) / len(sample_2))
    return numerator / np.sqrt(denominator_sq)

def welch_satterhwaithe_df(sample_1, sample_2):
    ss1 = len(sample_1)
    ss2 = len(sample_2)
    df = (
        ((np.var(sample_1)/ss1 + np.var(sample_2)/ss2)**(2.0)) / 
        ((np.var(sample_1)/ss1)**(2.0)/(ss1 - 1) + (np.var(sample_2)/ss2)**(2.0)/(ss2 - 1))
    )
    return df
# Load data
bench_test = pd.read_csv('data/bench_test.csv')
bench_test.describe()
test_statistic = welch_test_statistic(nyt_s_in_yes['CTR'], nyt_s_in_no['CTR'])
print("Welch Test Statistic signed in vs not: {:2.2f}".format(test_statistic))

df = welch_satterhwaithe_df(nyt_s_in_yes['CTR'], nyt_s_in_no['CTR'])
print("Degrees of Freedom for Welch's Test: {:2.2f}".format(df))

students = stats.t(df)
p_value = students.cdf(test_statistic) + (1 - students.cdf(-test_statistic))
print("p-value for signed in vs not different click-thru rate: {:2.2f}".format(p_value))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1328512396694215, 'fraction_numerical': 0.013223140495867768, 'mean_word_length': 4.6487747957993, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '351802', 'license': 'CC0-1.0', 'n_tokens_mistral': 1747, 'n_tokens_neox': 1685, 'n_words': 270, 'path': 'stat_comparisons.ipynb', 'repo_name': 'salvir1/a-better-recommender', 'size': '4840'}