<jupyter_start><jupyter_text>Import the pandas dataframe containing the info from the review scrape<jupyter_code>als_df = pd.read_pickle('../data/als_df.pkl')
als_df.reset_index(drop=True, inplace=True)
als_df = als_df[['user_id', 'item_id', 'rating', 'date']]
als_df.shape
len(als_df['user_id'].unique())<jupyter_output><empty_output><jupyter_text>Start a spark session to train an ALS model<jupyter_code>spark = SparkSession.builder.getOrCreate()

spark_als_df = spark.createDataFrame(als_df) 

spark_als_df.printSchema()
spark_als_df.limit(5).show()
pandas_als_df = spark_als_df.toPandas()
# unnecessary to split into test and train with ALS

# train = pandas_als_df[:int(len(pandas_als_df)*.8)]
# test = pandas_als_df[int(len(pandas_als_df)*.8):]
test = test[['user_id', 'item_id', 'rating']]
test.head()
train = train[['user_id', 'item_id', 'rating']]
train.head()
train.shape
als_model = ALS(
    itemCol='item_id',
    userCol='user_id',
    ratingCol='rating',
    nonnegative=True,    
    regParam=0.1,
    rank=10,
    coldStartStrategy = 'drop') 
spark_train = spark.createDataFrame(train) 
spark_test = spark.createDataFrame(test)
recommender = als_model.fit(spark_train)
train_preds = recommender.transform(spark_train)
train_preds
test_preds = recommender.transform(spark_test)
evaluator = RegressionEvaluator(metricName="rmse", labelCol="rating",
                                predictionCol="prediction")
rmse_train = evaluator.evaluate(train_preds)
rmse_train
rmse_test = evaluator.evaluate(test_preds)
rmse_test
#restaurant_recs = recommender.recommendForAllUsers(10)
restaurant_recs.limit(5).toPandas()<jupyter_output><empty_output><jupyter_text>Addressing the Cold Start Problem<jupyter_code>user_factors = recommender.userFactors.collect()
item_factors = recommender.itemFactors.collect()
user_factors[0]
item_factors[0]
uf_df = recommender.userFactors.toPandas()
if_df = recommender.itemFactors.toPandas()
if_df
user_factors_array = np.array(uf_df['features'].tolist())
item_factors_array = np.array(if_df['features'].tolist())
user_factors_array.shape
item_factors_array.shape
def predict_rating(user_idx, item_idx):
    """Return the predicted rating of item by user (by iloc)."""
    user_vector = user_factors_array[user_idx, :]
    item_vector = item_factors_array[item_idx, :].T
    return user_vector @ item_vector
def predict_rating_by_id(user_id, item_id):
    """Return the predicted rating of item by user (by id)."""
    user_idx = uf_df.index[uf_df['id'] == user_id][0]
    item_idx = if_df.index[if_df['id'] == item_id][0]
    return predict_rating(user_idx, item_idx)
predict_rating_by_id(471, 59)
predict_rating(1, 1)
user_idx = uf_df.index[uf_df['id'] == 471][0]
item_idx = if_df.index[if_df['id'] == 59][0]
item_idx
item_factors_array[item_idx]
pred_ratings = user_factors_array[user_idx, :] @ item_factors_array[554, :].T
def get_restaurant_indexes(user_ratings, item_factors_df):
    rest_idxs = []
    for item in user_ratings['item_id']:
        rest_idx = if_df.index[if_df['id']==item]
        rest_idxs.append(rest_idx[0])
    return np.array(rest_idxs)
        
rest_idxs = []
for item in user_ratings['item_id']:
    rest_idx = if_df.index[if_df['id']==item]
    rest_idxs.append(rest_idx[0])
#restaurant_indexes = get_restaurant_indexes(user_ratings, if_df)
item_factors_array[rest_idxs]<jupyter_output><empty_output><jupyter_text>X is our new user's factors vector; use to generate a predicted ratings row for new user<jupyter_code>X, residuals, rank, s = np.linalg.lstsq(item_factors_array[rest_idxs], ratings.values)
newuser_factors = X
def newuser_predict(newuser_factors, item_factors_array):
    new_factor_list =[]
    for i in range(len(item_factors_array)):
        new_factor_list.append(np.dot(newuser_factors, item_factors_array[i]))
    newuser_preds = pd.DataFrame([new_factor_list], index=['newuser'])
    return newuser_preds
example_new_user = newuser_predict(newuser_factors, item_factors_array)
example_new_user<jupyter_output><empty_output><jupyter_text>Now... get the actual restaurant names here and sort the predictions to get the top preds<jupyter_code>inv_alias_dict = {v: k for k, v in alias_dict.items()}
inv_alias_dict
example_new_user_rest_names = example_new_user.rename(inv_alias_dict, axis=1)
example_new_user_rest_names.sort_values(by='newuser', axis=1, ascending=False)<jupyter_output><empty_output><jupyter_text>Ok... so now try to use this with actual survey results<jupyter_code>from survey_results import survey_results, usernames
gabe = survey_results[0]
gabe
#divide ratings by two to put on the same scale
#convert aliases -> ids -> indexes for those ids
#find the item feature vectors for each of those restaurants by indexing into the item_factors_array
#make a "user ratings df" with the user, restaurant ids, and ratings
#divide ratings by two to put on the same scale
gabe_standardized = {k: v / 2 for k, v in gabe.items()}
gabe_standardized
#inv_alias_dict
#convert aliases -> ids -> indexes for those ids
id_to_rating = {k: gabe_standardized[v] for k, v in inv_alias_dict.items() if v in gabe_standardized}
user_ratings_df = pd.DataFrame.from_dict(id_to_rating, orient='index')
user_ratings_df.reset_index(inplace=True)
user_ratings_df.rename(columns={'index':'item_id', 0:'rating'}, inplace=True)

survey_ratings = user_ratings_df['rating'].values.reshape(-1, 1)
survey_ratings_notreshaped = user_ratings_df['rating'].values
rest_idx = get_restaurant_indexes(user_ratings_df, if_df)
latent_item_features = item_factors_array[rest_idx]
X, residuals, rank, s = np.linalg.lstsq(latent_item_features, survey_ratings_notreshaped)
X
gabes_preds = newuser_predict(X, item_factors_array)
gabes_preds
gabe_sorted_preds = gabes_preds.sort_values(by='newuser', axis=1, ascending=False).rename(inv_alias_dict, axis=1)
list(gabe_sorted_preds.columns)<jupyter_output><empty_output><jupyter_text>Make some functions that do all this, import it, try it out<jupyter_code>from ALS import ALSRecommender
nicole = survey_results[1]
a_recommender = ALSRecommender(uf_df, if_df, inv_alias_dict)
nicole_preds = a_recommender.get_preds_from_survey_results(nicole)
nicole_preds
jonny = survey_results[3]
jonny_preds = a_recommender.get_preds_from_survey_results(jonny)
jonny_preds
jonny_preds
alex = survey_results[2]
alex_preds = a_recommender.get_preds_from_survey_results(alex, inv_alias_dict)
alex_preds
all_users = pd.DataFrame(index=range(len(survey_results)), columns=inv_alias_dict.values())
all_users
a = ALSRecommender(uf_df, if_df, inv_alias_dict)
#all_users = pd.DataFrame(index=range(len(survey_results)), columns=inv_alias_dict.values())

for survey in survey_results:
    results = a.get_preds_from_survey_results(survey)
results<jupyter_output><empty_output><jupyter_text>bring in the alias_dict to translate rest aliases to ids<jupyter_code>more_than_one_review = pd.read_pickle('more_than_one_review_df.pkl')
aliases = more_than_one_review['alias'].unique()
alias_ids = zip(aliases, range(len(aliases)))
alias_dict = dict(alias_ids)
with open("alias_dict.txt", "wb") as myFile:
    pickle.dump(alias_dict, myFile)
gabe_df = pd.DataFrame(columns=alias_dict.keys(), index=usernames)
gabe_df
def create_user_df(survey, df):
    for key, value in survey.items():
        df.iloc[0][key] = survey[key]
    return df
gabe = create_user_df(gabe, gabe_df).loc['gabe']<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11296221048354327, 'fraction_numerical': 0.006230529595015576, 'mean_word_length': 7.195338512763596, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '166418', 'license': 'MIT', 'n_tokens_mistral': 2771, 'n_tokens_neox': 2690, 'n_words': 500, 'path': 'notebooks/ALS.ipynb', 'repo_name': 'Gnishimura/We_Eat', 'size': '7383'}