<jupyter_start><jupyter_text>In this notebook, we want to forcast the ratings of the first 3000 users<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
from sklearn import cluster
import pandas as pd<jupyter_output><empty_output><jupyter_text>Clustering the history data<jupyter_code>history = pd.read_csv('user_history.csv')
history.shape
# We want to first clustering the history data
# By using elbow method
sse = []
list_k = list(range(1, 11))

for k in list_k:
    km = cluster.KMeans(n_clusters=k)
    km.fit(history.drop(['USER ID'], axis=1)) # we need to drop the first column(user id)
    sse.append(km.inertia_)

# Plot sse against k
plt.figure(figsize=(5, 5))
plt.plot(list_k, sse, '-o')
plt.xlabel('Number of clusters')
plt.ylabel('Sum of squared distance')
plt.show()
# According to elbow method, the best k should be 3
kmeans = cluster.KMeans(3) 
kmeans.fit(history.drop(['USER ID'], axis=1))
kmeans_y = kmeans.predict(history.drop(['USER ID'], axis=1))
# Seperate first 3000 users by their labels
# Create three arrays for each cluster/group of users
First_Group = []
Second_Group = []
Third_Group = []

for i in range(3000): 
    if kmeans.labels_[i] == 0:
        First_Group.append(history['USER ID'].iloc[i])
    if kmeans.labels_[i] == 1:
        Second_Group.append(history['USER ID'].iloc[i])
    if kmeans.labels_[i] == 2:
        Third_Group.append(history['USER ID'].iloc[i])
        
First_Group = np.asarray(First_Group)
Second_Group = np.asarray(Second_Group)
Third_Group = np.asarray(Third_Group)
len(First_Group),len(Second_Group),len(Third_Group)<jupyter_output><empty_output><jupyter_text>Using Gradient Descent to predict the rating for the first 3000 users<jupyter_code># we can see that our reformat rating matrix is pretty sparse
reformat = pd.read_csv('user_data.csv')
reformat
#Generate Three Dataframes corresponding to train arrays for all three clusters
Clstr1_train = reformat[reformat['USER_ID'].isin(First_Group)]
Clstr2_train = reformat[reformat['USER_ID'].isin(Second_Group)]
Clstr3_train = reformat[reformat['USER_ID'].isin(Third_Group)]

##Since the target products in each dataframes are the same: exacly as those 100 products,
##So we store those products into a list
target_products = list(reformat.columns) 
target_products.remove(target_products[0]) #remove the item 'USER_ID'
target_products = np.asarray(target_products)

#type(target_products)
#print(target_products)
#len(target_products)
#After rechecking, we know that array is what we need.
def load_rating_clusters(dataframe):
    users = []
    products = []
    mapusers = []
    mapproducts = []
    ratings = []
    temp1 = dataframe.iloc[:,1:101]

    temp2 = np.array(temp1)
    rownumber = len(temp2)
    colnumber = temp2.shape[1]
    #Get ndArray of all column names 
    columnNames = list(dataframe.columns) 
    columnNames.remove(columnNames[0]) #remove the item 'USER_ID'
    columnNames = np.asarray(columnNames)
    #Get ndArray of all column names 
    rowNames = dataframe.iloc[:, 0]
    rowNames = np.array(rowNames)
    print(rownumber)
    print(colnumber)


    for i in range(rownumber): 
        for j in range(colnumber):
            if np.isnan(temp2[i,j]):
                continue
            users.append(rowNames[i])
            mapusers.append(i)
            products.append(columnNames[j])
            mapproducts.append(j)
            ratings.append(temp2[i,j])
    #reformat3_ratings = np.array(reformat3_ratings)
    users = np.array(users)

    products = np.array(products)
    mapusers = np.array(mapusers)
    mapproducts = np.array(mapproducts)
    ratings = np.array(ratings)
    return users,products,ratings,mapusers,mapproducts

users1_train,products1_train,ratings1_train,mapusers1_train,mapproducts1_train = load_rating_clusters(Clstr1_train)
users2_train,products2_train,ratings2_train,mapusers2_train,mapproducts2_train = load_rating_clusters(Clstr2_train)
users3_train,products3_train,ratings3_train,mapusers3_train,mapproducts3_train = load_rating_clusters(Clstr3_train)
learning_rate = 0.005

def initialize(n_users, n_products):
    """Initalize a random model, and normalize it so that it has sensible mean and variance"""
    # (The normalization helps make sure we start out at a reasonable parameter scale, which speeds up training)
    user_features = np.random.normal(size=(n_users,10)) # still setting the number of features as 10
    product_features = np.random.normal(size=(n_products,10)) # still setting the number of features as 10
    raw_predictions = predict((user_features, product_features))
    
    s = np.sqrt(2*raw_predictions.std()) # We want to start out with roughly unit variance
    b = np.sqrt(5 - raw_predictions.mean()/s) #We want to start out with average rating 5
    user_features /= s
    user_features += b
    product_features /= s
    product_features += b
    
    return (user_features, product_features)

def predict(model):
    """The model's predictions for all user/product pairs"""
    user_features, product_features = model
    return user_features @ product_features.T

def single_example_step(model, user, product, rating, train_users, train_products):
    """Update the model using the gradient at a single training example"""
    user_features, product_features = model
    # only consider rating which is not 'nan'

    residual = np.dot(user_features[user], product_features[product]) - rating
    #for i in range(len(residual)):
    #    residual[i]-=rating[i]
    grad_users = 2 * residual * product_features[product] # the gradient for the user_features matrix
    grad_products = 2 * residual * user_features[user] # the gradient for the product_features matrix
    #print(grad_users.shape,grad_products.shape)
    user_features[user] -= learning_rate*grad_users
    product_features[product] -= learning_rate*grad_products

def train_sgd(model, epochs, train_users, train_products, train_ratings, test_users, test_products, test_ratings,mapusers_train,mapproducts_train,mapusers_test,mapproducts_test):
    """Train the model for a number of epochs via SGD (batch size=1)"""
    user_features, movie_features = model
    # It's good practice to shuffle your data before doing batch gradient descent,
    # so that each mini-batch peforms like a random sample from the dataset
#     print()  
    training_process = []
    uniquetrain_users = list(np.unique(train_users))
    uniquetrain_products = list(np.unique(train_products))
    uniquetest_users = np.unique(test_users)
    uniquetest_products = np.unique(test_products)
    shuffle = np.random.permutation(len(train_users))
    shuffled_users = train_users[shuffle]
    shuffled_products = train_products[shuffle]
    shuffled_ratings = train_ratings[shuffle]
    
    for epoch in range(epochs):
        for user, product, rating in zip(shuffled_users, shuffled_products, shuffled_ratings):
            # update the model using the gradient at a single example
            if np.isnan(rating):
                continue
            user = uniquetrain_users.index(user)
            product = uniquetrain_products.index(product)
            single_example_step(model, user, product, rating, uniquetrain_users, uniquetrain_products)
        # after each Epoch, we'll evaluate our model
        
        predicted = predict(model)

        train_loss = np.mean((train_ratings - predicted[mapusers_train,mapproducts_train])**2)
        test_loss = np.mean((test_ratings - predicted[mapusers_test,mapproducts_test])**2)
        training_process.append((epoch,train_loss))
        print("Loss after epoch #{} is: train/{} --- test/{}".format(epoch+1, train_loss, test_loss))
    return training_process

sgd_model_1 = initialize(len(list(np.unique(users1_train))), 100)
sgd_model_2 = initialize(len(list(np.unique(users2_train))), 100)
sgd_model_3 = initialize(len(list(np.unique(users3_train))), 100)
training_process_1 = train_sgd(sgd_model_1, 20, users1_train, products1_train, ratings1_train, users1_train, products1_train, ratings1_train,mapusers1_train,mapproducts1_train,mapusers1_train,mapproducts1_train)
training_process_2 = train_sgd(sgd_model_2, 20, users2_train, products2_train, ratings2_train, users2_train, products2_train, ratings2_train,mapusers2_train,mapproducts2_train,mapusers2_train,mapproducts2_train)
training_process_3 = train_sgd(sgd_model_3, 20, users3_train, products3_train, ratings3_train, users3_train, products3_train, ratings3_train,mapusers3_train,mapproducts3_train,mapusers3_train,mapproducts3_train)
predicted_1 = predict(sgd_model_1)
predicted_2 = predict(sgd_model_2)
predicted_3 = predict(sgd_model_3)

newpredicted_1 = pd.DataFrame(predicted_1,index = Clstr1_train.index,columns = Clstr1_train.columns[1:])
newpredicted_1.insert(0,'USER_ID', Clstr1_train['USER_ID'])
newpredicted_2 = pd.DataFrame(predicted_2,index = Clstr2_train.index,columns = Clstr2_train.columns[1:])
newpredicted_2.insert(0,'USER_ID', Clstr2_train['USER_ID'])
newpredicted_3 = pd.DataFrame(predicted_3,index = Clstr3_train.index,columns = Clstr3_train.columns[1:])
newpredicted_3.insert(0,'USER_ID', Clstr3_train['USER_ID'])
Clstr1_train[Clstr1_train.isnull()] = newpredicted_1[Clstr1_train.isnull()]
Clstr2_train[Clstr2_train.isnull()] = newpredicted_2[Clstr2_train.isnull()]
Clstr3_train[Clstr3_train.isnull()] = newpredicted_3[Clstr3_train.isnull()]

Clstr_train = pd.concat([Clstr1_train,Clstr2_train,Clstr3_train])
Clstr_train = Clstr_train.sort_values(by="USER_ID",ascending=True).round(0).to_csv("tzzs_data.csv")<jupyter_output>//anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  """Entry point for launching an IPython kernel.
//anaconda3/lib/python3.7/site-packages/pandas/core/frame.py:3414: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  self._where(-key, value, inplace=True)
//anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/sta[...]
{'fraction_non_alphanumeric': 0.10230615924836291, 'fraction_numerical': 0.019834867609376482, 'mean_word_length': 5.05632183908046, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '344095', 'license': 'MIT', 'n_tokens_mistral': 3535, 'n_tokens_neox': 3490, 'n_words': 776, 'path': 'first 3000(commented by KM).ipynb', 'repo_name': 'Userssname/RecSYS122', 'size': '10537'}