<jupyter_start><jupyter_text><jupyter_code>from google.colab import drive
drive.mount('/content/drive')
# !pip install wandb
!pip install surprise
!pip install fast_ml
!pip install rake_nltk<jupyter_output>Collecting surprise
  Downloading surprise-0.1-py2.py3-none-any.whl (1.8 kB)
Collecting scikit-surprise
  Downloading scikit-surprise-1.1.1.tar.gz (11.8 MB)
[K     |████████████████████████████████| 11.8 MB 51 kB/s 
[?25hRequirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-surprise->surprise) (1.0.1)
Requirement already satisfied: numpy>=1.11.2 in /usr/local/lib/python3.7/dist-packages (from scikit-surprise->surprise) (1.19.5)
Requirement already satisfied: scipy>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from scikit-surprise->surprise) (1.4.1)
Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.7/dist-packages (from scikit-surprise->surprise) (1.15.0)
Building wheels for collected packages: scikit-surprise
  Building wheel for scikit-surprise (setup.py) ... [?25l[?25hdone
  Created wheel for scikit-surprise: filename=scikit_surprise-1.1.1-cp37-cp37m-linux_x86_64.whl size=1619403 sha256=a4f61a3e397ab73bc1058772c8[...]<jupyter_text>Load the Libraries and load the data<jupyter_code>#manipulating data
import pandas as pd
import numpy as np

from rake_nltk import Rake
#Sklearn
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.model_selection import train_test_split
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.decomposition import PCA, TruncatedSVD,SparsePCA
#load the data
df = pd.read_csv("/content/drive/Shareddrives/Team Stars/Data/df_clean.csv",index_col=0)
df.head(2)
df=df.rename(columns={'reviews_username':'username','reviews_rating':'ratings','reviews_text':'desc'})
from sklearn import preprocessing
le = preprocessing.LabelEncoder()
df.username=le.fit_transform(df.username)
df.id=le.fit_transform(df.id)
df.head()<jupyter_output><empty_output><jupyter_text>Feature Selection<jupyter_code>data=df[['id','username','name','brand','ratings','desc']]
data=data.rename(columns={'username':'userid','name':'productname','id':'productid'})
data.head()<jupyter_output><empty_output><jupyter_text>Content based  Text Preprocessing<jupyter_code>import re
#Text process
data['brand'] = [re.sub(r'[^\w\s]', '', t) for t in data['brand']]
data['desc'] = [re.sub(r'[^\w\s]', '', t) for t in data['desc']]

data['brand'] = [t.lower() for t in data['brand']]
data['desc'] = [t.lower() for t in data['desc']]
data.head()
for index, row in data.iterrows():
  row['desc']=[x.lower().replace(' ','')for x in row['desc']]
  row['desc'] = ''.join(row['desc']).lower()
  row['brand'] = ''.join(row['brand']).lower()
data.head()
 import nltk
nltk.download('stopwords')
nltk.download('punkt')
data['keywords']=""
for index,row in data.iterrows():
  desc=row['desc']
  rake_nltk_var = Rake()
  rake_nltk_var.extract_keywords_from_text(desc)
  keyword_extracted = rake_nltk_var.get_word_degrees()
  row['keywords']=list(keyword_extracted.keys())
  #Intialize rake
  # r=Rake()
  # #extract keywords
  # r.extract_keywords_from_text(desc)
  # #getting the dict with key words and their scores
  # row['keywords'] = r.get_ranked_phrases()

  #assigning the key words to new column
  #  = list(key_words_dict_scores.keys())

#dropping the desc column
#data.drop(columns=['desc'],inplace=True)
# data.keywords.unique()

print(data.keywords)
data.head()<jupyter_output><empty_output><jupyter_text>Collaborative filtering<jupyter_code># pivot table
rmat = data.pivot_table(
    columns = 'productid',
    index = 'userid',
    values = 'ratings'
)
rmat.fillna(0)
#Fit into the title
#Lets vectorize all these reviews. Iinitialize vectorizer
vect = CountVectorizer(analyzer = 'word', ngram_range = (1,2), stop_words = 'english', min_df = 0.002) 
#min_df = rare words, max_df = most used words
#ngram_range = (1,2) - if used more than  1(value), lots of features or noise
vect.fit(data['desc'])
title_matrix = vect.transform(data['desc'])
title_matrix.shape
#Lets find features
features = vect.get_feature_names()
#performing cosine similarity
cosine_sim = cosine_similarity(title_matrix, title_matrix)
cosine_sim.shape
from surprise import Dataset
from surprise import Reader
reader = Reader(rating_scale=(1, 5))
dff = Dataset.load_from_df(data[['userid', 'productid', 'ratings']], reader)
from surprise import SVD
from surprise.model_selection import cross_validate

svd = SVD(verbose=True, n_epochs=10)
# cross_validate(svd, dff, measures=['RMSE', 'MAE'], cv=3, verbose=True)
# split data into train test
trainset, testset = train_test_split(data, test_size=0.3,random_state=10)
trainset = dff.build_full_trainset()
svd.fit(trainset)
# trainset
# svd.predict(uid=10, iid=100)
# hybrid
def hybrid(userid, productid, svd_model,n_recs ):
    
    # sort similarity values in decreasing order and take top 50 results
    sim = list(enumerate(cosine_sim[int(productid)]))
    sim = sorted(sim, key=lambda x: x[1], reverse=True)
    sim = sim[1:50]
    
    # get product metadata
    product_idx = [i[0] for i in sim]
    products = data.iloc[product_idx][['productid', 'ratings']]
    
    # predict using the svd_model
    products['est'] = products.apply(lambda x: svd_model.predict(userid, x['productid'], x['ratings']).est, axis = 1)
    
    # sort predictions in decreasing order and return top n_recs
    products = products.sort_values('est', ascending=False)
    return products.head(n_recs)
  
# generate recommendations
hybrid(244,4539,svd,5)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1502313990744037, 'fraction_numerical': 0.02474190103239587, 'mean_word_length': 6.041353383458647, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '257289', 'license': 'MIT', 'n_tokens_mistral': 2136, 'n_tokens_neox': 2031, 'n_words': 425, 'path': 'Hybrid_Recommenders_for_reviews.ipynb', 'repo_name': 'vngeno/E-COMMERCE-RECOMMENDATION-SYSTEM-', 'size': '5618'}