<jupyter_start><jupyter_text><jupyter_code>import pandas as pd
from rake_nltk import Rake
import numpy as np
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.feature_extraction.text import CountVectorizer
import re
import string
pd.set_option('display.max_colwidth',100)
import pandas as pd
import nltk
from langdetect import detect
stopwords = nltk.corpus.stopwords.words('english')

#pip install rake_nltk
#pip install langdetect

#Read data
df = pd.read_csv('https://raw.githubusercontent.com/jessiececilya/ML-Data/main/book_data.csv')

#Taking 2000 samples to limit to the capacity
df= df.iloc[0:2000,:]

#convert author names from "<NAME>" to "harperlee"
def joinnames(text):
  l= ''.join(re.split(" ",text.lower()))
  return l

df['book_authors']= df['book_authors'].apply (lambda x: joinnames(x))
#Replace separators '|' by space
def to_string(x):
    if x:
        return str(x).replace('|',' ')
    else:
        return ''

df['book_authors']= df['book_authors']. apply(lambda x: to_string(x))
df['genres']= df['genres'].apply(lambda x: to_string(x))


#convert to lowercase
df['genres']= df['genres'].apply(lambda x: x.lower())
df['book_title']= df['book_title'].apply(lambda x: x.lower())

#identify the language function
def lang(text):
  try:
    return detect(text)
  except:
    return 'error' 

df['book_desc']= df['book_desc'].astype(str)
df['lang']= df['book_desc'].apply(lambda x: lang(x))
#keep only english language description
df= df[df['lang']=='en']
df=df.reset_index(drop=True)

#clean the book description to perform futher keyword analysis
def clean_text(text):
    text = "".join([word for word in text if word not in string.punctuation])
    tokens = re.split('\W+', text)
    text = [word for word in tokens if word not in stopwords]
    return text
df['book_desc']= df['book_desc'].apply(lambda x: clean_text(x.lower()))
df['book_desc']= df['book_desc'].apply(lambda x: ' '.join(x))

#Keeping desc in a separate variable to not disrupt the cleaned df data
j= pd.DataFrame(df, columns=['book_desc']).astype(str)
#function to identify keywords using Rake
j['Key_words'] = ""

for index, row in j.iterrows():
    desc = row['book_desc']
        
    # instantiating Rake, by default it uses english stopwords from NLTK
    # and discards all puntuation characters as well
    r = Rake()

    # extracting the words by passing the text
    r.extract_keywords_from_text(desc)

    # getting the dictionary whith key words as keys and their scores as values
    key_words_dict_scores = r.get_word_degrees()
    
    # assigning the key words to the new column for the corresponding movie
    row['Key_words'] = list(key_words_dict_scores.keys())



#Combine j keywords into df
df['Keywords']= j['Key_words']
#Create a bag of words using all content based attributes
df['bag_of_words']= df['book_authors'].astype(str) +' ' + df['book_title'].astype(str) +' '+ df['Keywords'].astype(str)
#drop duplicates based on book title columns
x= df.drop_duplicates(subset=['book_title'], keep= 'first')
x=x.reset_index(drop=True)
# instantiating and generating the count matrix
count = CountVectorizer()
count_matrix = count.fit_transform(x['bag_of_words'])

# generating the cosine similarity matrix
cosine_sim = cosine_similarity(count_matrix, count_matrix)
# creating a Series for the movie titles so they are associated to an ordered numerical
# list I will use in the function to match the indexes
indices = pd.Series(x.index)

#  defining the function that takes in movie title 
# as input and returns the top 10 recommended movies
def recommendations(title, cosine_sim = cosine_sim):
    
    # initializing the empty list of recommended movies
    recommended_movies = []
    
    # gettin the index of the movie that matches the title
    idx = indices[indices == title].index[0]

    # creating a Series with the similarity scores in descending order
    score_series = pd.Series(cosine_sim[idx]).sort_values(ascending = False)

    # getting the indexes of the 10 most similar movies
    top_10_indexes = list(score_series.iloc[1:11].index)
    
    # populating the list with the titles of the best 10 matching movies
    for i in top_10_indexes:
        recommended_movies.append(list(x.index)[i])
        
    return recommended_movies
#recommendation list
title= "the hunger games"
j=recommendations(df[df['book_title']==title].index[0], cosine_sim)
for i in j:
  print(df.iloc[i]['book_title'])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1167262969588551, 'fraction_numerical': 0.006037567084078712, 'mean_word_length': 4.8090909090909095, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '323660', 'license': 'MIT', 'n_tokens_mistral': 1471, 'n_tokens_neox': 1416, 'n_words': 443, 'path': 'Book_recommender_content_based.ipynb', 'repo_name': 'jessiececilya/book-recommender-nlp', 'size': '4476'}