<jupyter_start><jupyter_text>1.import data and convert it to dataframe<jupyter_code>business_json_file = 'business.json'
user_json_file = 'user.json'
review_json_file = 'review.json'
business = []
user = []
review = []
for line in open(business_json_file, 'r'):
    business.append(json.loads(line))
for line in open(user_json_file, 'r'):
    user.append(json.loads(line))
for line in open(review_json_file, 'r'):
    review.append(json.loads(line))
business_df = pd.DataFrame.from_records(business)
user_df = pd.DataFrame.from_records(user)
review_df = pd.DataFrame.from_records(review)<jupyter_output><empty_output><jupyter_text>2. extract the userid,businessid, text, rating  ***test: we extract 10k reviews<jupyter_code>review_df = review_df.loc[:10000,['business_id','user_id','text','stars']]
review_df.head()<jupyter_output><empty_output><jupyter_text>3.extract topic  3.1 remove the stopword<jupyter_code>review_stopword = gensim.parsing.preprocessing.STOPWORDS
add_stopword = ['food','place','lot']
review_stopword = review_stopword.union(set(add_stopword))
from nltk import pos_tag
is_noun = lambda pos: pos[:2] == 'NN'

# lemmatizing
def lemmatize(text):
    wordnet_lemmatizer = WordNetLemmatizer()
    return wordnet_lemmatizer.lemmatize(text)

# Tokenize and lemmatize
def preprocess(text):
    tokens = []
    for word in text.split():
        if word.isalpha() == True:
            tokens.append(word.lower())
    #remove stopwords
    tokens = [w for w in tokens if not w in review_stopword]
    tokens = [w for w in tokens if not len(w)<3]
    #keep only noun
    tokens = [w for (w, pos) in pos_tag(tokens) if is_noun(pos)]
    #remove duplicate
    tokens = list(dict.fromkeys(tokens))
    tokens = [lemmatize(w) for w in tokens] 
            
    return tokens

review_df['text'] = review_df['text'].apply(lambda x: preprocess(x))<jupyter_output><empty_output><jupyter_text>3.2 duplicate the words according to the stars<jupyter_code>review_df['text'] = review_df['text'] * review_df['stars'].apply(int)
reviews = review_df['text']
# Associate words to a numeric ID
dictionary = gensim.corpora.Dictionary(reviews) 
# Transform the collection of texts to a numerical form
bow_corpus = [dictionary.doc2bow(doc) for doc in reviews] 
def compute_coherence_values(dictionary, corpus, texts, limit, start=2, step=3):
    """
    Compute c_v coherence for various number of topics

    Parameters:
    ----------
    dictionary : Gensim dictionary
    corpus : Gensim corpus
    texts : List of input texts
    limit : Max num of topics

    Returns:
    -------
    model_list : List of LDA topic models
    coherence_values : Coherence values corresponding to the LDA model with respective number of topics
    """
    coherence_values = []
    model_list = []
    for num_topics in range(start, limit, step):
        model=LdaModel(corpus=corpus, id2word=dictionary, num_topics=num_topics)
        model_list.append(model)
        coherencemodel = CoherenceModel(model=model, texts=texts, dictionary=dictionary, coherence='c_v')
        coherence_values.append(coherencemodel.get_coherence())

    return model_list, coherence_values
# Warning: This cell takes a long time to run
model_list, coherence_values = compute_coherence_values(dictionary=dictionary, 
                                                        corpus=bow_corpus, texts=reviews, start=10, limit=20, step=1)<jupyter_output><empty_output><jupyter_text>3.3<jupyter_code>limit=20; start=10; step=1;
x = range(start, limit, step)

plt.plot(x,coherence_values)
plt.xlabel("Num Topics")
plt.ylabel("Coherence score")
plt.legend(("coherence_values"), loc='best')
plt.show()<jupyter_output><empty_output><jupyter_text>3.4 build the LDA model<jupyter_code>lda_model =  gensim.models.LdaMulticore(bow_corpus, 
                                   num_topics = 5, 
                                   id2word = dictionary,                                    
                                   passes = 10,
                                   workers = 2)
for idx in range(5):
    # Print the first 16 most representative topics
    print("Topic #%s:" % idx, lda_model.print_topic(idx, 10))
# Displaying topics in a table (a better visualisation)
word_dict = {};
for i in range(5):
    words = lda_model.show_topic(i, topn = 50)
    word_dict['Topic # ' + '{:02d}'.format(i)] = [i[0] for i in words]
word_dict = pd.DataFrame(word_dict)
topic_word = {}
for i in range(1,6):
    word_list = []
    for w in word_dict.iloc[:,i-1]:
        word_list.append(w)
    topic_word[i] = word_list
new_list = []
for k, v in topic_word.items():
    new_list.append([k,v])
    
new_list = pd.DataFrame(data = new_list)
def format_topics_sentences(ldamodel, corpus, texts):
    # Init output
    sent_topics_df = pd.DataFrame()

    # Get main topic in each document
    for i, row in enumerate(ldamodel[corpus]):
        row = sorted(row, key=lambda x: (x[1]), reverse=True)
        ##
        ##
        ##
        # Add the row code here
        ##
        ##
        ##
        # Get the Dominant topic, Perc Contribution and Keywords for each document
        for j, (topic_num, prop_topic) in enumerate(row):
            if j == 0:  # => dominant topic
                wp = ldamodel.show_topic(topic_num)
#                topic_keywords = ", ".join([word for word, prop in wp])
                sent_topics_df = sent_topics_df.append(pd.Series([int(topic_num), round(prop_topic,4)]), ignore_index=True)
            else:
                break
    sent_topics_df.columns = ['Dominant_Topic', 'Perc_Contribution']

    # Add original text to the end of the output
    contents = pd.Series(texts)
    sent_topics_df = pd.concat([sent_topics_df, contents], axis=1)
    return(sent_topics_df)
# Warning: This cell takes quite some time to run
df_topic_sents_keywords = format_topics_sentences(ldamodel=lda_model, corpus=bow_corpus, texts=reviews)

# Format
df_dominant_topic = df_topic_sents_keywords.reset_index()
df_dominant_topic.columns = ['Document_No', 'Dominant_Topic', 'Topic_Perc_Contrib', 'reviewTxt']

# Show
df_dominant_topic.head(10)
word_dict<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.114276412176461, 'fraction_numerical': 0.009604427804004558, 'mean_word_length': 3.507703595011005, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '229907', 'license': 'MIT', 'n_tokens_mistral': 2104, 'n_tokens_neox': 2004, 'n_words': 467, 'path': 'LDA_with_sudodocu.ipynb', 'repo_name': 'vivienfay/Recommendation-System-based-on-Yelp-Dataset', 'size': '6143'}