<jupyter_start><jupyter_text>Topic Modeling - Attempt 1 (Nouns and Adjectives) - Entire Dataset<jupyter_code>import nltk
nltk.download('punkt')
nltk.download('averaged_perceptron_tagger')
from nltk import word_tokenize, pos_tag
# Let's create a function to pull out nouns and adjectives from a string of text
def nouns_adj(text):
    '''Given a string of text, tokenize the text and pull out only the nouns and adjectives.'''
    is_noun_adj = lambda pos: pos[:2] == 'NN' or pos[:2] == 'JJ'
    tokenized = word_tokenize(text)
    nouns_adj = [word for (word, pos) in pos_tag(tokenized) if is_noun_adj(pos)] 
    return ' '.join(nouns_adj)
df_all = pd.DataFrame(df_all['review_text'].apply(nouns_adj))
ps = nltk.stem.PorterStemmer()<jupyter_output><empty_output><jupyter_text>Stemming<jupyter_code>def stem_sentences(sentence):
    tokens = sentence.split()
    stemmed_tokens = [ps.stem(token) for token in tokens]
    return ' '.join(stemmed_tokens)
df_all['review_text'] = df_all['review_text'].apply(stem_sentences)
df_all.head()
from sklearn.feature_extraction import text
add_stop_words = ['translated' , 'googl']
stop_words = text.ENGLISH_STOP_WORDS.union(add_stop_words)
from sklearn.feature_extraction.text import CountVectorizer
cv_noun_adj = CountVectorizer(stop_words = stop_words)
data_cv_noun_adj = cv_noun_adj.fit_transform(df_all['review_text']).toarray()
features_noun_adj = cv_noun_adj.get_feature_names()
df1 = pd.DataFrame(data_cv_noun_adj , columns = features_noun_adj)
term_document_matrix_noun_adj = df1.transpose()
term_document_matrix_noun_adj
sparse_matrix_noun_adj = scipy.sparse.csr_matrix(term_document_matrix_noun_adj)
corpus_noun_adj = matutils.Sparse2Corpus(sparse_matrix_noun_adj)
id2word_noun_adj = corpora.Dictionary([features_noun_adj])
# Number of Topics = 2
lda_noun_adj = models.LdaModel(corpus=corpus_noun_adj, id2word=id2word_noun_adj, num_topics=2, passes=40)
lda_noun_adj.print_topics()
gensim.models.coherencemodel.CoherenceModel(corpus = corpus_noun_adj , model=lda_noun_adj, coherence='u_mass').get_coherence()
# Number of Topics = 3
lda_noun_adj = models.LdaModel(corpus=corpus_noun_adj, id2word=id2word_noun_adj, num_topics=3, passes=40)
lda_noun_adj.print_topics()
gensim.models.coherencemodel.CoherenceModel(corpus = corpus_noun_adj , model=lda_noun_adj, coherence='u_mass').get_coherence()
# Number of Topics = 10
lda_noun_adj = models.LdaModel(corpus=corpus_noun_adj, id2word=id2word_noun_adj, num_topics=10, passes=40)
lda_noun_adj.print_topics()
gensim.models.coherencemodel.CoherenceModel(corpus = corpus_noun_adj , model=lda_noun_adj, coherence='u_mass').get_coherence()<jupyter_output><empty_output><jupyter_text>Topic Modeling - Attempt 1 (Nouns and Adjectives) - Negative Reviews<jupyter_code>df_maps = pd.read_csv(r"C:\Users\<NAME>an\Downloads\ParkReviewsLang.csv")
df_subway = pd.read_csv(r"C:\Users\<NAME>\Downloads\SubwayReviewsLang.csv")
df_yelp = pd.read_csv(r"C:\Users\<NAME>\Downloads\YelpParkReviewsScraped.csv")
df_maps.head()
df_yelp.head()
df_subway.head()
df_maps = df_maps[df_maps['num_stars'] < 4]
df_yelp = df_yelp[df_yelp['Rating'] < 4]
df_subway = df_subway[df_subway['num_stars'] < 4]
df_yelp = df_yelp['Comment']
df_subway = df_subway[df_subway['lang'] == 'en']
df_subway = df_subway['review_text']
df_all = df_maps[df_maps['lang'] == 'en']
df_all = df_all['review_text']
df_all.append(df_yelp)
df_all.append(df_subway)
df_all = pd.DataFrame(df_all)
df_all.shape
df_all = pd.DataFrame(df_all['review_text'].apply(nouns_adj))
ps = nltk.stem.PorterStemmer()<jupyter_output><empty_output><jupyter_text>Stemming<jupyter_code>def stem_sentences(sentence):
    tokens = sentence.split()
    stemmed_tokens = [ps.stem(token) for token in tokens]
    return ' '.join(stemmed_tokens)
df_all['review_text'] = df_all['review_text'].apply(stem_sentences)
from sklearn.feature_extraction import text
add_stop_words = ['translated' , 'googl']
stop_words = text.ENGLISH_STOP_WORDS.union(add_stop_words)
from sklearn.feature_extraction.text import CountVectorizer
cv_noun_adj = CountVectorizer(stop_words = stop_words)
data_cv_noun_adj = cv_noun_adj.fit_transform(df_all['review_text']).toarray()
features_noun_adj = cv_noun_adj.get_feature_names()
df2 = pd.DataFrame(data_cv_noun_adj , columns = features_noun_adj)
term_document_matrix_noun_adj = df2.transpose()
term_document_matrix_noun_adj
sparse_matrix_noun_adj = scipy.sparse.csr_matrix(term_document_matrix_noun_adj)
corpus_noun_adj = matutils.Sparse2Corpus(sparse_matrix_noun_adj)
id2word_noun_adj = corpora.Dictionary([features_noun_adj])
# Number of Topics = 2
lda_noun_adj = models.LdaModel(corpus=corpus_noun_adj, id2word=id2word_noun_adj, num_topics=2, passes=40)
lda_noun_adj.print_topics()
gensim.models.coherencemodel.CoherenceModel(corpus = corpus_noun_adj , model=lda_noun_adj, coherence='u_mass').get_coherence()
# Number of Topics = 3
lda_noun_adj = models.LdaModel(corpus=corpus_noun_adj, id2word=id2word_noun_adj, num_topics=3, passes=40)
lda_noun_adj.print_topics()
gensim.models.coherencemodel.CoherenceModel(corpus = corpus_noun_adj , model=lda_noun_adj, coherence='u_mass').get_coherence()
# Number of Topics = 10
lda_noun_adj = models.LdaModel(corpus=corpus_noun_adj, id2word=id2word_noun_adj, num_topics=10, passes=40)
lda_noun_adj.print_topics()
gensim.models.coherencemodel.CoherenceModel(corpus = corpus_noun_adj , model=lda_noun_adj, coherence='u_mass').get_coherence()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11462160176340926, 'fraction_numerical': 0.010102865540044085, 'mean_word_length': 8.81081081081081, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '367754', 'license': 'MIT', 'n_tokens_mistral': 2250, 'n_tokens_neox': 2150, 'n_words': 298, 'path': 'Topic_Modeling.ipynb', 'repo_name': 'mahajan-abhay/Concordia_University_MITACS', 'size': '5463'}