<jupyter_start><jupyter_text>Import Data from Amazon S3 into DataFrame<jupyter_code># connect to Amazon S3
s3 = boto3.resource('s3')
lyrics = get_file(s3,'s3ssp', download_file='NLP_Data/new_master_lyrics_audio_features.csv',rename_file='nlp.csv')
# create a pandas dataframe and drop 'na'
df = pd.read_csv(lyrics,sep='|',encoding='utf-8')
df_demo = df.copy().dropna()
# print top words in a topic
def print_top_words(model, feature_names, n_top_words):
    for topic_idx, topic in enumerate(model.components_):
        message = "Topic #%d: " % topic_idx
        message += " ".join([feature_names[i]
                             for i in topic.argsort()[:-n_top_words - 1:-1]])
        print(message)
    print()

# Styling
def color_green(val):
 color = 'green' if val > .1 else 'black'
 return 'color: {col}'.format(col=color)
def make_bold(val):
 weight = 700 if val > .1 else 400
 return 'font-weight: {weight}'.format(weight=weight)
# Remove more stopwords identified in pyLDAvis
df_demo['new_lyrics'] = df_demo['lyrics'].apply(lambda x: x.replace("wan", "")
                                                .replace("chorus", "")
                                                .replace("verse", "")
                                                .replace("gon", ""))
#create samples
df_test_one = df_demo.sample(1000)
df_test_two = df_demo.sample(3000)
df_test_three = df_demo.sample(5000)
#CountVectorizer hyperparameters 
max_df = .5
min_df = .005 

#LDA hyperparameters
n_topics = 30
n_words=10
#Fit CountVectorizer on lyrics data
vectorizer = CountVectorizer(analyzer='word',
                             min_df=min_df, max_df=max_df,     
                             stop_words='english',             
                             lowercase=True,                   
                             token_pattern='[a-zA-Z0-9]{3,}')  # num chars > 3  

data_vectorized = vectorizer.fit_transform(df_test_two['new_lyrics'])<jupyter_output><empty_output><jupyter_text>Begin Topic Modeling<jupyter_code># lda model
lda_model = LatentDirichletAllocation(n_components=n_topics, learning_method="online",
                                max_iter=35, random_state=0, doc_topic_prior=.01)
lda_model.fit(data_vectorized)<jupyter_output><empty_output><jupyter_text>Visualize Topics Spatially<jupyter_code># visualize n_topics spatially
pyLDAvis.sklearn.prepare(lda_model, data_vectorized, vectorizer, mds='tsne')
print("\nTopics in LDA model:")
cv_feature_names = vectorizer.get_feature_names()
print_top_words(lda_model, cv_feature_names, 30)<jupyter_output>Topics in LDA model:
Topic #0: girl time talk bad sit guess friend watch road stay wrong pretty walk crazy day play reason care lot fine fun win leave woman eye remember read catch spend wear
Topic #1: fly settle apple snake leave stack ooh yeah singe rock glass worry touch silver roll ball sing hook plane guilty damn fuck dead teach feel king crowd stupid music life
Topic #2: night tonight dance ready rock party alive fight slow jump rhythm building wave floor revolution play thrill hit beat foot monkey hand house shut smile safe rescue spot time step
Topic #3: day life leave hard win bring late forget water wrong reach walk trouble fool climb hang meaning bell nightmare seek decide wise ready hold numb row boat shout ground meet
Topic #4: ooh hand breathe throw bang air shoot leave window woah play foot worth piece cross boom tall scared fake start truck game playin feel lean bare tired stand trigger finger
Topic #5: money shit nigga fuck bitch hit bout niggas baby light stick ass l[...]<jupyter_text>Calculate probabilities and find dominant topics for each document<jupyter_code># Create Document — Topic Matrix

lda_output = lda_model.transform(data_vectorized)

# column names
topicnames = ["Topic" + str(i) for i in range(lda_model.n_components)]

# index names
docnames = [df_test_two['track_uri'].iloc[i] + str(i) for i in range(len(df_test_two))]

# Make the pandas dataframe
df_document_topic = pd.DataFrame(np.round(lda_output, 2), columns=topicnames, index=docnames)

# Get dominant topic for each document
dominant_topic = np.argmax(df_document_topic.values, axis=1)
df_document_topic['dominant_topic'] = dominant_topic

# Apply Style
df_document_topics = df_document_topic.sample(50).style.applymap(color_green).applymap(make_bold)
df_document_topics
# print documents by topic in order of probability
df_document_topic[['Topic1', 'dominant_topic']].sort_values(by='Topic1', ascending=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09290465631929047, 'fraction_numerical': 0.01352549889135255, 'mean_word_length': 3.919302071973828, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 4, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '25805', 'license': 'MIT', 'n_tokens_mistral': 1403, 'n_tokens_neox': 1348, 'n_words': 405, 'path': '5_Machine_Learning/Topic_Modeling_Visualization.ipynb', 'repo_name': 'VintageGold/TSP', 'size': '4510'}