<jupyter_start><jupyter_text>Hashtag analysis  Long code, fast execution: create multiple new entities and assemble hashtags This code creates new fields with multiple hashtags for each tweet and one field with popular hashtags.Credits go to [Our Coding Club and James](https://ourcodingclub.github.io/tutorials/topic-modelling-python/). Some preprocessing in the form of new modules:<jupyter_code>import pandas as pd
from os.path import join
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
from numpy import nan

# model building package
import sklearn

# package to clean text
import re<jupyter_output><empty_output><jupyter_text>Topic Modelling/Voyant text preparation To make topic topelling possible, you need to strip the text of all insignificant strips of words and make them as coherent as possible without losing quality. Key to this is a cleaning master function.<jupyter_code>import pandas as pd
from os.path import join
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
from numpy import nan

# alternative model building package
import sklearn
from sklearn.decomposition import LatentDirichletAllocation
from sklearn.model_selection import GridSearchCV
# from sklearn.decomposition import NMF

# package to clean text
import re

# Load modules for cleaning tasks
import nltk
from nltk.tokenize import RegexpTokenizer
from nltk.corpus import stopwords
from nltk.stem.snowball import GermanStemmer
src = "/media/s/Linux_storage/Analyse_Verkehrswende_Transformation/Data/Greenwashing"

fields = ['entities.hashtags', 'created_at', 'text', 'entities.urls', 'lang']

df = pd.read_csv(join(src, 'greenwashing.csv'), encoding='utf-8', dtype='unicode', 
                   parse_dates=['created_at'], usecols=fields)<jupyter_output><empty_output><jupyter_text>Cleaning Master Function<jupyter_code># Standard cleaning tasks

def remove_links(tweet):
    '''Takes a string and removes web links from it'''
    tweet = re.sub(r'http\S+', '', tweet) # remove http links
    tweet = re.sub(r'bit.ly/\S+', '', tweet) # remove bitly links
    tweet = tweet.strip('[link]') # remove [links]
    return tweet

def remove_users(tweet):
    '''Takes a string and removes retweet and @user information'''
    tweet = re.sub('RT @[\w_]+:', '', tweet) # remove retweet ## Alternatives: RT @[\w_]+: OR (RT\s@[A-Za-z]+[A-Za-z0-9-_]+)
    tweet = re.sub('@[A-Za-z0-9_]', '', tweet) # remove tweeted at ## Alternatives: @[A-Za-z0-9_]+ OR (@[A-Za-z]+[A-Za-z0-9-_]+)
    return tweet

my_stopwords = nltk.corpus.stopwords.words('english')
#word_rooter = nltk.stem.snowball.GermanStemmer(ignore_stopwords=False).stem
#word_rooter2 = nltk.stem.snowball.PorterStemmer(ignore_stopwords=False).stem
#my_punctuation = '!"$%&\'()*+,-./:;<=>?[\\]^_`{|}~•@' ## Alternatives: r'[^\w\s] OR !"$%&\'()*+,-./:;<=>?[\\]^_`{|}~•@ 

# cleaning master function
def clean_tweet(tweet, bigrams=False): # change bigrams to True to enable further analysis
    tweet = remove_users(tweet)
    tweet = remove_links(tweet)
    tweet = tweet.lower() # lower case
    #tweet = re.sub('['+my_punctuation + ']+', ' ', tweet) # strip punctuation
    tweet = re.sub(r'[^\w\s]', ' ', tweet) # strip punctuation
    tweet = re.sub(r'\b\w{1,3}\b', '', tweet, re.UNICODE)
    tweet = re.sub('\s+', ' ', tweet, re.UNICODE) # remove double spacing
    tweet = re.sub('([0-9]+)', '', tweet, re.UNICODE) # remove numbers
    tweet_token_list = [word for word in tweet.split(' ')
                            if word not in my_stopwords] # remove stopwords
    #tweet_token_list = [word_rooter(word) if '#' not in word else word 
    #                    for word in tweet_token_list] # apply word rooter
    #tweet_token_list = [word_rooter2(word) if '#' not in word else word 
    #                    for word in tweet_token_list] # apply word rooter
    if bigrams:
        tweet_token_list = tweet_token_list+[tweet_token_list[i]+'_'+tweet_token_list[i+1]
                                            for i in range(len(tweet_token_list)-1)]
    tweet = ' '.join(tweet_token_list)
    return tweet<jupyter_output><empty_output><jupyter_text>No filterProceed with the following cell or choose a filter below.<jupyter_code>df['text'] = df['text'].astype(str)
df['clean_tweet'] = df.text.apply(clean_tweet) # This gets cleaned tweets

#df['clean_tweet'] = df['clean_tweet'].apply(
#    lambda x: x.replace("test", "")) # Merge certain singular words
df['clean_tweet'].head()<jupyter_output><empty_output><jupyter_text>Language-based filterThe following cell allows to set a filter on content only published in a certain language. Note that the "lang" column produced by Twitter might produce artefacts.<jupyter_code>en = df[df['lang']=='en']
# de = df[df['lang']=='de']
# Test the results through uncommenting
# df
# en

en['text'] = en['text'].astype(str)
en['clean_tweet'] = en.text.apply(clean_tweet)
#de['text'] = de['text'].astype(str)
#de['clean_tweet'] = de.text.apply(clean_tweet)<jupyter_output><ipython-input-5-38694041710f>:5: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  en['text'] = en['text'].astype(str)
<ipython-input-5-38694041710f>:6: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  en['clean_tweet'] = en.text.apply(clean_tweet)<jupyter_text>Topic Modelling<jupyter_code># Dropping NA is only relevant for a focused analysis, with only a few columns available and hashtags at the centre of attention.
# en = en.dropna()
# df = df.dropna()
#en
# Add additional stopwords if necessary
df['clean_tweet'] = df['clean_tweet'].apply(
    lambda x: x.replace("greenwashing", "")) # Merge certain singular words
# Add additional stopwords if necessary
en['clean_tweet'] = en['clean_tweet'].apply(
    lambda x: x.replace("greenwashing", "")) # Merge certain singular words
from sklearn.feature_extraction.text import CountVectorizer

# the vectorizer object will be used to transform text to vector form
vectorizer = CountVectorizer(strip_accents = 'unicode',
                                stop_words = 'english',
                                lowercase = True,
                                token_pattern = r'\b[a-zA-Z]{3,}\b',
                                max_df = 0.9, 
                                min_df = 25)

# apply transformation to get Matrix
tf = vectorizer.fit_transform(en['clean_tweet']) #.toarray() # Change between df OR a language filter
#tf = vectorizer.fit_transform(df.pop('text'))

# adding "features" columns as SparseSeries
#for i, col in enumerate(vectorizer.get_feature_names()):
#    df[col] = pd.SparseSeries(tf[:, i].toarray().ravel(), fill_value=0)

# tf_feature_names tells us what word each column in the matric represents
tf_feature_names = vectorizer.get_feature_names()
#tf_feature_names<jupyter_output><empty_output><jupyter_text>Loop through number_of_topics to find the best numberThe quality of the topic modelling hinges on the ideal number of topics. Thanks to [this website (although the code needs an update)](https://www.machinelearningplus.com/nlp/topic-modeling-python-sklearn-examples/9buildldamodelwithsklearn).<jupyter_code># Search for optimal topics
search_params = {'n_components': [5, 50, 100]} #, 'learning_decay': [.5, .7, .9]}
search = LatentDirichletAllocation()

search_model = GridSearchCV(search, param_grid=search_params)
search_model.fit(tf)

# Best Model
best_model = search_model.best_estimator_

# Model Parameters
print("Best Model's Params: ", search_model.best_params_)

# Log Likelihood Score
print("Best Log Likelihood Score: ", search_model.best_score_)

# Perplexity
print("Model Perplexity of the best topic: ", best_model.perplexity(tf))
%%time

n_topics = [5, 50, 100]

#log_likelyhoods_5 = [round(search_model.cv_results_['mean_test_score'][index]) for index, gscore in enumerate(search_model.cv_results_['params']) if gscore['learning_decay']==0.5]
log_likelyhoods_7 = [round(search_model.cv_results_['mean_test_score'][index]) for index, gscore in enumerate(search_model.cv_results_['params'])]# if gscore['learning_decay']==0.7]
#log_likelyhoods_9 = [round(search_model.cv_results_['mean_test_score'][index]) for index, gscore in enumerate(search_model.cv_results_['params']) if gscore['learning_decay']==0.9]

# Show graph
plt.figure(figsize=(12, 8))
#plt.plot(n_topics, log_likelyhoods_5, label='0.5')
plt.plot(n_topics, log_likelyhoods_7, label='0.7')
#plt.plot(n_topics, log_likelyhoods_9, label='0.9')
plt.title("Choosing The Optimal LDA Model")
plt.xlabel("Num Topics")

plt.gca().spines['top'].set_visible(False)
plt.gca().spines['right'].set_visible(False)

plt.ylabel("Log Likelyhood Scores")
#plt.legend(title='Learning decay', loc='best')
plt.savefig('/media/s/Linux_storage/Analyse_Verkehrswende_Transformation/plots/Greenwashing/lda_model_wordtree_test.png', dpi=300)
plt.show()<jupyter_output><empty_output><jupyter_text>Set topics and start the model<jupyter_code>%%time
# Set topics
number_of_topics = 3

# Here we have two different models: LDA vs NFM. NFM might work better with Tweet data.
model = LatentDirichletAllocation(n_components=number_of_topics, random_state=0)
#model = NMF(n_components=None, random_state=0, alpha=.1, l1_ratio=.5)
model.fit(tf)<jupyter_output>/home/s/.local/lib/python3.8/site-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.
  and should_run_async(code)<jupyter_text>Print results here and export a file with top resultsOnly run the following cell to get an export file<jupyter_code># model.fit(tf)

def display_topics(model, feature_names, no_top_words):
    topic_dict = {}
    for topic_idx, topic in enumerate(model.components_):
        topic_dict["Topic %d words" % (topic_idx)]= ['{}'.format(feature_names[i])
                        for i in topic.argsort()[:-no_top_words - 1:-1]]
        topic_dict["Topic %d weights" % (topic_idx)]= ['{:.1f}'.format(topic[i])
                        for i in topic.argsort()[:-no_top_words - 1:-1]]
        df = pd.DataFrame(topic_dict)
        df.to_csv('/media/s/Linux_storage/Analyse_Verkehrswende_Transformation/Data/Greenwashing' + '/' + 'topics_3.csv')
    return df


# Select top word list based on interactive exploration
no_top_words = 50
display_topics(model, tf_feature_names, no_top_words)<jupyter_output>/home/s/.local/lib/python3.8/site-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.
  and should_run_async(code)<jupyter_text>Interactive Topic ModellingThanks to [Himanshu](https://towardsdatascience.com/topic-model-visualization-using-pyldavis-fecd7c18fbf6)<jupyter_code>from __future__ import print_function
import pyLDAvis
import pyLDAvis.sklearn
from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.decomposition import LatentDirichletAllocation
pyLDAvis.enable_notebook()

#tf = np.matrix(tf) # Enable if .toarray a few cells above is turned on
pyLDAvis.sklearn.prepare(model, tf, vectorizer)
# Export the result to an html
pyLDAvis.save_html(pyLDAvis.sklearn.prepare(model, tf, vectorizer), 'lda_3.html') # The fill be saved in the active path: ../code<jupyter_output>/home/s/.local/lib/python3.8/site-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.
  and should_run_async(code)<jupyter_text>Perhaps use a second cell to compare different languages and their results.<jupyter_code>from __future__ import print_function
import pyLDAvis
import pyLDAvis.sklearn
from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.decomposition import LatentDirichletAllocation
pyLDAvis.enable_notebook()

#tf = np.matrix(tf) # Enable if .toarray a few cells above is turned on
pyLDAvis.sklearn.prepare(model, tf, vectorizer)<jupyter_output>/home/s/.local/lib/python3.8/site-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.
  and should_run_async(code)
{'fraction_non_alphanumeric': 0.11732390126429862, 'fraction_numerical': 0.010836845273931367, 'mean_word_length': 4.999548532731377, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 44, 'https://': 5, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '404211', 'license': 'MIT', 'n_tokens_mistral': 4530, 'n_tokens_neox': 4327, 'n_words': 1124, 'path': 'code/Greenwashing_topics_clean.ipynb', 'repo_name': 'LaserSteff/inauthentic_protest', 'size': '13288'}