<jupyter_start><jupyter_text>Import Python Packages<jupyter_code>import numpy as np
import scipy as sci
import pyDOE
import matplotlib.pyplot as plt
import itertools
import pandas as pd
import statsmodels.api as sm
from statsmodels.formula.api import ols
from statsmodels.tsa.holtwinters import ExponentialSmoothing
%matplotlib inline
import sklearn
import nltk
from nltk.tokenize import word_tokenize, sent_tokenize, TweetTokenizer
from nltk import pos_tag
from nltk.corpus import stopwords
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.model_selection import cross_val_score, train_test_split
from sklearn.svm import LinearSVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import MultinomialNB, GaussianNB,BernoulliNB
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
nltk.download('punkt')
import re, string
from wordcloud import WordCloud
import seaborn as sns
sns.set_style('white')
nltk.download('stopwords')
from nltk.stem import WordNetLemmatizer 
nltk.download('wordnet')
#nltk.download('all')
import warnings
warnings.simplefilter("ignore", DeprecationWarning)
from sklearn.decomposition import LatentDirichletAllocation as LDA
from pyLDAvis import sklearn as sklearn_lda
import pyLDAvis
from textblob import TextBlob
import time<jupyter_output>[nltk_data] Downloading package punkt to
[nltk_data]     C:\Users\dw998\AppData\Roaming\nltk_data...
[nltk_data]   Package punkt is already up-to-date!
[nltk_data] Downloading package stopwords to
[nltk_data]     C:\Users\dw998\AppData\Roaming\nltk_data...
[nltk_data]   Package stopwords is already up-to-date!
[nltk_data] Downloading package wordnet to
[nltk_data]     C:\Users\dw998\AppData\Roaming\nltk_data...
[nltk_data]   Package wordnet is already up-to-date!<jupyter_text>Read in Data<jupyter_code>joe_biden_df=pd.read_csv('hashtag_joebiden.csv')
biden_subset=joe_biden_df[['tweet','user_location']].dropna()
joe_biden_df.dtypes
biden_subset<jupyter_output>C:\ProgramData\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py:3020: DtypeWarning: Columns (1,2,3,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20) have mixed types. Specify dtype option on import or set low_memory=False.
  interactivity=interactivity, compiler=compiler, result=result)<jupyter_text>Clean Data<jupyter_code>x = biden_subset["tweet"].copy()
y = biden_subset["user_location"].copy()
def clean_text(review):
    '''Make text lowercase, remove text in square brackets, remove punctuation and remove words containing numbers.'''
    review = review.lower()
    review = re.sub('\[.*?\]', '', review)
    review = re.sub('[%s]' % re.escape(string.punctuation), '', review)
    review = re.sub('\w*\d\w*', '', review)
    review = re.sub('[‘’“”…]', '', review)
    review = re.sub('\n', '', review)
    stop_words=set(stopwords.words('english'))
    return review
round1 = lambda x: clean_text(x)
biden_subset['clean'] = x.apply(round1)<jupyter_output><>:6: DeprecationWarning: invalid escape sequence \[
<>:8: DeprecationWarning: invalid escape sequence \w
<>:6: DeprecationWarning: invalid escape sequence \[
<>:8: DeprecationWarning: invalid escape sequence \w
<>:6: DeprecationWarning: invalid escape sequence \[
<>:8: DeprecationWarning: invalid escape sequence \w
<ipython-input-4-1653627da5f2>:6: DeprecationWarning: invalid escape sequence \[
  review = re.sub('\[.*?\]', '', review)
<ipython-input-4-1653627da5f2>:8: DeprecationWarning: invalid escape sequence \w
  review = re.sub('\w*\d\w*', '', review)<jupyter_text>Word Count Per Tweet<jupyter_code>biden_subset['word_count'] = biden_subset['clean'].apply(lambda x: len(str(x).split(" ")))
biden_subset[['clean','word_count']]<jupyter_output><empty_output><jupyter_text>Word Count Descriptive Statistics<jupyter_code>biden_subset.word_count.describe()<jupyter_output><empty_output><jupyter_text>Most Frequent Words<jupyter_code>freq = pd.Series(' '.join(biden_subset['clean']).split()).value_counts()[:20]
freq<jupyter_output><empty_output><jupyter_text>Least Frequent Words<jupyter_code>freq1 =  pd.Series(' '.join(biden_subset['clean']).split()).value_counts()[-20:]
freq1<jupyter_output><empty_output><jupyter_text>Word Cloud Visualization<jupyter_code>from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
import matplotlib.pyplot as plt
%matplotlib inline
wordcloud = WordCloud(
                          background_color='white',
                          max_words=100,
                          max_font_size=50, 
                          random_state=42
                         ).generate(str(biden_subset['clean']))
print(wordcloud)
fig = plt.figure(1)
plt.imshow(wordcloud)
plt.axis('off')
plt.show()<jupyter_output><wordcloud.wordcloud.WordCloud object at 0x000001C9072C65F8><jupyter_text>Keyword Extraction<jupyter_code>from sklearn.feature_extraction.text import CountVectorizer
import re
cv=CountVectorizer(max_df=0.8, max_features=10000, ngram_range=(1,3))
X=cv.fit_transform(biden_subset['clean'])
from sklearn.feature_extraction.text import TfidfTransformer
 
tfidf_transformer=TfidfTransformer(smooth_idf=True,use_idf=True)
tfidf_transformer.fit(X)
# get feature names
feature_names=cv.get_feature_names()
 
# fetch document for which keywords needs to be extracted
doc=biden_subset['clean'][0]
 
#generate tf-idf for the given document
tf_idf_vector=tfidf_transformer.transform(cv.transform([doc]))
#Function for sorting tf_idf in descending order
from scipy.sparse import coo_matrix
def sort_coo(coo_matrix):
    tuples = zip(coo_matrix.col, coo_matrix.data)
    return sorted(tuples, key=lambda x: (x[1], x[0]), reverse=True)
 
def extract_topn_from_vector(feature_names, sorted_items, topn=10):
    """get the feature names and tf-idf score of top n items"""
    
    #use only topn items from vector
    sorted_items = sorted_items[:topn]
 
    score_vals = []
    feature_vals = []
    
    # word index and corresponding tf-idf score
    for idx, score in sorted_items:
        
        #keep track of feature name and its corresponding score
        score_vals.append(round(score, 3))
        feature_vals.append(feature_names[idx])
 
    #create a tuples of feature,score
    #results = zip(feature_vals,score_vals)
    results= {}
    for idx in range(len(feature_vals)):
        results[feature_vals[idx]]=score_vals[idx]
    
    return results
#sort the tf-idf vectors by descending order of scores
sorted_items=sort_coo(tf_idf_vector.tocoo())
#extract only the top n; n here is 10
keywords=extract_topn_from_vector(feature_names,sorted_items,5)
 
# now print the results
print("\nAbstract:")
print(doc)
print("\nKeywords:")
for k in keywords:
    print(k,keywords[k])<jupyter_output>Abstract:
  en florida joebiden dice que donaldtrump solo se preocupa por él mismo el demócrata fue anfitrión de encuentros de electores en pembrokepines y miramar clic aquí ⬇️⬇️⬇️⠀🌐httpstcoqhiwpiuxstelsollatino yobrilloconelsol 

Keywords:
dice que 0.279
electores 0.275
en florida 0.275
él 0.272
el demócrata 0.271
{'fraction_non_alphanumeric': 0.1087378640776699, 'fraction_numerical': 0.019417475728155338, 'mean_word_length': 5.7329598506069095, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '450846', 'license': 'CC0-1.0', 'n_tokens_mistral': 2570, 'n_tokens_neox': 2461, 'n_words': 542, 'path': '2020 Election Tweet Keyword Extraction Project/2020 Election Tweet Keyword Extraction.ipynb', 'repo_name': 'dawei-wang/2020-Election-Tweet-Keyword-Extraction-Project', 'size': '7210'}