<jupyter_start><jupyter_text>Mining Melon's Twitter Feed  Set-up and loading data<jupyter_code>import pandas as pd
import csv
import os
import numpy as np
# step up directory to find data
#owd=os.getcwd()
os.chdir("..") # to move up directory
os.chdir("..") # to move up directory
path = os.getcwd()+"/data/boogie2988.csv"
data = pd.read_csv(path)
data.head()
print(data.dtypes)<jupyter_output>id                 float64
conversation_id    float64
created_at         float64
date                object
time                object
timezone            object
user_id              int64
username            object
name                object
place              float64
tweet               object
mentions            object
urls                object
photos              object
replies_count        int64
retweets_count       int64
likes_count          int64
hashtags            object
cashtags            object
link                object
retweet               bool
quote_url           object
video                int64
near               float64
geo                float64
source             float64
user_rt_id         float64
user_rt            float64
retweet_id         float64
reply_to            object
retweet_date       float64
dtype: object<jupyter_text>Natural Language Processing<jupyter_code>data['date'] = pd.to_datetime(data['date'], format='%Y-%m-%d')
data['retweet_date'] = pd.to_datetime(data['retweet_date'], format='%Y%m%d')
import nltk
nltk.download('punkt')
nltk.download('stopwords')
# initial set up and word tokenisation
frame = data['tweet'].str.lower().str.cat(sep=' ')
words = nltk.tokenize.word_tokenize(frame)
word_dist = nltk.FreqDist(words)
print(word_dist)
# text cleaning using nltk pre-made english stopword list
# convert this to a list before computation for speed
stopwords = set(nltk.corpus.stopwords.words('english'))
dict_filter = lambda word_dist, stopwords: \
    dict((word,word_dist[word]) 
    for word in word_dist if word not in stopwords 
         and word.isalpha()
         and word not in ['http', 'https'])
    
filtered_word_freq = dict_filter(word_dist, stopwords)
# convert to pandas dataframe
rslt = pd.DataFrame(list(filtered_word_freq.items()),
                    columns=['word', 'freq'])
rslt.head()
sorted_rslt = rslt.sort_values(by='freq', ascending=False)
sorted_rslt = sorted_rslt[:20]<jupyter_output><empty_output><jupyter_text>Visualising the data<jupyter_code>import matplotlib.pyplot as plt
import seaborn as sns; sns.set()
%matplotlib inline<jupyter_output><empty_output><jupyter_text><jupyter_code># most commonly used words
sorted_rslt.reset_index(level=0, inplace=True)
plt.rcParams["figure.figsize"] = (15, 12)

bar = sns.barplot(x = sorted_rslt['word'], 
                  y = sorted_rslt['freq'], 
                  data = sorted_rslt, 
                  palette = 'Blues_d')

plt.show()
# save to file
fig = bar.get_figure()
fig.savefig("img/output.png")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09709066305818674, 'fraction_numerical': 0.014546684709066306, 'mean_word_length': 2.8303108808290154, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '65078', 'license': 'MIT', 'n_tokens_mistral': 991, 'n_tokens_neox': 928, 'n_words': 224, 'path': 'code/boogie/BoogieTweets.ipynb', 'repo_name': 'KeiranJOC/code-collection', 'size': '2956'}