<jupyter_start><jupyter_text>Data Cleaning<jupyter_code># import the libraries
import pandas as pd
from collections import Counter
import seaborn as sns
import datetime as dt
import re
import nltk
from nltk.corpus import stopwords
from bs4 import BeautifulSoup
from wordcloud import WordCloud
import cufflinks as cf
cf.go_offline()
import plotly.graph_objects as go
import plotly.express as px
import matplotlib.pyplot as plt
%matplotlib inline
from nltk.stem import PorterStemmer
stemming = PorterStemmer()
# We need to download all the necessary packages
# nltk.download('all') for all the packages
nltk.download('stopwords')
# Load the scrapped reddit csv file
data_df = pd.read_csv('./india.csv')
data_df
# Since reddit has date in UNIX format
# We will convert it and add it to our dataframe
def getDate(created):
    return dt.datetime.fromtimestamp(created)

data_df['timestamp'] = data_df['created'].apply(getDate)
# Now we will check the timestamp 
data_df<jupyter_output><empty_output><jupyter_text>Drop unwanted Items<jupyter_code>data_df.dtypes
# We don't need "date" column as we already added timestamp
# We will use drop function of pandas
# We are droping column so set axis=1
data_df = data_df.drop(['created'], axis=1)
data_df.dtypes
# Check how many null values in our dataframe
print(data_df.isnull().sum())
data_df.count()
# We can see above we have many null values
# We will drop them using panda's dropna()
# data_df = data_df.dropna()
data_df = data_df.fillna("na")
# Now let's check whether there are any remaining null values
print(data_df.isnull().sum())
print(data_df.count())
# Let's check our df
data_df.head()
# As we can see comments section contains list
# Many of them are empty we will drop it
data_df = data_df.drop(['comments'], axis=1)
data_df
# Check how many null values in our dataframe
print(data_df.isnull().sum())
data_df.count()<jupyter_output>title        0
url          0
comms_num    0
flair        0
score        0
id           0
author       0
body         0
timestamp    0
dtype: int64<jupyter_text>Let's do some Text Preprocessing<jupyter_code># Let's clean some texts
# First we will replace some special characters
# We will use python regex
# And nltk STOPWORDS for removing commonly used word (such as “the”, “a”, “an”, “in”)
# We also lower the text
# We will use PorterStemming: The Porter stemming algorithm (or ‘Porter stemmer’) is a 
# process for removing the commoner morphological and inflexional endings from words in English.
REPLACE_BY_SPACE = re.compile('[/(){}\[\]\|@,;]')
REPLACE_SYMBOL = re.compile('[^0-9a-z #+_]')
STOPWORDS = set(stopwords.words('english'))
# Let's define the function for cleaning
def clean_data(text):
    text = BeautifulSoup(text, "lxml").text
    text = text.lower()
    text = REPLACE_BY_SPACE.sub(' ', text)
    text = REPLACE_SYMBOL.sub('', text)
    text = ' '.join(word for word in text.split() if word not in STOPWORDS)
    return text

# Define string function
def stringConvert(text):
    return str(text)
# -------------------------------------------
# Let's apply the clean_data() to our features
# I am considering following features
# This will help us later on to test our model
# --> Title
# --> Body
# --> Comments
# -----------------------------------

# Title
data_df['title'] = data_df['title'].apply(stringConvert)
data_df['title'] = data_df['title'].apply(clean_data)

# Body
data_df['body'] = data_df['body'].apply(stringConvert)
data_df['body'] = data_df['body'].apply(clean_data)
# Let's combine some features and add it to our dataframe
# This will help us getting better insights into our model
# We will use assign() method to assign 
# We will need to use join() method to insert multiple df

# Title + Body + URL [tbu]
feature_tbu = data_df['title'] + data_df['body'] + data_df['url']
data_df = data_df.assign(feature_tbu=feature_tbu)
data_df.dtypes
# Let's view our dataframe
data_df.head()
# We have done our cleaning of data
# Now let's convert it to csv for later use
data_df.to_csv('processedIndiafinal.csv', index=False)<jupyter_output><empty_output><jupyter_text>Let's Plot some Graphs and Visualize<jupyter_code># Let's start with bar chart plotting each flair
# We will use matplotlib for this

data_df['flair'].value_counts().plot(kind='bar')
plt.xlabel('Flairs')
plt.ylabel('No of Posts')
plt.title('Flairs Distribution')
# Above Graph doen't appeal much let's use plotly
# Love Plotly
# We can interact with the graph in the jupyter itself
data_df.groupby('flair').count()['id'].iplot(kind='bar', yTitle='No of Posts', 
                                                    xTitle='Flairs',linecolor='black', title='Flairs Distribution')
# Let's check our flair distribution by percentage value
# I will use Plotly pie chart and we can hover over it to get the value
# We are using plotly.graph_objects which is imported as go
fig = go.Figure(data=[go.Pie(labels=data_df['flair'].unique(),
                             values=data_df.groupby('flair').count()['id'])])
fig.update_traces(hoverinfo='label+percent+value', textinfo='value', textfont_size=20,
                  marker=dict(line=dict(color='#000000', width=2)))
fig.show()
# Let's do some more charts
# Don't worry Plotly got it covered :)
# We will plot distribution of Flairs based on "No of Comments"
fig = go.Figure(data=[go.Pie(labels=data_df['flair'].unique(),
                             values=data_df.groupby('comms_num').count()['id'])])
fig.update_traces(hoverinfo='label+value', textinfo='value', textfont_size=20,
                  marker=dict(line=dict(width=0)))
fig.show()
# Let's find out most common words in "body" column
# Most common top 10 words
commonBody = Counter(" ".join(data_df["body"]).split()).most_common(10)
print(commonBody)

# Now let's visualize it
cloud = WordCloud(max_words=10, background_color="white").generate(str(commonBody))

# Display the generated image:
plt.imshow(cloud, interpolation='bilinear')
plt.axis("off")
plt.show()
# Let's find out most common words in "title" column
# Most common top 10 words
commTitle = Counter(" ".join(data_df["title"]).split()).most_common(10)
print(commTitle)

# Now let's visualize it
cloud = WordCloud(max_words=10, background_color="white").generate(str(commTitle))

# Display the generated image:
plt.imshow(cloud, interpolation='bilinear')
plt.axis("off")
plt.show()<jupyter_output>[('india', 1736), ('covid19', 1357), ('lockdown', 1025), ('coronavirus', 812), ('indian', 570), ('news', 545), ('says', 396), ('indias', 379), ('times', 367), ('delhi', 335)]
{'fraction_non_alphanumeric': 0.13339452348171946, 'fraction_numerical': 0.011626128193360868, 'mean_word_length': 4.448333333333333, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '484370', 'license': 'MIT', 'n_tokens_mistral': 2144, 'n_tokens_neox': 2035, 'n_words': 669, 'path': 'notebooks/Part 2-Data EDA.ipynb', 'repo_name': 'imskr/reFlair', 'size': '6537'}