<jupyter_start><jupyter_text>Data Cleaning<jupyter_code>import re      # considering the text as regular expression
import string  # as the text data is nothing but string only
def clean_text(text):
    '''Make text lowercase, remove text in square brackets, remove punctuation and remove words containing numbers.'''
    text = text.lower()
    text = re.sub('\[.*?\]', '', text)
    text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
    text = re.sub('\w*\d\w*', '', text)
    text = re.sub("[0-9" "]+"," ",text)
    text = re.sub('[‘’“”…]', '', text)
    return text

clean = lambda x: clean_text(x)
# as our data is in text format so we cannot use and apply any user defined function directly so in order to use the function we will take help of lambda function

clean = lambda x : clean_text(x)    # x means each and every term in our text, to each and every term in our text we will apply the userdefined function
data['text'] = data['text'].apply(clean)
data.text
# Finding the word frequency 

freq = pd.Series(''.join(data['text']).split()).value_counts()[0:20]   # Joined the text data into a paragarph seperated by a space and then splitted the paragraphs into words
freq<jupyter_output><empty_output><jupyter_text>Inference: As maximum words are stopwords and there frequency is also high so now we will remove the stopwords<jupyter_code>from nltk.corpus import stopwords
stop = stopwords.words('english')
data['text'] = data['text'].apply(lambda x: " ".join(x for x in x.split() if x not in stop))
#word frequency after removal of stopwords
freq_Sw = pd.Series(' '.join(data['text']).split()).value_counts()[:20] # for top 20
freq_Sw
from sklearn.feature_extraction.text import CountVectorizer
import numpy as np
vectorizer = CountVectorizer(min_df = 1, max_df = 0.9)
X = vectorizer.fit_transform(data['text'])
word_freq_df = pd.DataFrame({'Term': vectorizer.get_feature_names(),'occurences': np.asarray(X.sum(axis = 0)).ravel().tolist()})
word_freq_df['frequency'] = word_freq_df['occurences']/np.sum(word_freq_df['occurences']) 
word_freq_df.head(30)
from sklearn.feature_extraction.text import TfidfVectorizer
vectorizer = TfidfVectorizer(stop_words='english', max_features= 1000, max_df = 0.5, smooth_idf=True) #keep top 1000 words
doc_vec = vectorizer.fit_transform(data["text"])
names_features = vectorizer.get_feature_names()
dense = doc_vec.todense()                                 #todense() will calculate the density of the term
denselist = dense.tolist()
df = pd.DataFrame(denselist, columns = names_features)
df<jupyter_output><empty_output><jupyter_text>* occurence : number of instances * frequency : Rate of occurences* Density   : Total number of events  N-Gram<jupyter_code>#Bi-gram
def get_top_words(corpus,n = None):
    vec1 = CountVectorizer(ngram_range = (2,2),max_features = 2000).fit(corpus)
    bag_of_words = vec1.transform(corpus)
    sum_of_words = bag_of_words.sum(axis = 0)
    words_freq = [(word, sum_of_words[0, idx]) for word, idx in vec1.vocabulary_.items()]
    words_freq  = sorted(words_freq, key = lambda x : x[1], reverse = True)
    return words_freq[:n]
top2_words = get_top_words(data["text"], n=200) #top 200
top2_df = pd.DataFrame(top2_words)
top2_df.columns=["Bi-gram", "Freq"]
top2_df.head()
#Bi-gram plot
import matplotlib.pyplot as plt
import seaborn as sns
top20_bigram = top2_df.iloc[0:20,:]
fig = plt.figure(figsize = (10, 5))
plot=sns.barplot(x=top20_bigram["Bi-gram"],y=top20_bigram["Freq"])
plot.set_xticklabels(rotation=45,labels = top20_bigram["Bi-gram"])
def get_top_n3_words(corpus, n=None):
    vec1 = CountVectorizer(ngram_range=(3,3), 
           max_features=2000).fit(corpus)
    bag_of_words = vec1.transform(corpus)
    sum_words = bag_of_words.sum(axis=0) 
    words_freq = [(word, sum_words[0, idx]) for word, idx in     
                  vec1.vocabulary_.items()]
    words_freq =sorted(words_freq, key = lambda x: x[1], 
                reverse=True)
    return words_freq[:n]

top3_words = get_top_n3_words(data["text"], n=200)
top3_df = pd.DataFrame(top3_words)
top3_df.columns=["Tri-gram", "Freq"]
top3_df
#Tri-gram plot
import seaborn as sns
top20_trigram = top3_df.iloc[0:20,:]
fig = plt.figure(figsize = (10, 5))
plot=sns.barplot(x=top20_trigram["Tri-gram"],y=top20_trigram["Freq"])
plot.set_xticklabels(rotation=45,labels = top20_trigram["Tri-gram"])
string_Total = " ".join(data["text"])
#wordcloud for entire corpus
from wordcloud import WordCloud
plt.figure(figsize = (15,8))
wordcloud_stw = WordCloud(
                background_color= 'black',
                width = 1800,
                height = 1500
                ).generate(string_Total)
plt.imshow(wordcloud_stw)
data.head()
# Tokenization
def split_into_words(i):
    return(i.split(" "))
from sklearn.model_selection import train_test_split
email_train,email_test = train_test_split(data, test_size = 0.3)
email_test
# converting the text data into structured format

email_bow = CountVectorizer(analyzer = split_into_words).fit(data.text)
all_emails_matrix = email_bow.transform(data.text)
all_emails_matrix.shape
# for training data
train_emails_matrix = email_bow.transform(email_train.text)
train_emails_matrix.shape
# for test data
test_emails_matrix = email_bow.transform(email_test.text)
test_emails_matrix.shape
# Finally building the model

from sklearn.naive_bayes import MultinomialNB as MB      # can be applied on text data for classification purpose
from sklearn.naive_bayes import GaussianNB as GB         # can be applied on numeric data

# Multinomial NB

classifier_MB = MB()
classifier_MB.fit(train_emails_matrix, email_train.type)
train_pred = classifier_MB.predict(train_emails_matrix)
accuracy = np.mean(train_pred == email_train.type)
accuracy
test_pred = classifier_MB.predict(test_emails_matrix)
accuracy = np.mean(test_pred == email_test.type)
accuracy
# Gaussian Naive Bayes 

classifier_GB = GB()
classifier_GB.fit(train_emails_matrix.toarray(), email_train.type)
train_pred = classifier_GB.predict(train_emails_matrix.toarray())
accuracy = np.mean(train_pred == email_train.type)
accuracy
test_pred = classifier_GB.predict(test_emails_matrix.toarray())
accuracy = np.mean(test_pred == email_test.type)
accuracy
# Using the Tfidf vectorizer

from sklearn.feature_extraction.text import TfidfTransformer

# Learning Term weighting and normalizing on entire emails
tfidf_transformer = TfidfTransformer().fit(all_emails_matrix)

# Preparing TFIDF for train emails
train_tfidf = tfidf_transformer.transform(train_emails_matrix)

train_tfidf.shape # (3891, 6661)

# Preparing TFIDF for test emails
test_tfidf = tfidf_transformer.transform(test_emails_matrix)

test_tfidf.shape #  (1668, 6661)

train_tfidf.shape # (3891, 6661)
# Preparing a naive bayes model on training data set 

from sklearn.naive_bayes import MultinomialNB as MB
from sklearn.naive_bayes import GaussianNB as GB

# Multinomial Naive Bayes
classifier_mb = MB()
classifier_mb.fit(train_tfidf,email_train.type)
train_pred_m = classifier_mb.predict(train_tfidf)
accuracy_train_m = np.mean(train_pred_m==email_train.type) # 96%

test_pred_m = classifier_mb.predict(test_tfidf)
accuracy_test_m = np.mean(test_pred_m==email_test.type) # 96%

 
accuracy_train_m
accuracy_test_m
# Gaussian Naive Bayes 
classifier_gb = GB()
classifier_gb.fit(train_tfidf.toarray(),email_train.type.values) # we need to convert tfidf into array format which is compatible for gaussian naive bayes
train_pred_g = classifier_gb.predict(train_tfidf.toarray())
accuracy_train_g = np.mean(train_pred_g==email_train.type) # 95%
test_pred_g = classifier_gb.predict(test_tfidf.toarray())
accuracy_test_g = np.mean(test_pred_g==email_test.type) # 88%

# inplace of tfidf we can also use train_emails_matrix and test_emails_matrix instead of term inverse document frequency matrix
accuracy_train_g
accuracy_test_g<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11102642276422764, 'fraction_numerical': 0.018927845528455285, 'mean_word_length': 5.4112377850162865, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '484109', 'license': 'MIT', 'n_tokens_mistral': 2844, 'n_tokens_neox': 2703, 'n_words': 641, 'path': 'text_classification_feature_extraction Naive Bayes.ipynb', 'repo_name': 'anagha0397/Text-Mining', 'size': '7872'}