<jupyter_start><jupyter_text><jupyter_code>from google.colab import files
#files.upload()

import re
import pandas as pd 
import numpy as np 
import matplotlib.pyplot as plt 
import seaborn as sns
import string
import nltk
import warnings 
from nltk.tokenize import TreebankWordTokenizer
from sklearn.model_selection import train_test_split

warnings.filterwarnings("ignore", category=DeprecationWarning)

%matplotlib inline

#train  = pd.read_csv('train_E6oV3lV.csv')
#test = pd.read_csv('test_tweets_anuFYb8.csv')

train = pd.read_csv('https://github.com/jainnipun/MachineLearning/blob/master/TextAnalytics/TwitterSentimentAnalysis/train_E6oV3lV.csv?raw=true')
test = pd.read_csv('https://raw.githubusercontent.com/jainnipun/MachineLearning/master/TextAnalytics/TwitterSentimentAnalysis/test_tweets_anuFYb8.csv')<jupyter_output><empty_output><jupyter_text>Let’s check the first few rows of the train dataset.<jupyter_code>print("train",len(train))
print("test",len(test))
train.head()<jupyter_output>train 31962
test 17197<jupyter_text>First combining train and test set. This saves the trouble of performing the same steps twice on test and train.The data has 3 columns id, label, and tweet. label is the binary target variable and tweet contains the tweets that we will clean and preprocess.<jupyter_code>combi = train.append(test, ignore_index=True)<jupyter_output><empty_output><jupyter_text>finall - find all matching wordssub - replace matching words with blank<jupyter_code>def remove_pattern(input_txt, pattern):
    r = re.findall(pattern, input_txt)
    for i in r:
        input_txt = re.sub(i, '', input_txt)
        
    return input_txt<jupyter_output><empty_output><jupyter_text>The vectorize function is provided primarily for convenience, not for performance. The implementation is essentially a for loop.If otypes is not specified, then a call to the function with the first argument will be used to determine the number of outputs. The results of this call will be cached if cache is True to prevent calling the function twice. However, to implement the cache, the original function must be wrapped which will slow down subsequent calls, so only do this if your function is expensive.The new keyword argument interface and excluded argument support further degrades performance.<jupyter_code># remove twitter handles (@user)
combi['tidy_tweet'] = np.vectorize(remove_pattern)(combi['tweet'], "@[\w]*")

combi.head()
# remove special characters, numbers, punctuations
combi['tidy_tweet'] = combi['tidy_tweet'].str.replace("[^a-zA-Z#]", " ")

combi['tidy_tweet'] = combi['tidy_tweet'].apply(lambda x: ' '.join([w for w in x.split() if len(w)>3]))

combi['tidy_tweet'].head()<jupyter_output><empty_output><jupyter_text>Tokenization<jupyter_code>tokenized_tweet = combi['tidy_tweet'].apply(lambda x: x.split())

#tokenized_tweet1 = combi['tidy_tweet'].apply(lambda x: TreebankWordTokenizer().tokenize(x))

tokenized_tweet.head()
from nltk.stem.porter import *
stemmer = PorterStemmer()

tokenized_tweet = tokenized_tweet.apply(lambda x: [stemmer.stem(i) for i in x]) # stemming
tokenized_tweet.head()

for i in range(len(tokenized_tweet)):
    tokenized_tweet[i] = ' '.join(tokenized_tweet[i])

combi['tidy_tweet'] = tokenized_tweet

from wordcloud import WordCloud

def printWordCloud(inputwords):
  wordcloud = WordCloud(width=800, height=500, random_state=21, max_font_size=110).generate(inputwords)
  plt.figure(figsize=(10, 7))
  plt.imshow(wordcloud, interpolation="bilinear")
  plt.axis('off')
  plt.show()

all_words = ' '.join([text for text in combi['tidy_tweet']])
printWordCloud(all_words)
normal_words = ' '.join([text for text in combi['tidy_tweet'][combi['label']==0]])
printWordCloud(normal_words)
negative_words = ' '.join([text for text in combi['tidy_tweet'][combi['label'] == 1]])
printWordCloud(negative_words)
# function to collect hashtags
def hashtag_extract(x):
    hashtags = []
    # Loop over the words in the tweet
    for i in x:
        ht = re.findall(r"#(\w+)", i)
        hashtags.append(ht)

    return hashtags
# extracting hashtags from non racist/sexist tweets

HT_regular = hashtag_extract(combi['tidy_tweet'][combi['label'] == 0])

# extracting hashtags from racist/sexist tweets
HT_negative = hashtag_extract(combi['tidy_tweet'][combi['label'] == 1])

# unnesting list
HT_regular = sum(HT_regular,[]) #make list 1-D
HT_negative = sum(HT_negative,[]) #make list 1-D
a = nltk.FreqDist(HT_regular) #FreqDist class is used to encode “frequency distributions”, which count the number of times that each outcome of an experiment occurs
d = pd.DataFrame({'Hashtag': list(a.keys()),
                  'Count': list(a.values())})
# selecting top 10 most frequent hashtags     
d = d.nlargest(columns="Count", n = 10) 
plt.figure(figsize=(16,5))
ax = sns.barplot(data=d, x= "Hashtag", y = "Count")
ax.set(ylabel = 'Count')
plt.show()
b = nltk.FreqDist(HT_negative)
e = pd.DataFrame({'Hashtag': list(b.keys()), 'Count': list(b.values())})
# selecting top 10 most frequent hashtags
e = e.nlargest(columns="Count", n = 10)   
plt.figure(figsize=(16,5))
ax = sns.barplot(data=e, x= "Hashtag", y = "Count")
ax.set(ylabel = 'Count')
plt.show()
from sklearn.feature_extraction.text import CountVectorizer
# Convert a collection of text documents to a matrix of token counts
#This implementation produces a sparse representation of the counts using scipy.sparse.csr_matrix.
# max_df When building the vocabulary ignore terms that have a document frequency strictly higher than the given threshold 
# max_features: top max_features ordered by term frequency across the corpus.

bow_vectorizer = CountVectorizer(max_df=0.90, min_df=2, max_features=1000, stop_words='english')
# bag-of-words feature matrix
#fit_transform : Learn the vocabulary dictionary and return term-document matrix
bow = bow_vectorizer.fit_transform(combi['tidy_tweet'])

#Since we have only binary classification so best to use logistic regression with a threshhold

from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
# to measure performance of classifier

from sklearn.metrics import f1_score

train_bow = bow[:31962,:]
test_bow = bow[31962:,:]

# splitting data into training and validation set
xtrain_bow, xvalid_bow, ytrain, yvalid = train_test_split(train_bow, train['label'], random_state=42, test_size=0.3)

lreg = LogisticRegression()
lreg.fit(xtrain_bow, ytrain) # training the model

thr  = 0.2
f1_max_var=0
th_max=0.0

for t in np.arange(0.0, 1.0, 0.01):
  prediction = lreg.predict_proba(xvalid_bow) # predicting on the validation set
  # return bool
  prediction_int = prediction[:,1] >= t # if prediction is greater than or equal to 0.3 than 1 else 0
  #convert to int
  prediction_int = prediction_int.astype(np.int)

  from sklearn.metrics import accuracy_score
 # print("Accuracy",accuracy_score(yvalid, prediction_int))

  f1_score_nw=f1_score(yvalid, prediction_int) # calculating f1 score
  
  if (f1_score_nw > f1_max_var) :
    f1_max_var = f1_score_nw
    th_max = t

print(th_max)
test_pred = lreg.predict_proba(test_bow)
test_pred_int = test_pred[:,1] >= 0.3
test_pred_int = test_pred_int.astype(np.int)
test['label'] = test_pred_int
submission = test[['id','label']]
submission.to_csv('sub_lreg_bow.csv', index=False) # writing data to a CSV file

from sklearn.feature_extraction.text import TfidfVectorizer
tfidf_vectorizer = TfidfVectorizer(max_df=0.90, min_df=2, max_features=1000, stop_words='english')
# TF-IDF feature matrix
tfidf = tfidf_vectorizer.fit_transform(combi['tidy_tweet'])

train_tfidf = tfidf[:31962,:]
test_tfidf = tfidf[31962:,:]

xtrain_tfidf = train_tfidf[ytrain.index]
xvalid_tfidf = train_tfidf[yvalid.index]

lreg.fit(xtrain_tfidf, ytrain)

prediction = lreg.predict_proba(xvalid_tfidf)
prediction_int = prediction[:,1] >= 0.3
prediction_int = prediction_int.astype(np.int)

from sklearn.metrics import accuracy_score
print("Accuracy",accuracy_score(yvalid, prediction_int))

f1_score(yvalid, prediction_int)<jupyter_output>Accuracy 0.9462926269684013
{'fraction_non_alphanumeric': 0.11178584706902962, 'fraction_numerical': 0.017722146486553476, 'mean_word_length': 6.078947368421052, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 28, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '589255', 'license': 'Apache-2.0', 'n_tokens_mistral': 2855, 'n_tokens_neox': 2721, 'n_words': 701, 'path': 'TextAnalytics/TwitterSentimentAnalysis/Twitter Hate Speech sentiment Analysis.ipynb', 'repo_name': 'jainnipun/MachineLearning', 'size': '8069'}