<jupyter_start><jupyter_text>Assignment P2: Response Type Classification in DiscussionsThis notebook illustrates the Assignment P2 of CSC 791 Natural Language Processing Fall 2020. In this assignment, you will learn feature engineering through a classification task. Background Interactions through question-answering play an important role in discussions. Through questioning, askers may want to elicit information (e.g., wh-questions), clarify situations (e.g., closed-ended questions), or even make a point (e.g., rhetorical questions). However, how a question is responded does not necessarily align with the intent of the asker.<jupyter_code>#Download Spacy 
!python -m spacy download en_core_web_md
!python3 -m spacy download en_core_web_sm
!python -m spacy.en.download all
# Upload Testing and Training datasets
# Please select both the files: p2_train and p2_test

from google.colab import files
uploaded = files.upload()
import pandas as pd
import numpy as np
import io
# Load Training and Testing data
df_train = pd.read_csv(io.BytesIO(uploaded['p2_train.csv']))
df_test  = pd.read_csv(io.BytesIO(uploaded['p2_test.csv']))
print(df_train.shape)
print(df_test.shape)
df_test.head()
# Analyzing the dataset
df_train.head()
classification = {'agreed':0, 'answered':1, 'attacked':2, 'irrelevant':3}

df_train['class'] = df_train.apply(lambda i : classification[i['type']], axis=1)
df_test['class'] = df_test.apply(lambda i : classification[i['type']], axis=1)

df_train['class'] = df_train['type']
df_train.groupby(by = ["type"]).count()["question"]
# Cleaning Text

def init():

  import html 
  esc = ['&amp;', '&lt;', '&quot;', '&#x27;', '&gt;', '<>']

  for index, row in df_train.iterrows():
    for e in esc:
      if(type(row['precedent']) == str):
        row['precedent']  = row['precedent'].replace(e, "")
      if(type(row['question']) == str):
        row['question']   = row['question'].replace(e, "")
      if(type(row['subsequent']) == str):
        row['subsequent'] = row['subsequent'].replace(e, "")
      if(type(row['response']) == str):
        row['response']   = row['response'].replace(e, "")

  for index, row in df_test.iterrows():
    for e in esc:
      if(type(row['precedent']) == str):
        row['precedent']  = row['precedent'].replace(e, "")
      if(type(row['question']) == str):
        row['question']   = row['question'].replace(e, "")
      if(type(row['subsequent']) == str):
        row['subsequent'] = row['subsequent'].replace(e, "")
      if(type(row['response']) == str):
        row['response']   = row['response'].replace(e, "")

  df_train["clean_quest"] = df_train["precedent"] + df_train["question"] + df_train["subsequent"] + df_train['response']
  df_test["clean_quest"]  = df_test["precedent"] + df_test["question"] + df_test["subsequent"] + df_train['response']

  X_train, Y_train = df_train['clean_quest'], df_train['type']
  X_test,  Y_test  = df_test['clean_quest'], df_test['type']

  X_train = list(X_train)
  X_test  = list(X_test)
  Y_train = list(Y_train)
  Y_test  = list(Y_test)
# Cleaning Text


import html 
esc = ['&amp;', '&lt;', '&quot;', '&#x27;', '&gt;', '<>']

for index, row in df_train.iterrows():
  for e in esc:
    if(type(row['precedent']) == str):
      row['precedent']  = row['precedent'].replace(e, "")
    if(type(row['question']) == str):
      row['question']   = row['question'].replace(e, "")
    if(type(row['subsequent']) == str):
      row['subsequent'] = row['subsequent'].replace(e, "")
    if(type(row['response']) == str):
      row['response']   = row['response'].replace(e, "")

for index, row in df_test.iterrows():
  for e in esc:
    if(type(row['precedent']) == str):
      row['precedent']  = row['precedent'].replace(e, "")
    if(type(row['question']) == str):
      row['question']   = row['question'].replace(e, "")
    if(type(row['subsequent']) == str):
      row['subsequent'] = row['subsequent'].replace(e, "")
    if(type(row['response']) == str):
      row['response']   = row['response'].replace(e, "")

df_train["clean_quest"] = df_train["precedent"] + df_train["question"] + df_train["subsequent"] + df_train['response']
df_test["clean_quest"]  = df_test["precedent"] + df_test["question"] + df_test["subsequent"] + df_train['response']

X_train, Y_train = df_train['clean_quest'], df_train['type']
X_test,  Y_test  = df_test['clean_quest'], df_test['type']

X_train = list(X_train)
X_test  = list(X_test)
Y_train = list(Y_train)
Y_test  = list(Y_test)<jupyter_output><empty_output><jupyter_text>Feature Extraction<jupyter_code>init()
# Sentence Embedding

# Load Spacy model
import spacy.cli

spacy.cli.download("en_core_web_lg")
nlp = spacy.load('en_core_web_lg')
# For Spacy word2vec
def createVector(sent):
  vect = nlp(sent)
  avg = np.zeros(300)
  # print(vect)
  
  # Calculate average vector
  for token in vect: 
    avg += token.vector
  vect_avg = avg/len(vect)

  return vect_avg.tolist()
# Apply Transformation
df_train['vector'] = df_train.apply(lambda i : createVector(str(i['clean_quest'])), axis=1)
df_test['vector']  = df_test.apply(lambda i  : createVector(str(i['clean_quest'])), axis=1)
# Trasforming Vector info

X_train = np.array(df_train['vector'].values.tolist())
X_test  = np.array(df_test['vector'].values.tolist())

from sklearn.svm import LinearSVC
classifier = LinearSVC()
classifier.fit(X_train, Y_train)

prediction1 = classifier.predict(X_test)
def accuracy(Y_test, prediction):
  same = 0 
  for i,j in zip(Y_test, prediction):
    if i==j:
      same += 1
  return (same/len(Y_test))
# Temporary splitting of testing and training data
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report, confusion_matrix

print(confusion_matrix(Y_test, prediction1))
print(classification_report(Y_test, prediction1))
print("Accuracy: ", accuracy(Y_test, prediction1))

# Install vaderSentiment
!pip install vaderSentiment
# Sentiment Analysis

init()

from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
import re
import string

analyzer = SentimentIntensityAnalyzer()

def sentiment_analyzer_scores(text):
    score = analyzer.polarity_scores(text)
    return score['compound']

# {'neg': 0.197, 'neu': 0.754, 'pos': 0.049, 'compound': -0.9764}: Example Output
X_train, X_test = [], []

import six

for i in df_train['clean_quest']:
  if isinstance(i, six.string_types):
    X_train.append([sentiment_analyzer_scores(i)])
  else:
    X_train.append([0])

for i in df_test['clean_quest']:
  if isinstance(i, six.string_types):
    X_test.append([sentiment_analyzer_scores(i)])
  else:
    X_test.append([0])


X_train = np.array(X_train)
X_test = np.array(X_test)
from sklearn.neighbors import KNeighborsClassifier

classifier = KNeighborsClassifier(n_neighbors=5)
classifier.fit(X_train, Y_train)
prediction2 = classifier.predict(X_test)

# Temporary splitting of testing and training data
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report, confusion_matrix

print(confusion_matrix(Y_test, prediction2))
print(classification_report(Y_test, prediction2))
print("Accuracy: ", accuracy(Y_test, prediction2))

import nltk
nltk.download('punkt')
nltk.download('averaged_perceptron_tagger')
nltk.download('tagsets')
from nltk import word_tokenize
from nltk.data import load

# POS Tagging
# Creating a one-hot encoder vector to represent the information

init()

tagdict = load('help/tagsets/upenn_tagset.pickle')
pos_tags_list = list(tagdict.keys())
pos_dict = dict.fromkeys(pos_tags_list, 0)

X_train, X_test = [], []

def tagger(df):
  res = []
  for sen in df['clean_quest']:
    tags = nltk.pos_tag(word_tokenize(str(sen)))
    sen_dict = pos_dict.copy()
    for w,t in tags:
      if t in sen_dict:
        sen_dict[t] += 1
    res.append(list(sen_dict.values()))
  return res

X_train, X_test = np.array(tagger(df_train)), np.array(tagger(df_test))

classifier = LinearSVC()
classifier.fit(X_train, Y_train)

prediction3 = classifier.predict(X_test)
# Temporary splitting of testing and training data
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report, confusion_matrix

print(confusion_matrix(Y_test, prediction3))
print(classification_report(Y_test, prediction3))
print("Accuracy: ", accuracy(Y_test, prediction3))

# Self defined Decission Tree Classifier

# Using the sentiment score to check for discussions which lead classify as 'attacked'
# Using lexicon based approach after reviewing elements in the training data set for the other 3 classes

init()

sentiment_score, prediction4 = [], []
agreed = ['yup', 'yes','correct','rational','sure','absolutely','agree']
answered = ['read','actually','fact','think']

for i in df_test['clean_quest']:
  if isinstance(i, six.string_types):
    sentiment_score.append([sentiment_analyzer_scores(i)])
  else:
    sentiment_score.append([0])

def check_if_exists( sentence, lst):
  for i in lst:
    if isinstance(sentence, six.string_types):
      if i in sentence:
        return True
  return False

i=0
for index, row in df_test.iterrows():
  if sentiment_score[i][0] < (-0.5):
    prediction4.append('attacked')
  elif check_if_exists( row['clean_quest'], agreed):
    prediction4.append('agreed') 
  elif check_if_exists( row['clean_quest'], answered):
    prediction4.append('answered')
  else:
    prediction4.append('irrelevant')
  i += 1

print(confusion_matrix(Y_test, prediction4))
print(classification_report(Y_test, prediction4))
print("Accuracy: ", accuracy(Y_test, prediction4))<jupyter_output><empty_output><jupyter_text>Ensemble Voting Classifier<jupyter_code>import operator
from sklearn.ensemble import VotingClassifier

classification = {'agreed':0, 'answered':1, 'attacked':2, 'irrelevant':3}
predictionA, predictionB = [], []

for i in range(len(prediction1)):
  result1 = {'agreed':0, 'answered':0, 'attacked':0, 'irrelevant':0}
  result2 = {'agreed':0, 'answered':0, 'attacked':0, 'irrelevant':0}

  result1[prediction1[i]] += 0.5
  result1[prediction2[i]] += 0.6
  result1[prediction3[i]] += 0.3
  result1[prediction4[i]] += 0.15

  max_value = max(result1.values())
  for cls in result1.keys():
    if( result1[cls] == max_value):
      predictionA.append(cls)

  result2[prediction1[i]] += 0.5
  result2[prediction3[i]] += 0.4

  max_value = max(result2.values())
  for cls in result2.keys():
    if( result2[cls] == max_value):
      predictionB.append(cls)

from sklearn.metrics import classification_report, confusion_matrix

print(confusion_matrix(Y_test, predictionB))
print(classification_report(Y_test, predictionB))
print("Accuracy: ", accuracy(Y_test, predictionB))
from sklearn.metrics import classification_report, confusion_matrix

print(confusion_matrix(Y_test, predictionA))
print(classification_report(Y_test, predictionA))
print("Accuracy: ", accuracy(Y_test, predictionA))<jupyter_output>[[  0  13   0   0]
 [  0 319   1   0]
 [  0  39   0   0]
 [  0  37   1   0]]
              precision    recall  f1-score   support

      agreed       0.00      0.00      0.00        13
    answered       0.78      1.00      0.88       320
    attacked       0.00      0.00      0.00        39
  irrelevant       0.00      0.00      0.00        38

    accuracy                           0.78       410
   macro avg       0.20      0.25      0.22       410
weighted avg       0.61      0.78      0.68       410

Accuracy:  0.7780487804878049<jupyter_text>References:*   https://www.analyticsvidhya.com/blog/2018/06/comprehensive-guide-for-ensemble-models/*   https://www.dataquest.io/blog/tutorial-text-classification-in-python-using-spacy/*   https://www.analyticsvidhya.com/blog/2018/04/a-comprehensive-guide-to-understand-and-implement-text-classification-in-python/*   https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.VotingClassifier.html<jupyter_code><jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1469377467865244, 'fraction_numerical': 0.021255145761572714, 'mean_word_length': 4.767441860465116, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 18, 'https://': 4, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '398897', 'license': 'MIT', 'n_tokens_mistral': 4241, 'n_tokens_neox': 4029, 'n_words': 870, 'path': 'P2_sbalas22.ipynb', 'repo_name': 'shiva96b/Response-Type-Classification', 'size': '11903'}