<jupyter_start><jupyter_text>Feature Engineering for NLP Codes to create text features are present in this repository<jupyter_code>import nltk
import string
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import matplotlib.pyplot as plt
import seaborn as sns

from nltk.corpus import stopwords
from sklearn.decomposition import TruncatedSVD
from sklearn import ensemble, metrics, model_selection, naive_bayes
color = sns.color_palette()

%matplotlib inline

eng_stopwords = set(stopwords.words("english"))
pd.options.mode.chained_assignment = None
"""
Data fetched from
https://www.kaggle.com/crowdflower/twitter-airline-sentiment/data
"""
df = pd.read_csv("Tweets.csv")
df.shape
df.head()
text = df.text.values[2]
print text<jupyter_output>@VirginAmerica I didn't today... Must mean I need to take another trip!<jupyter_text>Train Test split and some preprocessing<jupyter_code>### Do a train - test split ###
from sklearn.model_selection import train_test_split

train_df, test_df = train_test_split(df, test_size=0.25, random_state=2018)
print train_df.shape, test_df.shape
df.airline_sentiment.value_counts()
mapping_dict = {'negative':0, 'neutral':1, 'positive':2}
train_y = train_df['airline_sentiment'].map(mapping_dict)
test_y = test_df['airline_sentiment'].map(mapping_dict)
train_y.value_counts()<jupyter_output><empty_output><jupyter_text>Tokenizers<jupyter_code>### tokenize the words ###
from nltk.tokenize import sent_tokenize, word_tokenize

word_tokenize(text)
### n-grams ###
from nltk import ngrams

n = 2
[ng for ng in ngrams(text.split(), n)]<jupyter_output><empty_output><jupyter_text>Count Vectorizer<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer

text1 = "Hello world"
text2 = "Hello people"

feat_vec = CountVectorizer(ngram_range=(1,1), stop_words='english', analyzer='word')
cvec = feat_vec.fit_transform([text1, text2])
feat_vec.vocabulary_
cvec.todense()<jupyter_output><empty_output><jupyter_text>TFIDF Vectorizer<jupyter_code>feat_vec = TfidfVectorizer(ngram_range=(1,1), stop_words='english', analyzer='word')
tfidf_vec = feat_vec.fit_transform([text1, text2])
feat_vec.vocabulary_
tfidf_vec.todense()<jupyter_output><empty_output><jupyter_text>Build the sentiment model<jupyter_code>### Let us get the tfidf vector for the tweets text ###
tfidf_obj = TfidfVectorizer(ngram_range=(1,1), stop_words='english', analyzer='word')

tfidf_text = tfidf_obj.fit_transform(df.text.values)
train_tfidf = tfidf_obj.transform(train_df.text.values)
test_tfidf = tfidf_obj.transform(test_df.text.values)
print train_tfidf.shape, test_tfidf.shape
from sklearn import ensemble, metrics, model_selection, naive_bayes

model = naive_bayes.MultinomialNB()
model.fit(train_tfidf, train_y)
pred_test_y = model.predict_proba(test_tfidf)<jupyter_output><empty_output><jupyter_text>Confusion Matrix<jupyter_code>### Function to create confusion matrix ###
import itertools
from sklearn.metrics import confusion_matrix

### From http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html#sphx-glr-auto-examples-model-selection-plot-confusion-matrix-py #
def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        #print("Normalized confusion matrix")
    #else:
    #    print('Confusion matrix, without normalization')

    #print(cm)

    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')
cnf_matrix = confusion_matrix(test_y, np.argmax(pred_test_y,axis=1))
np.set_printoptions(precision=2)
sns.set_style("whitegrid", {'axes.grid' : False})

# Plot non-normalized confusion matrix
plt.figure(figsize=(8,8))
plot_confusion_matrix(cnf_matrix, classes=['Negative', 'Neutral', 'Positive'],
                      title='Confusion matrix, without normalization')
plt.show()<jupyter_output><empty_output><jupyter_text>Singular Value Decomposition<jupyter_code>from sklearn.decomposition import TruncatedSVD
from sklearn.linear_model import LogisticRegression
### Create the svd features ###
svd_obj = TruncatedSVD(n_components=20)

svd_obj.fit(tfidf_text)

train_svd = svd_obj.transform(train_tfidf)
test_svd = svd_obj.transform(test_tfidf)
print train_svd.shape, test_svd.shape
### Build a logistic regression model ###
model = LogisticRegression()
model.fit(train_svd, train_y)

pred_test_y = model.predict_proba(test_svd)
cnf_matrix = confusion_matrix(test_y, np.argmax(pred_test_y,axis=1))
np.set_printoptions(precision=2)
sns.set_style("whitegrid", {'axes.grid' : False})

# Plot non-normalized confusion matrix
plt.figure(figsize=(8,8))
plot_confusion_matrix(cnf_matrix, classes=['Negative', 'Neutral', 'Positive'],
                      title='Confusion matrix, without normalization')
plt.show()<jupyter_output><empty_output><jupyter_text>Meta Features Let us create some meta features from the text and check the performance.<jupyter_code>## Number of words in the text ##
train_df["num_words"] = train_df["text"].apply(lambda x: len(str(x).split()))
test_df["num_words"] = test_df["text"].apply(lambda x: len(str(x).split()))

## Number of unique words in the text ##
train_df["num_unique_words"] = train_df["text"].apply(lambda x: len(set(str(x).split())))
test_df["num_unique_words"] = test_df["text"].apply(lambda x: len(set(str(x).split())))

## Number of characters in the text ##
train_df["num_chars"] = train_df["text"].apply(lambda x: len(str(x)))
test_df["num_chars"] = test_df["text"].apply(lambda x: len(str(x)))

## Number of stopwords in the text ##
train_df["num_stopwords"] = train_df["text"].apply(lambda x: len([w for w in str(x).lower().split() if w in eng_stopwords]))
test_df["num_stopwords"] = test_df["text"].apply(lambda x: len([w for w in str(x).lower().split() if w in eng_stopwords]))

## Number of punctuations in the text ##
train_df["num_punctuations"] =train_df['text'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]) )
test_df["num_punctuations"] =test_df['text'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]) )

## Number of upper case words in the text ##
train_df["num_words_upper"] = train_df["text"].apply(lambda x: len([w for w in str(x).split() if w.isupper()]))
test_df["num_words_upper"] = test_df["text"].apply(lambda x: len([w for w in str(x).split() if w.isupper()]))

## Number of title case words in the text ##
train_df["num_words_title"] = train_df["text"].apply(lambda x: len([w for w in str(x).split() if w.istitle()]))
test_df["num_words_title"] = test_df["text"].apply(lambda x: len([w for w in str(x).split() if w.istitle()]))

## Average length of the words in the text ##
train_df["mean_word_len"] = train_df["text"].apply(lambda x: np.mean([len(w) for w in str(x).split()]))
test_df["mean_word_len"] = test_df["text"].apply(lambda x: np.mean([len(w) for w in str(x).split()]))
cols = ["num_words", "num_unique_words", "num_chars", "num_stopwords", "num_punctuations", "num_words_upper", "num_words_title", "mean_word_len", "retweet_count"]
train_X = train_df[cols]
test_X = test_df[cols]

model = LogisticRegression()
model.fit(train_X, train_y)

pred_test_y = model.predict_proba(test_X)
cnf_matrix = confusion_matrix(test_y, np.argmax(pred_test_y,axis=1))

# Plot non-normalized confusion matrix
plt.figure(figsize=(8,8))
plot_confusion_matrix(cnf_matrix, classes=['Negative', 'Neutral', 'Positive'],
                      title='Confusion matrix, without normalization')
plt.show()<jupyter_output><empty_output><jupyter_text>Text features + Meta features Now let us combine both text features and meta features and check the performance again.<jupyter_code>train_X2 = np.hstack([train_svd, train_X])
test_X2 = np.hstack([test_svd, test_X])

model = LogisticRegression()
model.fit(train_X2, train_y)

pred_test_y = model.predict_proba(test_X2)

cnf_matrix = confusion_matrix(test_y, np.argmax(pred_test_y,axis=1))

# Plot non-normalized confusion matrix
plt.figure(figsize=(8,8))
plot_confusion_matrix(cnf_matrix, classes=['Negative', 'Neutral', 'Positive'],
                      title='Confusion matrix, without normalization')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13715046604527298, 'fraction_numerical': 0.005659121171770972, 'mean_word_length': 5.9652241112828435, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 41, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '77416', 'license': 'MIT', 'n_tokens_mistral': 3199, 'n_tokens_neox': 3125, 'n_words': 662, 'path': 'src/Feature_Engg.ipynb', 'repo_name': 'varunnathan/NLP', 'size': '9012'}