<jupyter_start><jupyter_text>Import<jupyter_code>import pandas as pd
import numpy as np
import pickle
import sys
import nltk
from nltk.stem.porter import *

from sklearn.feature_extraction.text import TfidfVectorizer
import matplotlib.pyplot as plt
import seaborn
%matplotlib inline
from google.colab import drive
drive.mount('/content/drive')
data = pd.read_csv('/content/drive/My Drive/Shaastra Workshop Material/Text Analysis/spam.csv',encoding = "ISO-8859-1")
data.drop(['Unnamed: 2','Unnamed: 3','Unnamed: 4'], axis=1, inplace=True)
data.rename(columns={'v1':'target','v2':'text'},inplace=True)
data.head()<jupyter_output><empty_output><jupyter_text>Basic Preprocessing<jupyter_code>train = data.copy()

train['text'] = train['text'].astype(str)
### Fill your code here

## 1. Lowercasing
train['text'] = train['text'].apply(lambda x: " ".join(x.lower() for x in x.split()))

## 2. Punctuation Removal
train['text'] = train['text'].str.replace('[^\w\s]','')

## 3. Stopwords Removal
nltk.download('stopwords')
## for removing the stopwords
from nltk.corpus import stopwords

stop = stopwords.words('english')
train['text'] = train['text'].apply(lambda x: " ".join(x for x in x.split() if x not in stop))
train['text'].head()
""" 

fill your code for all these operations

"""
## lemmatization
## write your code here

from textblob import Word
nltk.download('wordnet')
train['text'] = train['text'].apply(lambda x: " ".join([Word(word).lemmatize() for word in x.split()]))
train['text'].head()<jupyter_output>[nltk_data] Downloading package wordnet to /root/nltk_data...
[nltk_data]   Unzipping corpora/wordnet.zip.<jupyter_text>Target creation<jupyter_code>train['target'].unique()#.isna().sum()
train['target'].replace(['ham','spam'],[0,1],inplace=True)
train['target'].value_counts()
train<jupyter_output><empty_output><jupyter_text>Basic Model on whole dataset<jupyter_code>from sklearn.model_selection import train_test_split
x_train,x_valid,y_t,y_v = train_test_split(train['text'],train['target'],test_size=0.2,random_state=4353)
from sklearn.feature_extraction.text import TfidfVectorizer
tfidf = TfidfVectorizer(max_features=100, lowercase=True, analyzer='word',
 stop_words= 'english',ngram_range=(1,1))

tfidf.fit(train['text'])

x_t = tfidf.transform(x_train)
x_v = tfidf.transform(x_valid)

x_v.toarray().shape
def model_training(clf, x_t, y_t, x_v=None , y_v=None ,task='binary:logistic'):
    
    clf.fit(x_t,y_t)
    
    print('training accuracy', clf.score(x_t,y_t))
    
    if task=='binary:logistic':
      print('validation accuracy', clf.score(x_v,y_v))
      print('validation f1_score',f1_score(clf.predict(x_v),y_v))
      print('validation roc_auc score',roc_auc_score(y_v,clf.predict_proba(x_v)[::,-1]))
      print('confusion matrix \n',confusion_matrix(y_v, clf.predict(x_v)))
    
    if task=='reg:linear':
        if x_v!=None:
            print('validation r2_score', clf.score(x_v,y_v))
            print('validation MSE',mean_squared_error(clf.predict(x_v),y_v))

            
    return clf
train['target'].mean()
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.metrics import f1_score, classification_report, roc_auc_score, confusion_matrix, accuracy_score
from xgboost import XGBClassifier

lgr =  LogisticRegression(n_jobs=1)
# model_training(lgr,x_t,y_t,x_v,y_v)

xgb = XGBClassifier(n_estimators=500, max_depth=5,learning_rate=0.1,scale_pos_weight=1.4266790777602751)
# xgb = XGBClassifier(n_estimators=500, max_depth=5,learning_rate=0.1,scale_pos_weight=1.6760372565622355)

model_training(lgr,x_t,y_t,x_v,y_v)

# training accuracy 0.974646623289208
# validation accuracy 0.967713004484305
# validation f1_score 0.859375
# validation roc_auc score 0.9588548752834467
# confusion matrix 
#  [[969  11]
#  [ 25 110]]
# XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
#               colsample_bynode=1, colsample_bytree=1, gamma=0,
#               learning_rate=0.1, max_delta_step=0, max_depth=5,
#               min_child_weight=1, missing=None, n_estimators=500, n_jobs=1,
#               nthread=None, objective='binary:logistic', random_state=0,
#               reg_alpha=0, reg_lambda=1, scale_pos_weight=1.4266790777602751,
#               seed=None, silent=None, subsample=1, verbosity=1)<jupyter_output><empty_output><jupyter_text>Training on pretrained word2vec model<jupyter_code>import gensim
import logging
from gensim.models import Word2Vec

wv = gensim.models.KeyedVectors.load_word2vec_format("/content/drive/My Drive/Shaastra Workshop Material/Text Analysis/GoogleNews-vectors-negative300.bin.gz", binary=True)
# above is only pretrained embeddings
# wv = gensim.models.KeyedVectors.load_word2vec_format("tmp.txt")

wv.init_sims(replace=True)

def word_averaging(wv, words):
    all_words, mean = set(), []
    
    for word in words:
        if isinstance(word, np.ndarray):
            mean.append(word)
        elif word in wv.vocab:
            mean.append(wv.syn0norm[wv.vocab[word].index])
            all_words.add(wv.vocab[word].index)

    if not mean:
        logging.warning("cannot compute similarity with no input %s", words)
        # FIXME: remove these examples in pre-processing
        return np.zeros(wv.vector_size,)

    mean = gensim.matutils.unitvec(np.array(mean).mean(axis=0)).astype(np.float32)
    return mean

def  word_averaging_list(wv, text_list):
    return np.vstack([word_averaging(wv, post) for post in text_list ])
import nltk
nltk.download('punkt')
from sklearn.model_selection import train_test_split
import nltk
nltk.download('punkt')
def w2v_tokenize_text(text):
    tokens = []
    for sent in nltk.sent_tokenize(text, language='english'):
        for word in nltk.word_tokenize(sent, language='english'):
            if len(word) < 2:
                continue
            tokens.append(word)
    return tokens
    
train_w2v, test_w2v = train_test_split(train, test_size=0.2, random_state = 234)
# x_t,x_v,y_t,y_v = train_test_split(train['tweet'],train['class'],test_size=0.2,random_state=234)

test_tokenized = test_w2v.apply(lambda r: w2v_tokenize_text(r['text']), axis=1).values
train_tokenized = train_w2v.apply(lambda r: w2v_tokenize_text(r['text']), axis=1).values

X_train_word_average = word_averaging_list(wv,train_tokenized)
X_test_word_average = word_averaging_list(wv,test_tokenized)
wv.similarity(w1='queen',w2='royal')
wv.similarity(w1='lion',w2='cub')
from sklearn.linear_model import LogisticRegression
from xgboost import XGBClassifier

from lightgbm import LGBMClassifier

from sklearn.ensemble import GradientBoostingClassifier

from sklearn.ensemble import GradientBoostingClassifier
from sklearn.metrics import f1_score, classification_report, roc_auc_score, confusion_matrix, accuracy_score<jupyter_output><empty_output><jupyter_text><jupyter_code>def model_training(clf, x_t, y_t, x_v=None , y_v=None ,task='binary:logistic'):
    clf.fit(x_t,y_t)
    print('training accuracy', clf.score(x_t,y_t))
    
    if task=='binary:logistic':
      print('validation accuracy', clf.score(x_v,y_v))
      print('validation f1_score',f1_score(clf.predict(x_v),y_v))
      print('validation roc_auc score',roc_auc_score(y_v,clf.predict_proba(x_v)[::,-1]))
      print('confusion matrix \n',confusion_matrix(y_v, clf.predict(x_v)))
    
    if task=='reg:linear':
      print('validation r2_score', clf.score(x_v,y_v))
      print('validation MSE',mean_squared_error(clf.predict(x_v),y_v))

            
    return clf
%%time
xgb_w2v = XGBClassifier(n_estimators=500, max_depth=5,learning_rate=0.1,scale_pos_weight=1.4266790777602751)
lgr_w2v = LogisticRegression(n_jobs=1)

lgbm_w2v = LGBMClassifier(n_estimators=500)

gbdt_w2v = GradientBoostingClassifier(n_estimators=500)

model_training(xgb_w2v,X_train_word_average,train_w2v['target'],X_test_word_average,test_w2v['target'])

# model_training(lgr_w2v,X_train_word_average,train_w2v['class'],X_test_word_average,test_w2v['class'])

# model_training(lgbm_w2v,X_train_word_average,train_w2v['class'],X_test_word_average,test_w2v['class'])

# model_training(gbdt_w2v,X_train_word_average,train_w2v['class'],X_test_word_average,test_w2v['class'])

tuned_pred = (xgb_w2v.predict_proba(X_test_word_average)[::,-1]>0.3).astype(int)
confusion_matrix(test_w2v['target'],tuned_pred)
f1_score(test_w2v['target'],tuned_pred)<jupyter_output><empty_output><jupyter_text>Word2Vec DOMAIN Training<jupyter_code># imports needed and set up logging
import gzip
import gensim 
import logging

logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)
def read_input(data):
    """This method reads the input file which is in gzip format"""
    i=0
    for line in data['text']: 
      i+=1
      if (i%10000==0):
        logging.info ("read {0} tweets".format(i))
      # do some pre-processing and return a list of words for each tweet, basically doing tokenizing
      yield gensim.utils.simple_preprocess (line)

# read the tokenized reviews into a list
# each review item becomes a serries of words
# so this becomes a list of lists
documents = list(read_input(train))
logging.info ("Done reading data file")
documents = []
i=0
for line in train['text']: 
    i+=1
    if (i%1000==0):
      logging.info ("read {0} messages".format(i))
      # do some pre-processing and return a list of words for each tweet, basically doing tokenizing
    # documents.append(gensim.utils.simple_preprocess(line))
    documents.append(nltk.word_tokenize(line))<jupyter_output><empty_output><jupyter_text>Training Time<jupyter_code>%%time

model = gensim.models.Word2Vec(documents, size=50, min_count=2, workers=5)
model.train(documents,total_examples=len(documents),epochs=10)


w1 = "discount"
model.wv.most_similar(positive=w1)
model.wv.save_word2vec_format('tmp.txt', binary=False)
## save the model
wv = gensim.models.KeyedVectors.load_word2vec_format('tmp.txt')
wv.init_sims(replace=True)

train_w2v, test_w2v = train_test_split(train, test_size=0.2, random_state = 234)
# x_t,x_v,y_t,y_v = train_test_split(train['tweet'],train['class'],test_size=0.2,random_state=234)

test_tokenized = test_w2v.apply(lambda r: w2v_tokenize_text(r['text']), axis=1).values
train_tokenized = train_w2v.apply(lambda r: w2v_tokenize_text(r['text']), axis=1).values

X_train_word_average = word_averaging_list(wv,train_tokenized)
X_test_word_average = word_averaging_list(wv,test_tokenized)
%%time
xgb_w2v = XGBClassifier(n_estimators=500, max_depth=5,learning_rate=0.1,scale_pos_weight=1.4266790777602751)
lgr_w2v = LogisticRegression(n_jobs=1)

lgbm_w2v = LGBMClassifier(n_estimators=500)

gbdt_w2v = GradientBoostingClassifier(n_estimators=500)

model_training(xgb_w2v,X_train_word_average,train_w2v['target'],X_test_word_average,test_w2v['target'])

# model_training(lgr_w2v,X_train_word_average,train_w2v['class'],X_test_word_average,test_w2v['class'])

# model_training(lgbm_w2v,X_train_word_average,train_w2v['class'],X_test_word_average,test_w2v['class'])

# model_training(gbdt_w2v,X_train_word_average,train_w2v['class'],X_test_word_average,test_w2v['class'])<jupyter_output><empty_output><jupyter_text>Transfer Learning<jupyter_code>from gensim.models import Word2Vec
sentences = documents
# size option needs to be set to 300 to be the same as Google's pre-trained model
 
word2vec_model = Word2Vec(size = 300, window=5,
min_count = 1, workers = 10)

word2vec_model.build_vocab(sentences)
 
# assign the vectors to the vocabs that are in Google's pre-trained model and your sentences defined above.

word2vec_model.intersect_word2vec_format('/content/drive/My Drive/Shaastra Workshop Material/Text Analysis/GoogleNews-vectors-negative300.bin.gz', lockf=1.0, binary=True)
# continue training with you own data
word2vec_model.train(sentences, total_examples=len(sentences), epochs = 5)
w1 = ["sale"]
word2vec_model.wv.most_similar (positive=w1)
word2vec_model.wv.save_word2vec_format('model_transfer_learning.txt', binary=False)
#### Training on this

wv = gensim.models.KeyedVectors.load_word2vec_format('model_transfer_learning.txt')
wv.init_sims(replace=True)

train_w2v, test_w2v = train_test_split(train, test_size=0.2, random_state = 234)
# x_t,x_v,y_t,y_v = train_test_split(train['tweet'],train['class'],test_size=0.2,random_state=234)

test_tokenized = test_w2v.apply(lambda r: w2v_tokenize_text(r['text']), axis=1).values
train_tokenized = train_w2v.apply(lambda r: w2v_tokenize_text(r['text']), axis=1).values

X_train_word_average = word_averaging_list(wv,train_tokenized)
X_test_word_average = word_averaging_list(wv,test_tokenized)
%%time
xgb_w2v = XGBClassifier(n_estimators=500, max_depth=5,learning_rate=0.1,scale_pos_weight=1.4266790777602751)
lgr_w2v = LogisticRegression(n_jobs=1)

lgbm_w2v = LGBMClassifier(n_estimators=500)

gbdt_w2v = GradientBoostingClassifier(n_estimators=500)

model_training(xgb_w2v,X_train_word_average,train_w2v['target'],X_test_word_average,test_w2v['target'])

# model_training(lgr_w2v,X_train_word_average,train_w2v['class'],X_test_word_average,test_w2v['class'])

# model_training(lgbm_w2v,X_train_word_average,train_w2v['class'],X_test_word_average,test_w2v['class'])

# model_training(gbdt_w2v,X_train_word_average,train_w2v['class'],X_test_word_average,test_w2v['class'])

### indeed an increase in F1-Score<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12820704057279236, 'fraction_numerical': 0.03453162291169451, 'mean_word_length': 6.6535388127853885, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '510821', 'license': 'MIT', 'n_tokens_mistral': 5337, 'n_tokens_neox': 5065, 'n_words': 780, 'path': 'Spam_Classifier.ipynb', 'repo_name': 'sumanthsadhu/Text_Analysis', 'size': '13408'}