<jupyter_start><jupyter_text>Comparison of TFIDF and word2vec . By- <NAME><jupyter_code>import logging
import pandas as pd
import numpy as np
from numpy import random
import gensim
import nltk
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.metrics import accuracy_score, confusion_matrix
import matplotlib.pyplot as plt
from nltk.corpus import stopwords
import re
from bs4 import BeautifulSoup

%matplotlib inline<jupyter_output><empty_output><jupyter_text>We are going to evaluate the two word embeddings first with a relatively small data set then with the data set of relatively larger corpus.<jupyter_code>import json

lst_dics = []
with open('data_news.json', mode='r', errors='ignore') as json_file:
    for dic in json_file:
        lst_dics.append( json.loads(dic) )
    
lst_dics[0]
df = pd.DataFrame(lst_dics)
df = df[ df["category"].isin(['ENTERTAINMENT','POLITICS','TECH']) ][["category","headline"]]
df = df.rename(columns={"headline":"post" ,"category":"tags"})
df.sample(5)<jupyter_output><empty_output><jupyter_text>let's take a look at our distribution of categories in the above data set.<jupyter_code>plt.figure(figsize=(10,4))
df.tags.value_counts().plot(kind='bar');
def print_plot(index):
    postValue = df[df.index == index][['post', 'tags']].values[0]
    if len(postValue) > 0:
        print(postValue[0])
        print('Tag:', postValue[1])
print_plot(5)<jupyter_output>Morgan Freeman 'Devastated' That Sexual Harassment Claims Could Undermine Legacy
Tag: ENTERTAINMENT<jupyter_text>Performing cleanup of text:<jupyter_code>REPLACE_BY_SPACE_RE = re.compile('[/(){}\[\]\|@,;]')
BAD_SYMBOLS_RE = re.compile('[^0-9a-z #+_]')
STOPWORDS = set(stopwords.words('english'))

def clean_text(text):
    """
        text: a string
        
        return: modified initial string
    """
#     print('text', text)
#     text = BeautifulSoup(text, "lxml").text # HTML decoding
#     text = BeautifulSoup(text,  "html.parser")
    text = text.lower() # lowercase text
    text = REPLACE_BY_SPACE_RE.sub(' ', text) # replace REPLACE_BY_SPACE_RE symbols by space in text
    text = BAD_SYMBOLS_RE.sub('', text) # delete symbols which are in BAD_SYMBOLS_RE from text
    text = ' '.join(word for word in text.split() if word not in STOPWORDS) # delete stopwors from text
    return text
df['post'] = df['post'].apply(clean_text)
print_plot(5)
df['post'].apply(lambda x: len(x.split(' '))).sum()<jupyter_output><empty_output><jupyter_text>As we can se above we have around 300k words to work with for our comparison.Let's first understand what is TF-IDF?  TF-IDF  Term Frequency (TF)The number of times a word appears in a document divded by the total number of words in the document. Every document has its own term frequency.   Inverse Data Frequency (IDF)The log of the number of documents divided by the number of documents that contain the word w. Inverse data frequency determines the weight of rare words across all documents in the corpus.  Finally, the TF-IDF is simply the TF multiplied by IDF.  In the next steps, We will convert our text documents to a matrix of token counts (CountVectorizer), then transform a count matrix to a normalized tf-idf representation (tf-idf transformer). After that, we train several classifiers.<jupyter_code>X = df.post
y = df.tags
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state = 42)
from sklearn.naive_bayes import MultinomialNB
from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import TfidfTransformer

nb = Pipeline([('vect', CountVectorizer()),
               ('tfidf', TfidfTransformer()),
               ('clf', MultinomialNB()),
              ])
nb.fit(X_train, y_train)
%%time
from sklearn.metrics import classification_report
import matplotlib.pyplot as plt  
from sklearn.datasets import make_classification
from sklearn.metrics import plot_confusion_matrix
from sklearn.model_selection import train_test_split

y_pred = nb.predict(X_test)

print('accuracy %s' % accuracy_score(y_pred, y_test))
print(classification_report(y_test, y_pred))

# for plotting report
def plot_classification_report(cr, title='Classification report ', with_avg_total=False, cmap=plt.cm.Blues):

    lines = cr.split('\n')

    classes = []
    plotMat = []
    for line in lines[2 : (len(lines) - 3)]:   
        #print(line)    
        t = line.split()      
        # print(t)    
        if(len(t)==0):         
            break
#     for line in lines[2 : (len(lines) - 3)]:
#         #print(line)
#         t = line.split()
#         # print(t)
        classes.append(t[0])
        v = [float(x) for x in t[1: len(t) - 1]]
        print(v)
        plotMat.append(v)

    if with_avg_total:
        aveTotal = lines[len(lines) - 1].split()
        classes.append('avg/total')
        vAveTotal = [float(x) for x in t[1:len(aveTotal) - 1]]
        plotMat.append(vAveTotal)


    plt.imshow(plotMat, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    x_tick_marks = np.arange(3)
    y_tick_marks = np.arange(len(classes))
    plt.xticks(x_tick_marks, ['precision', 'recall', 'f1-score'], rotation=45)
    plt.yticks(y_tick_marks, classes)
    plt.tight_layout()
    plt.ylabel('Classes')
    plt.xlabel('Measures')


plot_confusion_matrix(nb,y_test, y_pred)  
target_names = ['POLITICS','ENTERTAINMENT','TECH']
classificationReport = classification_report(y_test, y_pred, target_names=target_names)

plot_classification_report(classificationReport)
# plot_classification_report(sampleClassificationReport)

plt.show()<jupyter_output>accuracy 0.9001572327044025
               precision    recall  f1-score   support

ENTERTAINMENT       0.95      0.83      0.89      4905
     POLITICS       0.88      0.99      0.93      9759
         TECH       1.00      0.07      0.12       600

     accuracy                           0.90     15264
    macro avg       0.94      0.63      0.65     15264
 weighted avg       0.91      0.90      0.88     15264

[0.95, 0.83, 0.89]
[0.88, 0.99, 0.93]
[1.0, 0.07, 0.12]<jupyter_text>we achieved 90% accuracy for the above case  Word2Vec word2vec algorithm was developed by milkov and here we can se below the two parts of word2vec: 1.Skip-gram Word2Vec:Skip-gram Word2Vec is an architecture for computing word embeddings. Instead of using surrounding words to predict the center word, as with CBow Word2Vec, Skip-gram Word2Vec uses the central word to predict the surrounding words.The skip-gram objective function sums the log probabilities of the surrounding  words to the left and right of the target word  to produce the following objective:  2.Continuous Bag-of-Words Word2Vec is an architecture for creating word embeddings that uses  future words as well as  past words to create a word embedding. The objective function for CBOW is:<jupyter_code>%%time
from gensim.models import Word2Vec

wv = gensim.models.KeyedVectors.load_word2vec_format("GoogleNews-vectors-negative300.bin.gz", binary=True)
wv.init_sims(replace=True)

from itertools import islice
list(islice(wv.vocab, 13030, 13050))<jupyter_output><empty_output><jupyter_text>The common way is to average the two word vectors.we are using BOW based approaches which includes averaging.<jupyter_code>def word_averaging(wv, words):
    all_words, mean = set(), []
    
    for word in words:
        if isinstance(word, np.ndarray):
            mean.append(word)
        elif word in wv.vocab:
            mean.append(wv.syn0norm[wv.vocab[word].index])
            all_words.add(wv.vocab[word].index)

    if not mean:
        logging.warning("cannot compute similarity with no input %s", words)
        return np.zeros(wv.vector_size,)

    mean = gensim.matutils.unitvec(np.array(mean).mean(axis=0)).astype(np.float32)
    return mean

def  word_averaging_list(wv, text_list):
    return np.vstack([word_averaging(wv, post) for post in text_list ])<jupyter_output><empty_output><jupyter_text>tokenising the words<jupyter_code>def w2v_tokenize_text(text):
    tokens = []
    for sent in nltk.sent_tokenize(text, language='english'):
        for word in nltk.word_tokenize(sent, language='english'):
            if len(word) < 2:
                continue
            tokens.append(word)
    return tokens
train, test = train_test_split(df, test_size=0.3, random_state = 42)

test_tokenized = test.apply(lambda r: w2v_tokenize_text(r['post']), axis=1).values
train_tokenized = train.apply(lambda r: w2v_tokenize_text(r['post']), axis=1).values
X_train_word_average = word_averaging_list(wv,train_tokenized)
X_test_word_average = word_averaging_list(wv,test_tokenized)<jupyter_output><ipython-input-16-664dfaa6d69c>:8: DeprecationWarning: Call to deprecated `syn0norm` (Attribute will be removed in 4.0.0, use self.vectors_norm instead).
  mean.append(wv.syn0norm[wv.vocab[word].index])
WARNING:root:cannot compute similarity with no input []
WARNING:root:cannot compute similarity with no input ['mossep11']
WARNING:root:cannot compute similarity with no input ['laggies']
WARNING:root:cannot compute similarity with no input ['cait', 'isnt', 'caitlyn', 'jenner']
WARNING:root:cannot compute similarity with no input ['hagel', 'obamaland']
WARNING:root:cannot compute similarity with no input ['piketty', 'elysium']
WARNING:root:cannot compute similarity with no input []
WARNING:root:cannot compute similarity with no input ['thisisnotus', 'thisisalsowhoweare']
WARNING:root:cannot compute similarity with no input ['selma']
WARNING:root:cannot compute similarity with no input []
WARNING:root:cannot compute similarity with no input ['unobama']
WARNING:root:cannot compute similari[...]<jupyter_text>using logistic regression<jupyter_code>from sklearn.linear_model import LogisticRegression
logreg = LogisticRegression(n_jobs=1, C=1e5)
logreg = logreg.fit(X_train_word_average, train['tags'])
y_pred = logreg.predict(X_test_word_average)
print('accuracy %s' % accuracy_score(y_pred, test.tags))
print(classification_report(test.tags, y_pred))

# for plotting report
def plot_classification_report(cr, title='Classification report ', with_avg_total=False, cmap=plt.cm.Blues):

    lines = cr.split('\n')

    classes = []
    plotMat = []
    for line in lines[2 : (len(lines) - 3)]:   
        #print(line)    
        t = line.split()      
        # print(t)    
        if(len(t)==0):         
            break
#     for line in lines[2 : (len(lines) - 3)]:
#         #print(line)
#         t = line.split()
#         # print(t)
        classes.append(t[0])
        v = [float(x) for x in t[1: len(t) - 1]]
        print(v)
        plotMat.append(v)

    if with_avg_total:
        aveTotal = lines[len(lines) - 1].split()
        classes.append('avg/total')
        vAveTotal = [float(x) for x in t[1:len(aveTotal) - 1]]
        plotMat.append(vAveTotal)


    plt.imshow(plotMat, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    x_tick_marks = np.arange(3)
    y_tick_marks = np.arange(len(classes))
    plt.xticks(x_tick_marks, ['precision', 'recall', 'f1-score'], rotation=45)
    plt.yticks(y_tick_marks, classes)
    plt.tight_layout()
    plt.ylabel('Classes')
    plt.xlabel('Measures')


plot_confusion_matrix(nb,y_test, y_pred)  
target_names = ['POLITICS','ENTERTAINMENT','TECH']
classificationReport = classification_report(y_test, y_pred, target_names=target_names)

plot_classification_report(classificationReport)
# plot_classification_report(sampleClassificationReport)

plt.show()<jupyter_output>accuracy 0.9061844863731656
               precision    recall  f1-score   support

ENTERTAINMENT       0.89      0.86      0.88      4905
     POLITICS       0.92      0.95      0.93      9759
         TECH       0.73      0.61      0.67       600

     accuracy                           0.91     15264
    macro avg       0.85      0.81      0.83     15264
 weighted avg       0.90      0.91      0.90     15264

[0.89, 0.86, 0.88]
[0.92, 0.95, 0.93]
[0.73, 0.61, 0.67]<jupyter_text>we achieved 90.6% accuracy for the above caseas with this set of corpus we can see both the cases are almost equal and any one can be used for a small data set of words around 300k  we are now going to perfom the same functions as done above, Now our corpus is significantly bigger for comparison.<jupyter_code>df2 = pd.read_csv('stack-overflow-data.csv')
df2 = df2[pd.notnull(df2['tags'])]
df2.head(10)
df2['post'].apply(lambda x: len(x.split(' '))).sum()<jupyter_output><empty_output><jupyter_text>as we can see above we have more than 10 million words in the data<jupyter_code>my_tags = ['java','html','asp.net','c#','ruby-on-rails','jquery','mysql','php','ios','javascript','python','c','css','android','iphone','sql','objective-c','c++','angularjs','.net']
plt.figure(figsize=(10,4))
df2.tags.value_counts().plot(kind='bar');<jupyter_output><empty_output><jupyter_text>As we can see above this is a balanced data set, as classes are balanced.<jupyter_code>def print_plot(index):
    example = df2[df2.index == index][['post', 'tags']].values[0]
    if len(example) > 0:
        print(example[0])
        print('Tag:', example[1])
print_plot(10)
REPLACE_BY_SPACE_RE = re.compile('[/(){}\[\]\|@,;]')
BAD_SYMBOLS_RE = re.compile('[^0-9a-z #+_]')
STOPWORDS = set(stopwords.words('english'))

def clean_text(text):
    """
        text: a string
        
        return: modified initial string
    """
    text = BeautifulSoup(text, "lxml").text # HTML decoding
    text = text.lower() # lowercase text
    text = REPLACE_BY_SPACE_RE.sub(' ', text) # replace REPLACE_BY_SPACE_RE symbols by space in text
    text = BAD_SYMBOLS_RE.sub('', text) # delete symbols which are in BAD_SYMBOLS_RE from text
    text = ' '.join(word for word in text.split() if word not in STOPWORDS) # delete stopwors from text
    return text
df2['post'] = df2['post'].apply(clean_text)<jupyter_output><empty_output><jupyter_text>after cleaning the text:<jupyter_code>print_plot(10)
df2['post'].apply(lambda x: len(x.split(' '))).sum()<jupyter_output><empty_output><jupyter_text>after cleaning text, we have around 3 million words to work with.<jupyter_code>X = df2.post
y = df2.tags
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state = 42)<jupyter_output><empty_output><jupyter_text>TF-IDF on second data set<jupyter_code>nb2 = Pipeline([('vect', CountVectorizer()),
               ('tfidf', TfidfTransformer()),
               ('clf', MultinomialNB()),
              ])
nb2.fit(X_train, y_train)
y_pred = nb2.predict(X_test)

print('accuracy %s' % accuracy_score(y_pred, y_test))
print(classification_report(y_test, y_pred, target_names=my_tags))

# for plotting report
def plot_classification_report(cr, title='Classification report ', with_avg_total=False, cmap=plt.cm.Blues):

    lines = cr.split('\n')

    classes = []
    plotMat = []
    for line in lines[2 : (len(lines) - 3)]:   
        #print(line)    
        t = line.split()      
        # print(t)    
        if(len(t)==0):         
            break
#     for line in lines[2 : (len(lines) - 3)]:
#         #print(line)
#         t = line.split()
#         # print(t)
        classes.append(t[0])
        v = [float(x) for x in t[1: len(t) - 1]]
        print(v)
        plotMat.append(v)

    if with_avg_total:
        aveTotal = lines[len(lines) - 1].split()
        classes.append('avg/total')
        vAveTotal = [float(x) for x in t[1:len(aveTotal) - 1]]
        plotMat.append(vAveTotal)


    plt.imshow(plotMat, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    x_tick_marks = np.arange(3)
    y_tick_marks = np.arange(len(classes))
    plt.xticks(x_tick_marks, ['precision', 'recall', 'f1-score'], rotation=45)
    plt.yticks(y_tick_marks, classes)
    plt.tight_layout()
    plt.ylabel('Classes')
    plt.xlabel('Measures')


plot_confusion_matrix(nb,y_test, y_pred)  
target_names = ['java','html','asp.net','c#','ruby-on-rails','jquery','mysql','php','ios','javascript','python','c','css','android','iphone','sql','objective-c','c++','angularjs','.net']

classificationReport = classification_report(y_test, y_pred, target_names=target_names)

plot_classification_report(classificationReport)
# plot_classification_report(sampleClassificationReport)

plt.show()<jupyter_output>accuracy 0.7393333333333333
               precision    recall  f1-score   support

         java       0.63      0.65      0.64       613
         html       0.94      0.86      0.90       620
      asp.net       0.87      0.92      0.90       587
           c#       0.70      0.77      0.73       586
ruby-on-rails       0.73      0.87      0.79       599
       jquery       0.72      0.51      0.60       589
        mysql       0.77      0.74      0.75       594
          php       0.69      0.89      0.78       610
          ios       0.63      0.59      0.61       617
   javascript       0.57      0.65      0.60       587
       python       0.70      0.50      0.59       611
            c       0.79      0.78      0.79       594
          css       0.84      0.59      0.69       619
      android       0.65      0.84      0.74       574
       iphone       0.64      0.83      0.72       584
          sql       0.66      0.64      0.65       578
  objective-c       0.79      0.77  [...]<jupyter_text>Word2Vec<jupyter_code>list(islice(wv.vocab, 13030, 13050))
def word_averaging(wv, words):
    all_words, mean = set(), []
    
    for word in words:
        if isinstance(word, np.ndarray):
            mean.append(word)
        elif word in wv.vocab:
            mean.append(wv.syn0norm[wv.vocab[word].index])
            all_words.add(wv.vocab[word].index)

    if not mean:
        logging.warning("cannot compute similarity with no input %s", words)
        # FIXME: remove these examples in pre-processing
        return np.zeros(wv.vector_size,)

    mean = gensim.matutils.unitvec(np.array(mean).mean(axis=0)).astype(np.float32)
    return mean

def  word_averaging_list(wv, text_list):
    return np.vstack([word_averaging(wv, post) for post in text_list ])
def w2v_tokenize_text(text):
    tokens = []
    for sent in nltk.sent_tokenize(text, language='english'):
        for word in nltk.word_tokenize(sent, language='english'):
            if len(word) < 2:
                continue
            tokens.append(word)
    return tokens
train, test = train_test_split(df2, test_size=0.3, random_state = 42)

test_tokenized = test.apply(lambda r: w2v_tokenize_text(r['post']), axis=1).values
train_tokenized = train.apply(lambda r: w2v_tokenize_text(r['post']), axis=1).values
X_train_word_average = word_averaging_list(wv,train_tokenized)
X_test_word_average = word_averaging_list(wv,test_tokenized)<jupyter_output><ipython-input-43-664dfaa6d69c>:8: DeprecationWarning: Call to deprecated `syn0norm` (Attribute will be removed in 4.0.0, use self.vectors_norm instead).
  mean.append(wv.syn0norm[wv.vocab[word].index])
WARNING:root:cannot compute similarity with no input []
WARNING:root:cannot compute similarity with no input ['ngrepeat']
WARNING:root:cannot compute similarity with no input []<jupyter_text>perfoming logistic regression<jupyter_code>logreg = LogisticRegression(n_jobs=1, C=1e5)
logreg = logreg.fit(X_train_word_average, train['tags'])
y_pred = logreg.predict(X_test_word_average)

print('accuracy %s' % accuracy_score(y_pred, test.tags))
print(classification_report(test.tags, y_pred,target_names=my_tags))<jupyter_output>accuracy 0.63225
               precision    recall  f1-score   support

         java       0.61      0.57      0.59       613
         html       0.76      0.75      0.76       620
      asp.net       0.66      0.64      0.65       587
           c#       0.52      0.52      0.52       586
ruby-on-rails       0.70      0.75      0.72       599
       jquery       0.40      0.39      0.40       589
        mysql       0.64      0.61      0.63       594
          php       0.74      0.78      0.76       610
          ios       0.59      0.61      0.60       617
   javascript       0.54      0.53      0.54       587
       python       0.58      0.53      0.55       611
            c       0.60      0.60      0.60       594
          css       0.63      0.60      0.61       619
      android       0.58      0.59      0.58       574
       iphone       0.70      0.73      0.71       584
          sql       0.40      0.42      0.41       578
  objective-c       0.68      0.70      0.69   [...]
{'fraction_non_alphanumeric': 0.11661637301664557, 'fraction_numerical': 0.042636036211428016, 'mean_word_length': 3.024877571008815, 'pattern_counts': {'":': 2, '<': 67, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '198331', 'license': 'MIT', 'n_tokens_mistral': 7397, 'n_tokens_neox': 6858, 'n_words': 1665, 'path': 'comparing TF IDF and Word2vec.ipynb', 'repo_name': 'kanav10/word-embedding-performace-comparison', 'size': '20551'}