<jupyter_start><jupyter_text>**Feature extraction (vectorization) :** Build Bag-Of-Words model using CountVectorizer and TfidfVectorizer.<jupyter_code># Convert text to word count vectors with CountVectorizer
# create the transform
cvec = CountVectorizer()

# tokenize, build vocab and encode training data
traindata_cvec = cvec.fit_transform(traindata['new'].values)

# summarize
print(cvec.vocabulary_)
# print(cvec.get_feature_names())<jupyter_output>{'we': 9447, 'have': 4160, 'less': 5141, 'americans': 710, 'working': 9605, 'now': 5977, 'than': 8725, 'in': 4483, 'the': 8739, '70s': 323, 'however': 4354, 'hartzler': 4146, 'was': 9419, 'talking': 8619, 'about': 413, 'entire': 3203, 'decade': 2501, 'of': 6051, 'first': 3619, 'eight': 3073, 'years': 9645, '1970': 104, 'through': 8802, '1977': 110, 'lower': 5297, 'employment': 3143, 'population': 6653, 'ratio': 7087, 'and': 733, 'labor': 4985, 'force': 3704, 'participation': 6338, 'rate': 7079, '2015': 153, 'when': 9510, 'obama': 6006, 'sworn': 8584, 'into': 4684, 'office': 6063, 'he': 4176, 'did': 2715, 'not': 5961, 'use': 9224, 'holy': 4301, 'bible': 1214, 'but': 1499, 'instead': 4631, 'kuran': 4975, 'their': 8745, 'equivalency': 3235, 'to': 8836, 'our': 6175, 'very': 9291, 'different': 2724, 'beliefs': 1167, 'ellison': 3108, 'used': 9225, 'koran': 4960, 'that': 8730, 'once': 6098, 'belonged': 1175, 'thomas': 8780, 'jefferson': 4801, 'borrowing': 1344, 'rare': 7076, 'book': 1325, 'fr[...]<jupyter_text>says': 9676, 'the': 10988, 'annies': 1044, 'list': 6639, 'political': 8376,<jupyter_code># summarize encoded vector
print(traindata_cvec.shape)
print(type(traindata_cvec))
print(traindata_cvec.toarray())
# Calculate inverse document frequencies
# create the transform
tfidf_vec = TfidfTransformer()

# tokenize, build vocab and encode training data
traindata_tfidf_vec = tfidf_vec.fit_transform(traindata_cvec)
# tfidf score
tfidf_vec.transform(traindata_cvec)
# summarize encoded vector
print(traindata_tfidf_vec.shape)
print(traindata_tfidf_vec.toarray())
# tfidf + ngrams
tfidf_vectorizer = TfidfVectorizer(stop_words='english', ngram_range=(1,2), use_idf=True, smooth_idf=True)
tfidf_vectorizer_vectors = tfidf_vectorizer.fit_transform(traindata['new'].values)
# print(tfidf_ngram.toarray())
# tfidf.vocabulary_
first_vector_tfidfvectorizer = tfidf_vectorizer_vectors[0]


# place tf-idf values in a pandas data frame
df = pd.DataFrame(first_vector_tfidfvectorizer.T.todense(), index=tfidf_vectorizer.get_feature_names(), columns=["tfidf"])
df.sort_values(by=["tfidf"],ascending=False)
 
nltk.download('treebank')
nltk.download('universal_tagset')
# POS tagging using CRF
tagged_sentence = nltk.corpus.treebank.tagged_sents(tagset='universal')
# print(len(tagged_sentence))
# load training sentences
training_sentences = traindata['new']
# print(training_sentences)
# function to extract features from dataset
def features(sentence, index):
  #returns a dictionary containing extracted features
  return {
      'is_first_capital': int(sentence[index][0].isupper()),
      'is_first_word': int(index==0),
      'is_last_word': int(index==len(sentence)-1),
      'is_complete_capital': int(sentence[index].upper()==sentence[index]),
      'is_complete_lower': int(sentence[index].lower()==sentence[index]),
      'prev_word':'' if index==0 else sentence[index-1],
      'next_word':'' if index==len(sentence)-1 else sentence[index+1],
      'is_numeric': int(sentence[index].isdigit()),
      'is_alphanumeric': int(bool((re.match('^(?=.*[0-9]$)(?=.*[a-zA-Z])',sentence[index])))),
      'prefix_1': sentence[index][0],
      'prefix_2': sentence[index][:2],
      'prefic_3': sentence[index][:3],
      'prefix_4': sentence[index][:4],
      'suffix_1': sentence[index][-1],
      'suffix_2': sentence[index][-2:],
      'suffix_3': sentence[index][-3:],
      'suffix_4': sentence[index][-4:],
      'word_has_hyphen': 1 if '-' in sentence[index] else 0
  }
# function to strip tags from tagged corpus
def untag(sentence):
  return [word for word,tag in sentence]
# # download GloVe word embeddings file
# !wget http://nlp.stanford.edu/data/glove.6B.zip
# from google.colab import drive
# drive.mount('/content/drive')

# extract the zip file at drive
# import zipfile
# zip_ref = zipfile.ZipFile("glove.6B.zip", 'r')
# zip_ref.extractall('content/drive/')
# zip_ref.close()
# load GloVe pretrained vectors from glove.6B.50d.txt
# with open("content/drive/glove.6B.50d.txt", "rb") as lines:
#   w2v = {line.split()[0]: np.array(map(float, line.split()[1:]))
#         for line in lines}
# print(len(w2v))
# w2v
# now we can use the above dictionary to build features. The simplest way to do that is by averaging word vectors for all words in a text.
# # I will build a sklearn compatible transformer that is initalised with a word-> vector dictionary
# class MeanEmbeddingVectorizer(object):
#   def __init__(self, word2vec):
#     self.word2vec = word2vec
#     # if a text is empty, return a vector of zeros with same dimensions
#     self.dim = len(word2vec.iteravalues.next())
    
  
#   def fit(self, X, y):
#     return self
  
#   def transform(self, X):
#     return np.array([
#         np.mean([self.word2vec[w] for w in words if w in self.word2vec]
#                 or [np.zeros(self.dim)], axis=0)
#         for words in X
#     ])<jupyter_output><empty_output><jupyter_text>**Task: Six-way classification** Training models with word-count-vectors from CountVectorizer method.<jupyter_code># using LogisticRegression
logReg_pipeline_cv = Pipeline([
    ('LogRCV', cvec),
    ('LogR_model', LogisticRegression())
])

logReg_pipeline_cv.fit(traindata['new'], traindata['label'])
predictions_logReg = logReg_pipeline_cv.predict(testdata['statement'])
logReg_cv = np.mean(predictions_logReg == testdata['label'])

# using SVM
svm_pipeline_cv = Pipeline([
    ('svmCV', cvec),
    ('svm_model', svm.LinearSVC())
])

svm_pipeline_cv.fit(traindata['new'], traindata['label'])
predictions_svm = svm_pipeline_cv.predict(testdata['statement'])
svm_cv = np.mean(predictions_svm == testdata['label'])

# using naive bayes
nb_pipeline_cv = Pipeline([
    ('nbCV', cvec),
    ('nb_model', MultinomialNB())
])

nb_pipeline_cv.fit(traindata['new'], traindata['label'])
predictions_nb = nb_pipeline_cv.predict(testdata['statement'])
nb_cv = np.mean(predictions_nb == testdata['label'])

# using random forest
rf_pipeline_cv = Pipeline([
    ('rfCV', cvec),
    ('rf_model', RandomForestClassifier(n_estimators=400, n_jobs=4))
])

rf_pipeline_cv.fit(traindata['new'], traindata['label'])
predictions_rf = rf_pipeline_cv.predict(testdata['statement'])
rf_cv = np.mean(predictions_rf == testdata['label'])
print(logReg_cv)
print(svm_cv)
print(nb_cv)
print(rf_cv)
def kFoldCV(model):
  k_fold = KFold(n_splits=5)
  scores = []
  conf_matrix = np.zeros(shape=(6,6))
  
  for train_index, test_index in k_fold.split(traindata['new']):
    trainText = traindata.iloc[train_index]['new']
    train_y = traindata.iloc[train_index]['label']
    
    testText = traindata.iloc[test_index]['new']
    test_y = traindata.iloc[test_index]['label']
    
    model.fit(trainText, train_y)
    pred_y = model.predict(testText)
    
    conf_matrix += confusion_matrix(test_y, pred_y)
    scores.append(f1_score(test_y, pred_y, average='micro'))
    
  return (print('Total statements classified:', len(traindata)),
         print('Score:', sum(scores)/len(scores)),
         print('Confusion Matrix:'),
         print(conf_matrix),
         print("---------------------------------------------------"))
kFoldCV(logReg_pipeline_cv)
kFoldCV(svm_pipeline_cv)
kFoldCV(nb_pipeline_cv)
kFoldCV(rf_pipeline_cv)<jupyter_output>Total statements classified: 1284
Score: 0.2679322714007782
Confusion Matrix:
[[69. 54. 38. 45.  9. 22.]
 [65. 64. 41. 49. 21. 23.]
 [41. 43. 82. 57.  4. 21.]
 [46. 42. 47. 81.  7. 28.]
 [17. 46. 11. 21. 12.  9.]
 [23. 44. 27. 35.  4. 36.]]
---------------------------------------------------
Total statements classified: 1284
Score: 0.25546267023346303
Confusion Matrix:
[[66. 50. 40. 43. 15. 23.]
 [69. 60. 38. 43. 26. 27.]
 [53. 39. 69. 54.  7. 26.]
 [46. 44. 45. 77. 11. 28.]
 [18. 40. 11. 23. 19.  5.]
 [25. 37. 24. 39.  7. 37.]]
---------------------------------------------------
Total statements classified: 1284
Score: 0.25624696011673154
Confusion Matrix:
[[ 44.  69.  66.  54.   0.   4.]
 [ 49.  91.  50.  70.   0.   3.]
 [ 31.  49.  80.  83.   0.   5.]
 [ 27.  49.  61. 109.   0.   5.]
 [ 11.  51.  26.  24.   1.   3.]
 [ 18.  34.  47.  65.   1.   4.]]
---------------------------------------------------
Total statements classified: 1284
Score: 0.28896522373540856
Confusion Matrix:
[[ 7[...]<jupyter_text>Training models with word-count-vectors from TfidfVectorizer method.<jupyter_code># using LogisticRegression
logReg_pipeline_ngram = Pipeline([
    ('LogR_tfidf', tfidf_vectorizer),
    ('LogR_model', LogisticRegression())
])

logReg_pipeline_ngram.fit(traindata['new'], traindata['label'])
predictions_logReg_ngram = logReg_pipeline_ngram.predict(testdata['statement'])
logReg_ngram = np.mean(predictions_logReg_ngram == testdata['label'])

# using SVM
svm_pipeline_ngram = Pipeline([
    ('svmCV', tfidf_vectorizer),
    ('svm_model', svm.LinearSVC())
])

svm_pipeline_ngram.fit(traindata['new'], traindata['label'])
predictions_svm = svm_pipeline_ngram.predict(testdata['statement'])
svm_ngram = np.mean(predictions_svm == testdata['label'])

# using naive bayes
nb_pipeline_ngram = Pipeline([
    ('nbCV', tfidf_vectorizer),
    ('nb_model', MultinomialNB())
])

nb_pipeline_ngram.fit(traindata['new'], traindata['label'])
predictions_nb = nb_pipeline_ngram.predict(testdata['statement'])
nb_ngram = np.mean(predictions_nb == testdata['label'])

# using random forest
rf_pipeline_ngram = Pipeline([
    ('rfCV', tfidf_vectorizer),
    ('rf_model', RandomForestClassifier(n_estimators=400, n_jobs=4))
])

rf_pipeline_ngram.fit(traindata['new'], traindata['label'])
predictions_rf = rf_pipeline_ngram.predict(testdata['statement'])
rf_ngram = np.mean(predictions_rf == testdata['label'])
print(logReg_ngram)
print(svm_ngram)
print(nb_ngram)
print(rf_ngram)
kFoldCV(logReg_pipeline_ngram)
kFoldCV(svm_pipeline_ngram)
kFoldCV(nb_pipeline_ngram)
kFoldCV(rf_pipeline_ngram)<jupyter_output>Total statements classified: 1284
Score: 0.26560372081712064
Confusion Matrix:
[[ 35. 102.  49.  50.   0.   1.]
 [ 32. 122.  36.  72.   0.   1.]
 [ 23.  66.  79.  79.   0.   1.]
 [ 20.  71.  56. 104.   0.   0.]
 [  8.  67.  13.  28.   0.   0.]
 [ 16.  48.  42.  62.   0.   1.]]
---------------------------------------------------
Total statements classified: 1284
Score: 0.2640381809338522
Confusion Matrix:
[[55. 67. 48. 44.  4. 19.]
 [50. 84. 37. 64.  6. 22.]
 [37. 43. 76. 71.  1. 20.]
 [28. 49. 55. 96.  2. 21.]
 [13. 44. 18. 25. 13.  3.]
 [28. 37. 41. 47.  1. 15.]]
---------------------------------------------------
Total statements classified: 1284
Score: 0.2648133511673152
Confusion Matrix:
[[ 22. 104.  56.  55.   0.   0.]
 [ 21. 134.  38.  70.   0.   0.]
 [ 16.  71.  82.  79.   0.   0.]
 [ 10.  81.  58. 102.   0.   0.]
 [  6.  72.  15.  23.   0.   0.]
 [ 10.  54.  41.  64.   0.   0.]]
---------------------------------------------------
Total statements classified: 1284
Score: 0.28272[...]<jupyter_text>**Task: Binary Classification**<jupyter_code># function to convert 6 classes of label into 2 classes
# def classLabel(dataset):
#   for index in range(len(dataset['label'])):
#     if(dataset.iloc[index]['label'] == 'true' or dataset.iloc[index]['label'] == 'mostly-true' or dataset.iloc[index]['label'] == 'half-true'):
#       dataset.iloc[index]['label'] = 'true'
      
#     if(dataset.iloc[index]['label'] == 'barely-true' or dataset.iloc[index]['label'] == 'false' or dataset.iloc[index]['label'] == 'pants-fire'):
#       dataset.iloc[index]['label'] = 'false'
      
#   return dataset
# load dataset containing only two classes of label
new_train = pd.read_csv('newtrain.csv')
new_test = pd.read_csv('newtest.csv')
new_val = pd.read_csv('newvalid.csv')
new_test.head()
# concatenate validation set with training dataset
new_train = pd.concat([new_val])

# drop Null rows
new_train = new_train.dropna()
new_train.head()<jupyter_output><empty_output><jupyter_text>**Feature extraction (vectorization) :** Build Bag-Of-Words model using CountVectorizer and TfidfVectorizer.<jupyter_code># Convert text to word count vectors with CountVectorizer
# create the transform
cvec = CountVectorizer()

# tokenize, build vocab and encode training data
traindata_cvec = cvec.fit_transform(new_train['Statement'].values)

# summarize
print(cvec.vocabulary_)
# print(cvec.get_feature_names())
# Calculate inverse document frequencies
# create the transform
tfidf_vec = TfidfTransformer()

# tokenize, build vocab and encode training data
traindata_tfidf_vec = tfidf_vec.fit_transform(traindata_cvec)
# tfidf score
tfidf_vec.transform(traindata_cvec)
# tfidf + ngrams
tfidf_vectorizer = TfidfVectorizer(use_idf=True)
tfidf_vectorizer_vectors = tfidf_vectorizer.fit_transform(new_train['Statement'].values)
# using LogisticRegression
logReg_pipeline_cv = Pipeline([
    ('LogRCV', cvec),
    ('LogR_model', LogisticRegression())
])

logReg_pipeline_cv.fit(new_train['Statement'], new_train['Label'])
predictions_logReg = logReg_pipeline_cv.predict(new_test['Statement'])
logReg_cv = np.mean(predictions_logReg == new_test['Label'])

# using SVM
svm_pipeline_cv = Pipeline([
    ('svmCV', cvec),
    ('svm_model', svm.LinearSVC())
])

svm_pipeline_cv.fit(new_train['Statement'], new_train['Label'])
predictions_svm = svm_pipeline_cv.predict(new_test['Statement'])
svm_cv = np.mean(predictions_svm == new_test['Label'])

# using naive bayes
nb_pipeline_cv = Pipeline([
    ('nbCV', cvec),
    ('nb_model', MultinomialNB())
])

nb_pipeline_cv.fit(new_train['Statement'], new_train['Label'])
predictions_nb = nb_pipeline_cv.predict(new_test['Statement'])
nb_cv = np.mean(predictions_nb == new_test['Label'])

# using random forest
rf_pipeline_cv = Pipeline([
    ('rfCV', cvec),
    ('rf_model', RandomForestClassifier(n_estimators=400, n_jobs=4))
])

rf_pipeline_cv.fit(new_train['Statement'], new_train['Label'])
predictions_rf = rf_pipeline_cv.predict(new_test['Statement'])
rf_cv = np.mean(predictions_rf == new_test['Label'])
print(logReg_cv)
print(svm_cv)
print(nb_cv)
print(rf_cv)
# new_predictions_logReg
# using LogisticRegression
logReg_pipeline_cv = Pipeline([
    ('LogRCV', tfidf_vectorizer),
    ('LogR_model', LogisticRegression())
])

logReg_pipeline_cv.fit(new_train['Statement'], new_train['Label'])
predictions_logReg = logReg_pipeline_cv.predict(new_test['Statement'])
logReg_ngram = np.mean(predictions_logReg == new_test['Label'])

# using SVM
svm_pipeline_cv = Pipeline([
    ('svmCV', tfidf_vectorizer),
    ('svm_model', svm.LinearSVC())
])

svm_pipeline_cv.fit(new_train['Statement'], new_train['Label'])
predictions_svm = svm_pipeline_cv.predict(new_test['Statement'])
svm_ngram = np.mean(predictions_svm == new_test['Label'])

# using naive bayes
nb_pipeline_cv = Pipeline([
    ('nbCV', tfidf_vectorizer),
    ('nb_model', MultinomialNB())
])

nb_pipeline_cv.fit(new_train['Statement'], new_train['Label'])
predictions_nb = nb_pipeline_cv.predict(new_test['Statement'])
nb_ngram = np.mean(predictions_nb == new_test['Label'])

# using random forest
rf_pipeline_cv = Pipeline([
    ('rfCV', tfidf_vectorizer),
    ('rf_model', RandomForestClassifier(n_estimators=400, n_jobs=4))
])

rf_pipeline_cv.fit(new_train['Statement'], new_train['Label'])
predictions_rf = rf_pipeline_cv.predict(new_test['Statement'])
rf_ngram = np.mean(predictions_rf == new_test['Label'])
print(logReg_ngram)
print(svm_ngram)
print(nb_ngram)
print(rf_ngram)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17836776074008365, 'fraction_numerical': 0.057850716005575974, 'mean_word_length': 4.446169772256729, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '265287', 'license': 'Apache-2.0', 'n_tokens_mistral': 6417, 'n_tokens_neox': 5798, 'n_words': 1205, 'path': 'Aditya Rakhecha/fakeNewsDetector.ipynb', 'repo_name': 'bharatnishant/UdacityOpenSource', 'size': '15782'}