<jupyter_start><jupyter_text>Splitting into Train and Test<jupyter_code>restart_here = final.copy()
final = restart_here

# Splitting the dataset into Train, Test.
# Let's go for an 80 - 20 split.
labels = final["propaganda"]
finals = final.drop(columns=["propaganda"], axis=1, inplace=False)

from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(finals, labels, test_size=0.2, random_state=42)
print(len(X_train))
print(len(X_test))
finals.head()<jupyter_output><empty_output><jupyter_text>BAG OF WORDS<jupyter_code>from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.feature_extraction.text import TfidfVectorizer

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics import confusion_matrix
from sklearn import metrics
from sklearn.metrics import roc_curve, auc
from nltk.stem.porter import PorterStemmer

#BoW
count_vect = CountVectorizer() #in scikit-learn
count_vect.fit(preprocessed_text)
print("some feature names ", count_vect.get_feature_names()[:10])
print('='*50)

final_counts = count_vect.transform(preprocessed_text)
print("the type of count vectorizer ",type(final_counts))
print("the shape of out text BOW vectorizer ",final_counts.get_shape())
print("the number of unique words ", final_counts.get_shape()[1])
from sklearn.neighbors import KNeighborsClassifier

# Please write all the code with proper documentation
def bag_of_words(train_data, test_data, validation_data = None):
    count_vect = CountVectorizer() #in scikit-learn
    count_vect.fit(train_data)
    print("some feature names ", count_vect.get_feature_names()[:10])
    print('='*50)
    train_data_vectorized = count_vect.transform(train_data)
    test_data_vectorized = count_vect.transform(test_data)
    if validation_data:
        validation_data_vectorized = count_vect.tranform(validation_data)
    else:
        validation_data_vectorized = None
    print("the type of count vectorizer ",type(train_data_vectorized))
    print("the shape of out text BOW vectorizer ",train_data_vectorized.get_shape())
    print("the number of unique words ", train_data_vectorized.get_shape()[1])
    return train_data_vectorized, test_data_vectorized
train_data_vectorized , test_data_vectorized = bag_of_words(X_train["text"], X_test["text"])
from sklearn.model_selection import GridSearchCV

knn = KNeighborsClassifier()
n_list = range(1,10,2)
parameters = {"n_neighbors":n_list}
clf = GridSearchCV(knn, parameters, cv=3, verbose=2, n_jobs=2)
clf.fit(train_data_vectorized,y_train)
clf.best_params_
clf2 = KNeighborsClassifier(n_neighbors=7)
clf2.fit(train_data_vectorized,y_train)
clf2.score(test_data_vectorized,y_test)

y_pred = clf2.predict(test_data_vectorized)
y_true = y_test
from sklearn.metrics import f1_score
from sklearn.metrics import precision_score
from sklearn.metrics import recall_score

print(f1_score(y_true, y_pred))
print(recall_score(y_true, y_pred))
print(precision_score(y_true, y_pred))

# >>> y_true = [0, 1, 2, 0, 1, 2]
# >>> y_pred = [0, 2, 1, 0, 0, 1]
# >>> f1_score(y_true, y_pred, average='macro')  
# test_data_vectorized_dense = test_data_vectorized.toarray()
# train_data_vectorized_dense = train_data_vectorized.toarray()
# Using MultiNomial Naive Bayes
from sklearn.naive_bayes import MultinomialNB

mnb = MultinomialNB()
mnb.fit(train_data_vectorized,y_train)
mnb_pred = mnb.predict(test_data_vectorized)
from sklearn.metrics import f1_score
from sklearn.metrics import precision_score
from sklearn.metrics import recall_score

print(f1_score(y_true, mnb_pred))
print(recall_score(y_true, mnb_pred))
print(precision_score(y_true, mnb_pred))

# >>> y_true = [0, 1, 2, 0, 1, 2]
# >>> y_pred = [0, 2, 1, 0, 0, 1]
# >>> f1_score(y_true, y_pred, average='macro')  
from sklearn import svm, datasets
>>> from sklearn.model_selection import GridSearchCV
>>> iris = datasets.load_iris()
>>> parameters = {'kernel':('linear', 'rbf'), 'C':[1, 10]}
>>> svc = svm.SVC(gamma="scale")
>>> clf = GridSearchCV(svc, parameters, cv=5)
>>> clf.fit(iris.data, iris.target)
...                             
GridSearchCV(cv=5, error_score=...,
       estimator=SVC(C=1.0, cache_size=..., class_weight=..., coef0=...,
                     decision_function_shape='ovr', degree=..., gamma=...,
                     kernel='rbf', max_iter=-1, probability=False,
                     random_state=None, shrinking=True, tol=...,
                     verbose=False),
       fit_params=None, iid=..., n_jobs=None,
       param_grid=..., pre_dispatch=..., refit=..., return_train_score=...,
       scoring=..., verbose=...)
>>> sorted(clf.cv_results_.keys())
...                             
['mean_fit_time', 'mean_score_time', 'mean_test_score',...
 'mean_train_score', 'param_C', 'param_kernel', 'params',...
 'rank_test_score', 'split0_test_score',...
 'split0_train_score', 'split1_test_score', 'split1_train_score',...
 'split2_test_score', 'split2_train_score',...
 'std_fit_time', 'std_score_time', 'std_test_score', 'std_train_score'...]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1290704558910598, 'fraction_numerical': 0.014999013222814288, 'mean_word_length': 5.342928660826033, 'pattern_counts': {'":': 1, '<': 9, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '325657', 'license': 'MIT', 'n_tokens_mistral': 1836, 'n_tokens_neox': 1762, 'n_words': 351, 'path': 'Datathon.ipynb', 'repo_name': 'ooeyad/datathonstepup', 'size': '5067'}