<jupyter_start><jupyter_text>First layer irrelevant prediction<jupyter_code>df_my_train['Irrelevant'] = df_my_train['topic']=='IRRELEVANT'
df_my_test['Irrelevant'] = df_my_test['topic']=='IRRELEVANT'
svm_model = svm.SVC( probability = True)
new_dataset = df_my_train
new_dataset = new_dataset.replace(np.nan, '', regex=True)
kw_10_wc =  CountVectorizer().fit_transform(new_dataset.key_word_10)
kw_10_tf = TfidfVectorizer().fit_transform(new_dataset.key_word_10)

kw_20_wc =  CountVectorizer().fit_transform(new_dataset.key_word_20)
kw_20_tf = TfidfVectorizer().fit_transform(new_dataset.key_word_20)

kw_50_wc =  CountVectorizer().fit_transform(new_dataset.key_word_50)
kw_50_tf = TfidfVectorizer().fit_transform(new_dataset.key_word_50)

kw_100_wc =  CountVectorizer().fit_transform(new_dataset.key_word_100)
kw_100_tf = TfidfVectorizer().fit_transform(new_dataset.key_word_100)

full_wc = CountVectorizer().fit_transform(new_dataset.article_words)
full_tf = TfidfVectorizer().fit_transform(new_dataset.article_words)
accuracy_list = []
x_train_list = [ kw_10_wc, kw_10_tf, kw_20_wc, kw_20_tf,kw_50_wc,kw_50_tf,kw_100_wc,kw_100_tf,full_wc, full_tf  ]
x_train_name = [ 'kw_10_wc', 'kw_10_tf', 
                'kw_20_wc', 'kw_20_tf','kw_50_wc','kw_50_tf','kw_100_wc','kw_100_tf','full_wc','full_tf']
for i in range(len(x_train_list)):
    accs = cross_val_score(svm_model, 
                           x_train_list[i], df_my_train.Irrelevant, cv=5)
    print(x_train_name[i])
    print(f"Mean accuracy of (std): {np.mean(accs): .3f} ({np.std(accs): .3f})")
    accuracy_list.append(np.mean(accs))<jupyter_output>kw_10_wc
Mean accuracy of (std):  0.821 ( 0.004)
kw_10_tf
Mean accuracy of (std):  0.838 ( 0.006)
kw_20_wc
Mean accuracy of (std):  0.835 ( 0.008)
kw_20_tf
Mean accuracy of (std):  0.859 ( 0.008)
kw_50_wc
Mean accuracy of (std):  0.849 ( 0.005)
kw_50_tf
Mean accuracy of (std):  0.867 ( 0.006)
kw_100_wc
Mean accuracy of (std):  0.855 ( 0.005)
kw_100_tf
Mean accuracy of (std):  0.870 ( 0.005)
full_wc
Mean accuracy of (std):  0.862 ( 0.001)
full_tf
Mean accuracy of (std):  0.870 ( 0.005)<jupyter_text>We pick the best one in Cross-validation to train the first layer model kw_100_tf<jupyter_code>svm_ir = svm_model.fit(kw_100_tf, df_my_train.Irrelevant)
from joblib import dump, load
def save_variable(variable,filename):
    file_name = "Models/"+filename+".joblib"
    if not os.path.exists(file_name):
    # Export the model (TFIDF+logistic regressor)
        with open(file_name, "wb") as file:
            dump(variable, file, compress=True)
    else:
        print("Warn: this model already exits...")

save_variable(svm_ir,'svm_ir_clf')<jupyter_output><empty_output><jupyter_text>Check its performance in test set (this part should not be included in assessmble codes)<jupyter_code>y_test_true= df_my_test['Irrelevant']

train_df = [kw_50_tf,kw_100_tf,full_tf ]

test_df = [TfidfVectorizer().fit(new_dataset.key_word_50).transform(df_my_test.key_word_50), 
          TfidfVectorizer().fit(new_dataset.key_word_100).transform(df_my_test.key_word_100),
          TfidfVectorizer().fit(new_dataset.article_words).transform(df_my_test.article_words)]

for i in range(3):
    svm_ir = svm_model.fit(train_df[i],df_my_train.Irrelevant)
    y_pred = svm_ir.predict(test_df[i])
    print(accuracy_score(y_test_true,y_pred))
    

TfidfVectorizer().fit(new_dataset.key_word_100).transform(df_my_test.key_word_100)<jupyter_output><empty_output><jupyter_text>Junk for testing<jupyter_code>import warnings
warnings.filterwarnings("ignore")
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
sns.set_style("whitegrid")
#import altair as alt
# need to 'pip install vega' before using renderer
#alt.renderers.enable("notebook")
# Code for hiding seaborn warnings
import warnings
warnings.filterwarnings("ignore")
from imblearn.over_sampling import SMOTE
from sklearn import preprocessing,metrics
from sklearn.metrics import accuracy_score,f1_score,recall_score,roc_auc_score
from sklearn.feature_extraction.text import TfidfVectorizer,CountVectorizer
from sklearn.model_selection import train_test_split, GridSearchCV, learning_curve,StratifiedKFold, cross_val_score
from sklearn import svm
from sklearn.pipeline import make_pipeline, Pipeline
from sklearn.decomposition import PCA,TruncatedSVD
import scipy
import os
from joblib import dump, load
from sklearn.metrics import classification_report,confusion_matrix,plot_confusion_matrix
from sklearn.metrics import precision_recall_fscore_support as prfs
# test for Selecting The Best Number Of Components For TSVD
from sklearn.preprocessing import StandardScaler
from scipy.sparse import csr_matrix
#from varname import varname
%matplotlib inline
#fix random seed
np.random.seed(1)
TRAINING_FILE = "../keyword.csv"
TEST_FILE = "../key_word_test.csv"
df_train = pd.read_csv(TRAINING_FILE)
df_test = pd.read_csv(TEST_FILE)
print(df_train.isnull().sum())
df_train.head(5)
new_dataset = df_train.replace(np.nan, '', regex=True)
junk_train = df_train.rename(columns={ "article_words": "key_word"})

new_df = junk_train
new_df = new_df.dropna()

train_x = new_df.key_word.apply(lambda x: x.replace('_', ''))

the_vect = TfidfVectorizer().fit(train_x)
df_train.shape
junk_train = df_train.rename(columns={ "article_words": "key_word"})
new_df = junk_train
new_df = new_df.dropna()
new_df.shape
train_x = new_df.key_word.apply(lambda x: x.replace('_', ''))
train_x.shape
 TfidfVectorizer().fit_transform(junk_train.key_word)
TRAINING_FILE = "../keyword.csv"
TEST_FILE = "../key_word_test.csv"
df_train = pd.read_csv(TRAINING_FILE)
df_test = pd.read_csv(TEST_FILE)

junk_train = df_train.rename(columns={ "article_words": "key_word"})
new_df = junk_train
new_df = new_df.dropna()
train_x = new_df.key_word.apply(lambda x: x.replace('_', ''))
the_vect = TfidfVectorizer().fit(train_x)

test_x_full = the_vect.transform(df_test.article_words)
test_x_100 = the_vect.transform(df_test.key_word_100)
print(test_x_full.shape)
print(test_x_100.shape)
the_vect
train_x.shape
test_x_full = the_vect.transform(df_test.article_words)
test_x_full.shape
df_test.tail()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1148364107547781, 'fraction_numerical': 0.034499514091350825, 'mean_word_length': 6.521315468940316, 'pattern_counts': {'":': 3, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '143931', 'license': 'Apache-2.0', 'n_tokens_mistral': 2502, 'n_tokens_neox': 2349, 'n_words': 416, 'path': "Yangqi's work/Yangqi_test_results.ipynb", 'repo_name': 'HenryLiangzy/COMP9417_Project', 'size': '6174'}