<jupyter_start><jupyter_text>One-vs-all classifier This notebook provides the code to train a set of one-vs-all classifiers and compare their performance against each other. We choose a simple pipelie consisting of tf-idf static embeddings, lsa feature reduction and probabilistic modelling approaches.<jupyter_code>import os
import sys
sys.path.insert(0, os.path.abspath(os.path.join('..', 'src')))
sys.setrecursionlimit(20500)
import pandas as pd
import category_encoders as ce

import tools as tools

'''features'''
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import TruncatedSVD
from sklearn.preprocessing import label_binarize

'''Classifiers'''
from sklearn.dummy import DummyClassifier
from sklearn.linear_model import SGDClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.multiclass import OneVsRestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn import svm
from sklearn.pipeline import Pipeline

'''Metrics/Evaluation'''
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_curve, auc, confusion_matrix
from scipy import interp
from itertools import cycle
from sklearn.model_selection import cross_val_score
from sklearn.metrics import classification_report
from sklearn.pipeline import make_pipeline

import joblib
'''import cleaned data'''
df = pd.read_csv(os.path.abspath(os.path.join('..', 'data/processed/'))+'/data_processed.csv')
'''one-hot-encode labels'''
df['sdg'] = df['sdg'].astype(str)
encoder = ce.OneHotEncoder(cols=['sdg'], use_cat_names=True)
df = encoder.fit_transform(df)
df.head(5)
'''sort column names'''
df = df[list(sorted(df.columns.tolist()))]
categories = [
       'sdg_1', 'sdg_2','sdg_3', 'sdg_4', 'sdg_5',
       'sdg_6', 'sdg_7', 'sdg_8','sdg_9', 'sdg_10', 
       'sdg_11', 'sdg_12','sdg_13', 'sdg_14', 'sdg_15']
df[categories].sum(axis=0).plot.bar()<jupyter_output><empty_output><jupyter_text>Let's try the same without dimension reduction<jupyter_code>categories_used = []
scores = []
# iterate over categories and save best models:
    #Creating a dict of the models
model_dict = {
              'LR_balanced' : LogisticRegression(random_state = 3, class_weight = "balanced"),
              'LR' : LogisticRegression(random_state = 3),
              'SDG' : SGDClassifier(random_state=3, loss='log'),
              'SDG_balanceed' : SGDClassifier(random_state=3, loss='log',class_weight = "balanced" ),
              'RF': RandomForestClassifier(random_state=3),
              'Decsision_Tree': DecisionTreeClassifier(random_state=3),
              'AdaBoost': AdaBoostClassifier(random_state=3),
              'KNB': KNeighborsClassifier()
                }

for category in categories: 

    print('processing:', category)
    
    X_train, X_test, y_train, y_test = train_test_split(df['text_clean_spacy'],                                                    
                                            df[category],
                                            test_size = .3,
                                            random_state = 1,
                                            stratify = df[category].values, 
                                            shuffle = True)

    tfidf_vectorizer = TfidfVectorizer(ngram_range=(1, 2), 
                                        min_df = 2, 
                                        max_df = .95)
    
    vect = tfidf_vectorizer.fit(X_train)
    X_train = vect.transform(X_train).toarray()
    X_test = vect.transform(X_test).toarray()


    #save vectorizer:
    filename = '../models/tf_idf/tf_idf_only/'+category+'_'+'vectorizer.sav'
    joblib.dump(vect, filename)    


    results, score = tools.model_score_df_all(model_dict, category, 'tf_idf_only', X_train, X_test, y_train, y_test)
    
    
    categories_used.append(category)
    scores.append(score)

    print('_____')
info_dict = dict(zip(categories_used, scores))

with open(os.path.abspath(os.path.join('..', 'data/processed/'))+"/tfidf_only_f1.pkl", 'wb') as handle:
    pickle.dump(info_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)<jupyter_output><empty_output><jupyter_text>Let's try to build set of classifiers using tf-idf and lsa dimension reduction<jupyter_code># iterate over categories and save best models:
    #Creating a dict of the models
model_dict = {
              'LR_balanced' : LogisticRegression(random_state = 3, class_weight = "balanced"),
              'LR' : LogisticRegression(random_state = 3),
              'SDG' : SGDClassifier(random_state=3, loss='log'),
              'SDG_balanceed' : SGDClassifier(random_state=3, loss='log',class_weight = "balanced" ),
              'RF': RandomForestClassifier(random_state=3),
              'Decsision_Tree': DecisionTreeClassifier(random_state=3),
              'AdaBoost': AdaBoostClassifier(random_state=3),
              'GNB': GaussianNB(),
              'KNB': KNeighborsClassifier()}
    
for category in categories: 
   
    print('processing:', category)
    #Creating the features (tf-idf weights) for the processed text
    texts = df['text_clean_spacy'].astype('str')

    tfidf_vectorizer = TfidfVectorizer(ngram_range=(1, 2), 
                                    min_df = 2, 
                                    max_df = .95)
    X = tfidf_vectorizer.fit_transform(texts)     
    y = df[category].values


    filename = '../models/tf_idf/tf_idf_lsa/'+category+'_'+'vectorizer.sav'
    joblib.dump(tfidf_vectorizer, filename)    

    #Dimenionality reduction. Only using the 100 best features er category
    lsa = TruncatedSVD(n_components=100, 
                    n_iter=10, 
                    random_state=3)
    X = lsa.fit_transform(X)


    #save vectorizer:
    filename = '../models/tf_idf/tf_idf_lsa/'+category+'_'+'lsa.sav'
    joblib.dump(lsa, filename)  

    #Train test split with stratified sampling for evaluation
    X_train, X_test, y_train, y_test = train_test_split(X, 
                                                        y, 
                                                        test_size = .3, 
                                                        shuffle = True, 
                                                        stratify = y, 
                                                        random_state = 3)

    tools.model_score_df_all(model_dict, category, 'tf_idf_lsa', X_train, X_test, y_train, y_test)
    
    print('_____')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10217903415783275, 'fraction_numerical': 0.010011778563015312, 'mean_word_length': 2.6619946091644207, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '653102', 'license': 'MIT', 'n_tokens_mistral': 2122, 'n_tokens_neox': 2002, 'n_words': 434, 'path': 'notebooks/One-vs-All-Classifier.ipynb', 'repo_name': 'jonas-nothnagel/sdg_text_classification', 'size': '6792'}