<jupyter_start><jupyter_text>Libs<jupyter_code>#Libs ############################################
import pandas as pd
import numpy as np
import re
import unicodedata
import pickle
import nltk
import os

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.multiclass import OneVsRestClassifier
from sklearn.metrics import classification_report, accuracy_score
from sklearn.pipeline import Pipeline
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import GridSearchCV

nltk.download('stopwords')
from nltk.corpus import stopwords
stop_words = set(stopwords.words('portuguese'))<jupyter_output>[nltk_data] Downloading package stopwords to /home/jovyan/nltk_data...
[nltk_data]   Package stopwords is already up-to-date!<jupyter_text>Data extractionLoads a dataset with product data from a specified path available in the environment variable DATASET_PATH.<jupyter_code>try:
    DATASET_PATH = os.environ['DATASET_PATH']
except:
    DATASET_PATH = 'data/sample_products.csv'

df_file = pd.read_csv(DATASET_PATH, sep=',')
df_file.head(2)
df_file.groupby(['category']).size()<jupyter_output><empty_output><jupyter_text>Data formattingProcesses the dataset to use it for training and validation.<jupyter_code>#copy dataset
df_data = df_file.copy()
df_data.isnull().sum()


#remove duplicate itens and especial caracters
def dedup_esp_caract(txt):
    x = str(txt)
    nfkd = unicodedata.normalize('NFKD', x)
    palavraSemAcento = u"".join([c for c in nfkd if not unicodedata.combining(c)])
    x = re.sub('[^a-zA-Z0-9 \\\]','', palavraSemAcento)
    x = x.split(' ')
    x = ' '.join([str(elem) for elem in set(x)])
    return x
#Data Aug for Outros/Papel e Cia categories
df_tmp = df_file[df_file['category']=='Outros']
df_tmp = df_tmp.loc[df_tmp.index.repeat(3)]
df_data = df_data.append(df_tmp)

df_tmp = df_file[df_file['category']=='Papel e Cia']
df_tmp = df_tmp.loc[df_tmp.index.repeat(2)]
df_data = df_data.append(df_tmp)
#sizing after data aug
df_data.groupby(['category']).size()
#Label encoder
LE = LabelEncoder()
df_data['cod_category'] = LE.fit_transform(df_data['category'])

#generate a text column combining query + title + concatenated_tags
df_data['text_train'] = df_data['query'].str.lower() +' ' + df_data['title'].str.lower() + ' ' + df_data['concatenated_tags'].str.lower()
df_data['text_train'] = df_data.apply(lambda row: dedup_esp_caract(row['text_train']), axis=1)<jupyter_output><empty_output><jupyter_text>ModelingSpecifies a model to handle the categorization problem.<jupyter_code>#Split dataset into train and test
train, test = train_test_split(df_data, random_state=42, test_size=0.33, shuffle=True)
X_train = train.text_train
X_test = test.text_train
y_train = train['cod_category']
y_test = test['cod_category']
print(X_train.shape)
print(X_test.shape)
#Logistic Regression #########
LR_Pipe = Pipeline([
                ('tfidf', TfidfVectorizer(stop_words=stop_words)),
                ('clf', OneVsRestClassifier(LogisticRegression(solver='sag'), n_jobs=1))
           ])
LR_Pipe.fit(X_train.values.astype('U'), y_train)
y_pred = LR_Pipe.predict(X_test.values.astype('U'))

print(classification_report(y_test,y_pred))
print(accuracy_score(y_test, y_pred))
#Random Forest ###############
RF_Pipe = Pipeline([
                ('tfidf', TfidfVectorizer(stop_words=stop_words)),
                ('RF', RandomForestClassifier(n_estimators=200,random_state=2)),
           ])


RF_Pipe.fit(X_train.values.astype('U'), y_train)
y_pred2 = RF_Pipe.predict(X_test.values.astype('U'))

print(classification_report(y_test,y_pred2))
print(accuracy_score(y_test, y_pred2))
#Decision Tree ###############
DT_Pipe = Pipeline([
                ('tfidf', TfidfVectorizer(stop_words=stop_words)),
                ('DT', DecisionTreeClassifier(criterion="entropy",max_depth=60)),
           ])


DT_Pipe.fit(X_train.values.astype('U'), y_train)
y_pred3 = DT_Pipe.predict(X_test.values.astype('U'))

print(classification_report(y_test,y_pred3))
print(accuracy_score(y_test, y_pred3))
#Choose the best model by acc
best_model = []
models = {'RF_Pipe' : accuracy_score(y_test, y_pred2),'LR_Pipe' : accuracy_score(y_test, y_pred), 'DT_Pipe' : accuracy_score(y_test, y_pred3)}
models = {k: v for k, v in sorted(models.items(), key=lambda item: item[1])}
best_model = list(models.keys())[-1] #get best model by last position
print('\nModels:', models)
print('\nBest Model Selected:',best_model)<jupyter_output>Models: {'DT_Pipe': 0.8091361374943464, 'LR_Pipe': 0.8807262389351942, 'RF_Pipe': 0.9342249790010984}

Best Model Selected: RF_Pipe<jupyter_text>Model validationGenerates metrics about the model accuracy (precision, recall, F1, etc.) for each category and exports them to a specified path available in the environment variable METRICS_PATH.<jupyter_code>try:
    METRICS_PATH = os.environ['METRICS_PATH']
except:
    METRICS_PATH = 'metrics.txt'
#generate a categories list
df_categorias = df_data[['cod_category', 'category']].copy()
df_categorias = df_categorias.drop_duplicates()
categories = df_categorias['category'].tolist()

#predict function to each category and save it to file
def test_predict(Model, Save, txt):    
    metric_file =''
    metric_file += '####################################\n'
    metric_file += txt + '\n'
    metric_file += '####################################\n'
        
    for cat in categories:    
        x_cat_test = test['text_train'][test['category']==cat]
        y_cat_test = test['cod_category'][test['category']==cat]
        y_pred = Model.predict(x_cat_test)
        print('ACC: ',cat, accuracy_score(y_cat_test, y_pred))        
        metric_file += 'Category: ' + cat + ' - ACC:' + str(accuracy_score(y_cat_test, y_pred))  + '\n'
        metric_file += str(classification_report(y_cat_test,y_pred)) + '\n'
        print(classification_report(y_cat_test,y_pred))
    
    if Save == 1:                        
        f = open(METRICS_PATH, "a")        
        f.truncate(0)
        f.write(metric_file)
        f.close()            
#choose the best model and call test_predict function(above)
if best_model == 'RF_Pipe':
    print('############### Random Forest Metrics for categories #################')
    test_predict(RF_Pipe, 1, 'Random Forest - Metrics for categories')
elif best_model == 'DT_Pipe':
    print('############### Logistic Regression Metrics for categories #################')
    test_predict(DT_Pipe, 1, 'Decision Tree - Metrics for categories')
elif best_model == 'LR_Pipe':
    print('############### Logistic Regression Metrics for categories #################')
    test_predict(LR_Pipe, 1, 'Logistic Regression - Metrics for categories')<jupyter_output>############### Random Forest Metrics for categories #################
ACC:  Decoração 0.9000344708721131<jupyter_text>Model exportationExports a candidate model to a specified path available in the environment variable MODEL_PATH.<jupyter_code>#save best model as pickle file
def save_model(model):
    try:
        MODEL_PATH = os.environ['MODEL_PATH']
    except:
        MODEL_PATH = 'model.pkl'
    with open(MODEL_PATH, 'wb') as picklefile:
        pickle.dump(model,picklefile)
    
#choose the best model and call save_model function   
if best_model == 'RF_Pipe':
    save_model(RF_Pipe)
elif best_model == 'DT_Pipe':
    save_model(LR_Pipe)
elif best_model == 'LR_Pipe':
    save_model(LR_Pipe)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1528287061288633, 'fraction_numerical': 0.013357778941854375, 'mean_word_length': 5.254709254709255, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '417320', 'license': 'MIT', 'n_tokens_mistral': 2689, 'n_tokens_neox': 2592, 'n_words': 541, 'path': 'training/trainer.ipynb', 'repo_name': 'rsilveira7/intelligent-systems-project', 'size': '7636'}