<jupyter_start><jupyter_text>0.0. Planejamento  Input Dados: * **Id**: Identificador Único;* **Title**: Título da vaga;* **FullDescription**: Texto completo com a descrição da vaga;* **LocationRaw**: Texto cru da localizacao da vaga;* **LocationNormalized**: Localizacao normalizada - **Carrega erro**;* **ContractType**: Tipo de contrato. Indica como é o regime de horas da vaga. Pode ser:    * full_time;    * part_time.* **ContractTime**: Tempo de validade do contrato. Pode ser:    * permanent;    * contract.* **Company**: Nome da compania que está contratanto;* **Category**: 30 categorias de trabalho - **Carrega erro**;* **SalaryNormalised**: Salario anualizado da vaga. O que estamos buscando prever;* **SourceName**: Nome to site o qual recebemos a vaga. Problema:* Temos um conjunto de dados com as informacoes de vagas abertas. Com base nessas \informacoes, devemos criar um algoritmo capaz de prever o salário em novas vagas. Output* **Modelo**: Um modelo capaz de prever o salario de novos usuarios;* **WebApp**: Um webapp que carregue esse modelo e faça previsões;* **EDA**: Uma Análise Exploratória dos Dados e do Modelo:    - Relatório do Modelo;    - Relatório do Projeto. Tasks1. Planejamento;2. Ciclos;3. Construcao do Modelo Final;    - Métrica Principal: Mean Absolute Error (MAE)4. Desenvolvimento do WebApp;5. Deploy;  1.0. Imports<jupyter_code># Data Manipulation
import pandas as pd
import numpy as np

# Data Plot
import seaborn as sns
from matplotlib import pyplot as plt
from IPython.display import HTML

# Model
from sklearn.preprocessing import OneHotEncoder
from sklearn.compose import ColumnTransformer

from sklearn.ensemble import RandomForestRegressor
from sklearn.linear_model import LinearRegression
from sklearn.tree import DecisionTreeRegressor
from sklearn.gaussian_process import GaussianProcessRegressor

from sklearn.model_selection import train_test_split, cross_validate, KFold
from sklearn.metrics import mean_absolute_error, mean_absolute_percentage_error, mean_squared_error

from xgboost import XGBRegressor

# NLP
import re
import nltk
from nltk.corpus import stopwords
from nltk.stem.wordnet import WordNetLemmatizer
from nltk.tokenize import word_tokenize

nltk.download('punkt')
nltk.download('stopwords')
nltk.download('wordnet')

# stats
from scipy.stats import pointbiserialr<jupyter_output>[nltk_data] Downloading package punkt to /home/tadeu/nltk_data...
[nltk_data]   Package punkt is already up-to-date!
[nltk_data] Downloading package stopwords to /home/tadeu/nltk_data...
[nltk_data]   Package stopwords is already up-to-date!
[nltk_data] Downloading package wordnet to /home/tadeu/nltk_data...
[nltk_data]   Package wordnet is already up-to-date!<jupyter_text>1.1. Helper Functions<jupyter_code>def jupyter_settings():
    """
    
    """
    %matplotlib inline
    %pylab inline
    
    plt.style.use('ggplot')
    plt.rcParams['figure.figsize'] = [24, 9]
    plt.rcParams['font.size'] = 24
    
    display(HTML('<style>.container{width:100% !important;}</style>'))
    pd.options.display.max_columns = None
    pd.options.display.max_rows = None
    pd.set_option('display.expand_frame_repr', False)
    
    sns.set()
    
jupyter_settings()


def cross_val_performance(X_train, y_train, model, cv):
    
    # Cross Val Train
    results = cross_validate(model, 
                             X_train, 
                             y_train, 
                             cv=cv,
                             scoring=[
                                 'neg_mean_absolute_error', 
                                 'neg_mean_absolute_percentage_error', 
                                 'neg_root_mean_squared_error'
                             ]
                            )
    
    # Performance
    mae = round(np.mean(results['test_neg_mean_absolute_error'] * -1), 2)
    mape = round(np.mean(results['test_neg_mean_absolute_percentage_error'] * -1), 4)
    rmse = round(np.mean(results['test_neg_root_mean_squared_error'] * -1), 2)
    time = round(np.mean(results['fit_time']), 2)
    
    df_results = pd.DataFrame(
                {
                    'Modelo': type(model).__name__,
                    'MAE': mae,
                    'MAPE': mape,
                    'RMSE': rmse,
                    'time': time
                },
                index=[0]
            )
    
    return df_results


def baseline_performance(y_train, n_splits):
    """
    Return a cross validation metric from the baseline model - mean
    Input:
        y_train: array with train response variables
        n_splits: number of folds
    """
    kf = KFold(n_splits=n_splits)
    
    mae_list = []
    mape_list = []
    rmse_list = []
    
    for train_index, valid_index in kf.split(y_train): # split index train - valid
        
        new_y_train = y_train.iloc[train_index]
        new_y_valid = y_train.iloc[valid_index]
        
        y_hat = np.repeat(new_y_train.mean(), len(new_y_valid))
        
        mae_list.append(mean_absolute_error(new_y_valid, y_hat))
        mape_list.append(mean_absolute_percentage_error(new_y_valid, y_hat))
        rmse_list.append(sqrt(mean_squared_error(new_y_valid, y_hat)))
        
    df_results = pd.DataFrame(
                {
                    'Modelo': 'Baseline',
                    'MAE': round(np.mean(mae_list), 3),
                    'MAPE': round(np.mean(mape_list), 3),
                    'RMSE': round(np.mean(rmse_list), 3),
                    'time': 0
                },
                index=[0]
            )
    
    return df_results


def tokenize(text):
    # normalize case and remove punctuation
    text = re.sub(r"[^a-zA-Z0-9]", " ", text.lower())
    
    # tokenize text
    tokens = word_tokenize(text)
    
    # lemmatize andremove stop words and remove single letters:
    tokens = [lemmatizer.lemmatize(word) for word in tokens if (word not in stop_words) and (len(word) >= 2)]

    return tokens


def count_list_series(df, column, only_values=False):
    """
    Input:
    df - pandas dataframe containing the column to be counted;
    column - columns with the list entries that must have the values counted;
    describe - if True, return count and percentage. 
    
    Output:
    df - a new dataframe containing:
        Value: the name of the variable to be counted;
        Count: count of the variable;
        Percentage: the percentage of the value in relation to the number of lines.
    """
    final_list = []
    df_aux = df[column].dropna(axis=0) # droping na values to analysis. selecting only entries with a list
    for index, value in enumerate(df_aux):
        for language in value:
            final_list.append(language)
            
    if only_values == False:
        value, counts = np.unique(final_list, return_counts=True)
        df_final = pd.DataFrame(counts, value).sort_values(by=0, ascending=False).reset_index().rename({'index': 'Value', 0: 'Count'}, axis=1)
        df_final['Percentage'] = round((100 * df_final['Count'] / len(df)), 2)
        
        return df_final
    
    else:
        
        return set(final_list)
    

def corr_words(df, x_column, y_column, list_words):

    dict_words = {}
    df_corr = df.loc[:, [x_column, y_column]]
    
    for word in list_words:            
        
        df_corr['Corr_test'] = [1 if word in row else 0 for row in df_corr[x_column] ] 
        result = pointbiserialr(df_corr['Corr_test'], df_corr[y_column])

        dict_words[word] = [result[0], result[1]]
        
    return pd.DataFrame(dict_words, index=['Corr', 'p-value']).T.sort_values(by='Corr')<jupyter_output>Populating the interactive namespace from numpy and matplotlib<jupyter_text>1.2. Data<jupyter_code>data_import = pd.read_csv('../data/Train_rev1.csv')

data_import.head()<jupyter_output><empty_output><jupyter_text>2.0. Data Description<jupyter_code>df2 = data_import.copy()<jupyter_output><empty_output><jupyter_text>2.1. Shape Data<jupyter_code>print(f'Rows: {df2.shape[0]}')
print(f'Columns: {df2.shape[1]}')<jupyter_output>Rows: 244768
Columns: 12<jupyter_text>2.2. Data Types<jupyter_code>df2.dtypes<jupyter_output><empty_output><jupyter_text>2.3. Check Na<jupyter_code>tot_na = df2.isnull().sum()
percentage_na = round(df2.isnull().mean(), 4) * 100

# dataframe of na
pd.DataFrame([tot_na, percentage_na], index=['Total', 'Percentual']).T<jupyter_output><empty_output><jupyter_text>2.4. Pre-Analysis<jupyter_code># Plotando colunas com poucos valores
columns = ['ContractType', 'ContractTime']

fig, ax = plt.subplots(1, 2)

for i, val in enumerate(columns):
    sns.countplot(y = df2[val].astype('str'), color='#548CFF', ax=ax[i])
    plt.title(val)<jupyter_output><empty_output><jupyter_text>2.4.1. ContractType<jupyter_code>contract_null = df2.loc[df2['ContractType'].isnull(), 'SalaryNormalized']
contract_full = df2.loc[df2['ContractType'] == 'full_time', 'SalaryNormalized']            
contract_part = df2.loc[df2['ContractType'] == 'part_time', 'SalaryNormalized']      



fig, ax = plt.subplots(3, 1)

sns.boxplot(x=contract_null, ax=ax[0])

sns.boxplot(x=contract_full, ax=ax[1])

sns.boxplot(x=contract_part, ax=ax[2])

fig.tight_layout()<jupyter_output><empty_output><jupyter_text>2.4.2. ContractTime<jupyter_code>null = df2.loc[df2['ContractTime'].isnull(), 'SalaryNormalized']
contract = df2.loc[df2['ContractTime'] == 'contract', 'SalaryNormalized']            
permanent =  df2.loc[df2['ContractTime'] == 'permanent', 'SalaryNormalized']      



fig, ax = plt.subplots(3, 1)

sns.boxplot(x=null, ax=ax[0])
sns.boxplot(x=contract, ax=ax[1])
sns.boxplot(x=permanent, ax=ax[2])

fig.tight_layout()<jupyter_output><empty_output><jupyter_text>3.0. Variable Filtering + Split<jupyter_code>df3 = df2.copy()
df3.drop(['Id', 'LocationRaw', 'SalaryRaw', 'FullDescription', 'SourceName', 'Company', 'LocationNormalized'], axis=1, inplace=True)
df3.dropna(subset=['Title'], axis=0, inplace=True)<jupyter_output><empty_output><jupyter_text>3.1. Split Dataset<jupyter_code>y = df3.loc[:, 'SalaryNormalized']
X = df3.drop('SalaryNormalized', axis=1)

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)<jupyter_output><empty_output><jupyter_text>4.0. Feature Engineering  4.1. Title<jupyter_code># list de stop_words
stop_words = stopwords.words("english")

# instanciando o lemmatizer
lemmatizer = WordNetLemmatizer()
# tokenizando a coluna title
X_train['Title'] = X_train['Title'].apply(lambda x: tokenize(x))
# dafatrame com a contagem e porcentagem de palavras na coluna title
df_word_count = count_list_series(X_train, 'Title', only_values=False)

df_word_count.head()
# juntando x e y para fazer a correlacao
#df_aux = pd.merge(X_train, y_train, right_index=True, left_index=True)

# list_words
#list_words = df_word_count['Value'].to_list()

# Criando o dataframe com as correlacoes atraves do ponto bisserial
#df_corr = corr_words(df_aux, 'Title', 'SalaryNormalized', list_words)

# salvando em csv
#df_corr.to_csv('../data/corr_title.csv')
# importando o dataset com as correlacoes
df_corr = pd.read_csv('../data/corr_title.csv', index_col=0).sort_values(by='Corr', ascending=False)

# transformando as correlacoes em valores absolutos
df_corr['Corr'] = df_corr['Corr'].apply(lambda x: abs(x))

# Colunas que serao usadas
cols = df_corr.loc[df_corr['Corr'] > 0.02, :].index.to_list()

# imputando as colunas
for value in cols:
    X_train[value] = X_train['Title'].apply(lambda x: 1 if value in x else 0)
    
# dropando title
X_train = X_train.drop('Title', axis=1)<jupyter_output>/tmp/ipykernel_22190/3993504408.py:12: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`
  X_train[value] = X_train['Title'].apply(lambda x: 1 if value in x else 0)<jupyter_text>5.0. Exploratory Data Analysis  6.0. Data Preparation<jupyter_code>X_train_6 = X_train.copy()<jupyter_output><empty_output><jupyter_text>6.1. NA Values<jupyter_code>X_train_6.loc[X_train['ContractType'].isnull(), 'ContractType'] = 'null'
X_train_6.loc[X_train['ContractTime'].isnull(), 'ContractTime'] = 'null'<jupyter_output><empty_output><jupyter_text>6.2. Categorical  6.3.1. One Hot Encoder<jupyter_code>trf2 = ColumnTransformer(transformers =[
    ('enc', OneHotEncoder(sparse = False, drop ='first'), list(range(3))),
], remainder ='passthrough')
X_train_6 = pd.DataFrame(trf2.fit_transform(X_train_6))<jupyter_output><empty_output><jupyter_text>7.0. Feature Selection  8.0. Machine Learning Pipeline  8.1. Baseline - Mean<jupyter_code>base = baseline_performance(y_train, 5)<jupyter_output><empty_output><jupyter_text>8.2. Linear Regression<jupyter_code>lr = LinearRegression()

lr_model = cross_val_performance(X_train_6, y_train, cv=5, model=lr)
lr_model<jupyter_output><empty_output><jupyter_text>8.3. DecisionTreeRegressor<jupyter_code>dtr = DecisionTreeRegressor()

dtr_model = cross_val_performance(X_train_6, y_train, cv=5, model=dtr)
dtr_model<jupyter_output><empty_output><jupyter_text>8.4. Comparance<jupyter_code>cross_performance = pd.concat([base, lr_model, dtr_model]).sort_values(by='MAE').reset_index(drop=True)

cross_performance
# save performance
cross_performance.to_csv('../data/cross_performance_c02.csv')<jupyter_output><empty_output><jupyter_text>9.0. Performance - Test Data<jupyter_code># Tokenizando
X_test['Title'] = X_test['Title'].apply(lambda x: tokenize(x))

# Transformando os tokens
for value in cols:
    X_test[value] = X_test['Title'].apply(lambda x: 1 if value in x else 0)

X_test = X_test.drop('Title', axis=1)

# Filtrando null values
X_test.loc[X_test['ContractType'].isnull(), 'ContractType'] = 'null'
X_test.loc[X_test['ContractTime'].isnull(), 'ContractTime'] = 'null'

# Transformando as colunas em binarias
trf2 = ColumnTransformer(transformers =[
    ('enc', OneHotEncoder(sparse = False, drop ='first'), list(range(3))),
], remainder ='passthrough')

X_test = pd.DataFrame(trf2.fit_transform(X_test))
dtr = DecisionTreeRegressor()

dtr.fit(X_train_6, y_train)

y_hat = dtr.predict(X_test)
mean_absolute_error(y_test, y_hat)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12303060009803235, 'fraction_numerical': 0.014774875708983965, 'mean_word_length': 3.9693806541405707, 'pattern_counts': {'":': 0, '<': 83, '<?xml version=': 0, '>': 85, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '21', 'dir': 'jupyter-structured-clean-dedup', 'id': '215995', 'license': 'MIT', 'n_tokens_mistral': 5164, 'n_tokens_neox': 4989, 'n_words': 1063, 'path': 'notebooks/c02_jsp_feature_engineering-title.ipynb', 'repo_name': 'tadeucbm/job-salary-predict', 'size': '14281'}