<jupyter_start><jupyter_text>Pre-Processing<jupyter_code>#Imputing data in Total column
def impute_func_total(data):
    if(pd.isnull(data['Total'])):        
        if(data['Object_Type'] == 'Base Salary/Compensation'):
            return 24146
        if(data['Object_Type'] == 'Benefits'):
            return 38163
        if(data['Object_Type'] == 'Contracted Services'):
            return 24146
        if(data['Object_Type'] == 'Equipment & Equipment Lease'):
            return 11257
        if(data['Object_Type'] == 'NO_LABEL'):
            return 58545
        if(data['Object_Type'] == 'Other Compensation/Stipend'):
            return 1605
        if(data['Object_Type'] == 'Other Non-Compensation'):
            return 10646
        if(data['Object_Type'] == 'Rent/Utilities'):
            return 46611
        if(data['Object_Type'] == 'Substitute Compensation'):
            return 1090
        if(data['Object_Type'] == 'Supplies/Materials'):
            return 7745
        if(data['Object_Type'] == 'Travel & Conferences'):
            return 1659
    else:
        return data['Total']
#Imputing data in FTE column
def impute_func_FTE(data):
    if(pd.isnull(data['FTE'])):        
        if(data['Object_Type'] == 'Base Salary/Compensation'):
            return 0.45
        if(data['Object_Type'] == 'Benefits'):
            return 0.0
        if(data['Object_Type'] == 'Contracted Services'):
            return 0.0
        if(data['Object_Type'] == 'Equipment & Equipment Lease'):
            return 0.0
        if(data['Object_Type'] == 'NO_LABEL'):
            return 0.75
        if(data['Object_Type'] == 'Other Compensation/Stipend'):
            return 0.000107
        if(data['Object_Type'] == 'Other Non-Compensation'):
            return 0.0
        if(data['Object_Type'] == 'Rent/Utilities'):
            return 0.0
        if(data['Object_Type'] == 'Substitute Compensation'):
            return 0.000059
        if(data['Object_Type'] == 'Supplies/Materials'):
            return 0.0
        if(data['Object_Type'] == 'Travel & Conferences'):
            return 0.0
    else:
        return data['FTE']
def preProcessing(training):
    # Remove inconsistent data
    training.loc[training['FTE'] < 0, 'FTE'] = np.nan
    training.loc[training['Total'] < 0, 'Total'] = np.nan
    
    training['Total'] = training.apply(impute_func_total, axis = 1)
    
    #Put FET data greater than 1 in percentage 
    mask = training['FTE'] > 1
    df_ToAtt = training[mask]
    training.loc[mask, 'FTE'] = df_ToAtt['FTE'].div(100)
    
    training['FTE'] = training.apply(impute_func_FTE, axis = 1)
    
    for category in categoric:
        training[category] = training[category].str.lower()
    
    training[categoric] = training[categoric].fillna("")
    
    return training
df_training = preProcessing(training)
df_training = df_training.reset_index(drop = True)<jupyter_output><empty_output><jupyter_text>Test ANN - scikt<jupyter_code>#from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
#from sklearn.preprocessing import StandardScaler

from sklearn.preprocessing import FunctionTransformer
from sklearn.feature_selection import chi2, SelectKBest
from sklearn.feature_extraction.text import CountVectorizer

from sklearn.model_selection import cross_val_score
from sklearn.model_selection import cross_val_predict
from sklearn.metrics import confusion_matrix
import itertools

from sklearn.neural_network import MLPClassifier
target = pd.get_dummies(df_training[['Object_Type']])
train = df_training[categoric]
#X_train, X_test, y_train, y_test = train_test_split(df_training,
#                                                    pd.get_dummies(df_training[['Object_Type']]),
#                                                    test_size=0.2, 
#                                                    random_state=42)
def combine_text_columns(dataset):
    return dataset[categoric].apply(lambda x: " ".join(x), axis = 1)
get_text_data = FunctionTransformer(combine_text_columns, validate = False)
stop_words = ['a', 'e', 'i', 'o', 'u', 'to', 'all', 'of', 'for', 'and', 'or', 'b', 'with', 'not', 'than', 'other', 'top', 's', 'k', 'in', 'the']
vec = CountVectorizer(token_pattern="[A-Za-z0-9]+(?=\\s+)", ngram_range=(1,2), stop_words = stop_words)
# Adding the input layer and the first hidden, the hidden layer number is (11)/2
mlp = MLPClassifier(hidden_layer_sizes=(20,), max_iter=100, alpha=1e-4,
                    solver='sgd', verbose=True, tol=1e-4, random_state=42, momentum = 0.8, 
                    validation_fraction = 0.3, early_stopping = True, n_iter_no_change = 10,
                    learning_rate_init=.1)
# Edit model step in pipeline
pl = Pipeline([
        ('selector', get_text_data),
        ('vectorizer', vec),
        #('reduce_dim', TruncatedSVD(n_iter=7, n_components = 1000)),
        #('reduce_dim', NMF(n_components = 200)),
        ('reduce_dim', SelectKBest(chi2, 1000)),
        ('mlp', mlp)
    ])
#cross_val_score(pl, train, target, cv=2)
y_pred = cross_val_predict(pl, train, target, cv=5)
y_pred.shape
# Look at confusion matrix 

def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    #fig, ax = plt.subplots(figsize=(20, 10))
    plt.figure(figsize=(10,7))
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=90)
    plt.yticks(tick_marks, classes)

    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]

    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, cm[i, j],
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")
    
    #plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

# Predict the values from the validation dataset
#Y_pred = model.predict(X_val)
# Convert predictions classes to one hot vectors 
#Y_pred_classes = np.argmax(Y_pred,axis = 1) 
# Convert validation observations to one hot vectors
#Y_true = np.argmax(Y_val,axis = 1) 
# compute the confusion matrix
target.info()
confusion_mtx = confusion_matrix(target.values.argmax(axis=1), y_pred.argmax(axis=1)) 
classes = ["Base Salary/Compensation", "Benefits", "Contracted Services", "Equipment & Equipment Lease", "NO_LABEL", 
          "Other Compensation/Stipend", "Other Non-Compensation", "Rent/Utilities", "Substitute Compensation", 
          "Supplies/Materials", "Travel & Conferences"]
# plot the confusion matrix
plot_confusion_matrix(confusion_mtx, classes = classes)
target.values.argmax(axis=1)
y_pred.argmax(axis=1)
confusion_mtx = confusion_matrix(target.values.argmax(axis=1), y_pred.argmax(axis=1)) 
confusion_mtx
pl.fit(X_train, y_train)
from sklearn.metrics import accuracy_score

y_pred = pl.predict(X_test)
accuracy_score(y_test.values, y_pred, normalize=True)
df_training.head(1)
y_pred
y_test.values<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13323332878767216, 'fraction_numerical': 0.022091913268784943, 'mean_word_length': 3.0744444444444445, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '562723', 'license': 'MIT', 'n_tokens_mistral': 2455, 'n_tokens_neox': 2308, 'n_words': 496, 'path': 'notebooks/NeuralNetwork.ipynb', 'repo_name': 'nymarya/school-budgets-for-education', 'size': '7333'}