<jupyter_start><jupyter_text>Data cleaning Part 1: numeric dataFirst of all, the numeric columns `FTE` and `Total` must be cleaned. The outliers will be treated as `NaN` values and imputation will be applied to the missing data.<jupyter_code># Remove inconsistent data
training[training['FTE'] < 0] = np.nan
training[training['Total'] < 0] = np.nan
training['FTE'].describe()
training[training['FTE'] < 0]
training['Total'].describe()
training[numeric].info()
400277 - 125849
400277 - 351845<jupyter_output><empty_output><jupyter_text>Part 2: Categoric dataThe strings must be normalized: symbols removed, all letters to lower case. Then, they can be transformed to numeric data.<jupyter_code>for category in categoric:
    training[category] = training[category].str.lower()
#Check if the unique values decrease
training[categoric].nunique(dropna=False)
#Convert columns
training[categoric] = training[categoric].apply(lambda x: x.astype('object'), axis = 0)
training[categoric].info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 400277 entries, 134338 to 415831
Data columns (total 14 columns):
Facility_or_Department    53186 non-null object
Function_Description      298676 non-null object
Fund_Description          179157 non-null object
Job_Title_Description     272317 non-null object
Location_Description      155504 non-null object
Object_Description        332143 non-null object
Position_Extra            244180 non-null object
Program_Description       280943 non-null object
SubFund_Description       263731 non-null object
Sub_Object_Description    70748 non-null object
Text_1                    251891 non-null object
Text_2                    87431 non-null object
Text_3                    108613 non-null object
Text_4                    53384 non-null object
dtypes: object(14)
memory usage: 45.8+ MB<jupyter_text>Before removing symbols, let's take a peek on some values and so we can detect the diferrences later<jupyter_code>some_functions = training['Function_Description'].value_counts(normalize=True) > 0.05
some_functions.head(6)
# Remove symbols
'''for category in categoric:
    training[category] = training[category].str.replace(r"[&/\)('\"-.,]","")'''
vec = HashingVectorizer(token_pattern="[A-Za-z0-9]+(?=\\s+)", ngram_range=(1,2))
training['Function_Description'].value_counts(normalize=True).head(6)
training[categoric] = training[categoric].fillna("")
training['Function_Description'].isnull().sum()
# Join all text items in a row that have a space in between
text_data = training[categoric].apply(lambda x: " ".join(x), axis = 1)
text_data
hec = HashingVectorizer(token_pattern="[A-Za-z0-9]+(?=\\s+)", norm=None, binary=False,
                                                     ngram_range=(1,2))
hashed_text = hec.fit_transform(text_data)
hashed_df = pd.DataFrame(hashed_text.data)
print(hashed_df.info()
hashed_df<jupyter_output><empty_output><jupyter_text>Frequency of labels<jupyter_code>training[labels].info()
training.groupby('Object_Type')['Total'].mean()
training['Total'].mean()
def impute_func_total(data):
    if(pd.isnull(data['Object_Type'])):
        return 16964
    if(pd.isnull(data['Total'])):        
        if(data['Object_Type'] == 'Base Salary/Compensation'):
            return 24146
        if(data['Object_Type'] == 'Benefits'):
            return 38163
        if(data['Object_Type'] == 'Contracted Services'):
            return 24146
        if(data['Object_Type'] == 'Equipment & Equipment Lease'):
            return 11257
        if(data['Object_Type'] == 'NO_LABEL'):
            return 58545
        if(data['Object_Type'] == 'Other Compensation/Stipend'):
            return 1605
        if(data['Object_Type'] == 'Other Non-Compensation'):
            return 10646
        if(data['Object_Type'] == 'Rent/Utilities'):
            return 46611
        if(data['Object_Type'] == 'Substitute Compensation'):
            return 1090
        if(data['Object_Type'] == 'Supplies/Materials'):
            return 7745
        if(data['Object_Type'] == 'Travel & Conferences'):
            return 1659
    else:
        return data['Total']
training['Total'] = training.apply(impute_func_total, axis = 1)
training.groupby('Object_Type')['FTE'].mean()
training['FTE'].mean()
def impute_func_FTE(data):
    if(pd.isnull(data['Object_Type'])):
        return 0.42
    if(pd.isnull(data['FTE'])):        
        if(data['Object_Type'] == 'Base Salary/Compensation'):
            return 0.5
        if(data['Object_Type'] == 'Benefits'):
            return 0.0
        if(data['Object_Type'] == 'Contracted Services'):
            return 0.0
        if(data['Object_Type'] == 'Equipment & Equipment Lease'):
            return 0.0
        if(data['Object_Type'] == 'NO_LABEL'):
            return 0.75
        if(data['Object_Type'] == 'Other Compensation/Stipend'):
            return 0.000107
        if(data['Object_Type'] == 'Other Non-Compensation'):
            return 0.0
        if(data['Object_Type'] == 'Rent/Utilities'):
            return 0.0
        if(data['Object_Type'] == 'Substitute Compensation'):
            return 0.000059
        if(data['Object_Type'] == 'Supplies/Materials'):
            return 0.0
        if(data['Object_Type'] == 'Travel & Conferences'):
            return 0.0
    else:
        return data['FTE']
training['FTE'] = training.apply(impute_func_FTE, axis = 1)
mask = training['FTE'] > 1
df_ToAtt = training[mask]

training.loc[mask, 'FTE'] = df_ToAtt['FTE'].div(100)
training.describe()
training[numeric].info()
CountStatus = pd.value_counts(training['Object_Type'].values, sort=True)
CountStatus.plot.bar()<jupyter_output><empty_output><jupyter_text>Intro to Neural Network<jupyter_code>from sklearn.model_selection import train_test_split
from sklearn.neural_network import MLPClassifier
numeric_data = training[numeric]
#imp = Imputer(missing_values='NaN', strategy='mean', axis=0)
#Simple imputations
#numeric_data[['FTE']] = imp.fit(numeric_data[['FTE']]).transform(numeric_data[['FTE']])
#numeric_data[['Total']] = imp.fit(numeric_data[['Total']]).transform(numeric_data[['Total']])
numeric_data.info()
numeric_data.describe()
numeric_data.describe()
df_dummies = pd.get_dummies(training['Object_Type'])
df_dummies.head()
df_dummies.info()
X_train, X_test, y_train, y_test = train_test_split(numeric_data,
                                                    df_dummies,
                                                    test_size=0.2, 
                                                    random_state=42)
X_train
# Feature Scaling
from sklearn.preprocessing import StandardScaler
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.fit_transform(X_test)
# Adding the input layer and the first hidden, the hidden layer number is (11 + 2)/2
mlp = MLPClassifier(hidden_layer_sizes=(13,), max_iter=100, alpha=1e-4,
                    solver='sgd', verbose=10, tol=1e-4, random_state=1,
                    learning_rate_init=.1)
mlp.fit(X_train, y_train)
#Making predictions and evaluating the model

# Predicting the Test set results
y_pred = mlp.predict(X_test)
y_test_np = y_test.values
y_test_np.shape
y_pred.shape
from sklearn.metrics import accuracy_score

accuracy_score(y_test_np, y_pred, normalize=True)
accuracy_score(y_test_np, y_pred, normalize=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12869399428026693, 'fraction_numerical': 0.03745063325616233, 'mean_word_length': 3.575700934579439, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '649357', 'license': 'MIT', 'n_tokens_mistral': 2486, 'n_tokens_neox': 2261, 'n_words': 488, 'path': 'notebooks/ANN_model.ipynb', 'repo_name': 'nymarya/school-budgets-for-education', 'size': '7343'}