<jupyter_start><jupyter_text>1. Load Data<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn import datasets, linear_model
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.preprocessing import MultiLabelBinarizer
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import QuantileTransformer
from time import time
from sklearn.pipeline import make_pipeline
from sklearn.linear_model import LogisticRegression
from sklearn import metrics
from sklearn.metrics import confusion_matrix

from sklearn.neural_network import MLPRegressor

years = ['2004', '2005','2006','2007','2008','2009','2010','2011','2012','2013','2014','2015']

cols = pd.read_csv('data_flag/2004_flag.csv').columns
data = pd.DataFrame(columns = cols[1:].append(pd.Index(['YEAR']))) 
for year_ in years:
    #np.asarray(data['default_flag'].astype(int))
#     flag = np.asarray(.astype(int))
#     print(flag.values())
    data_path = 'data_flag/{year}_flag.csv'.format(year=year_)  
    cols = pd.read_csv(data_path).columns
    data_this_year = pd.read_csv(data_path, usecols = cols[1:])
    print(data_this_year['default_flag'].value_counts())
    




import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn import datasets, linear_model
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.preprocessing import MultiLabelBinarizer
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import QuantileTransformer
from time import time
from sklearn.pipeline import make_pipeline
from sklearn.linear_model import LogisticRegression
from sklearn import metrics
from sklearn.metrics import confusion_matrix

from sklearn.neural_network import MLPRegressor

#Load data
data_list = []
for fname in sorted(os.listdir(data_path)):
    subject_data_path = os.path.join(data_path, fname)
    print(subject_data_path)

    if not os.path.isfile(subject_data_path): continue
    data_list.append(
        pd.read_csv(
                    subject_data_path,
                    sep='|', 
                    header=None,
                    names = [
                            'CREDIT_SCORE',
                            'FIRST_PAYMENT_DATE',
                            'FIRST_TIME_HOMEBUYER_FLAG',
                            '4','5','6',
                            'NUMBER_OF_UNITS',
                            'OCCUPANCY_STATUS',
                            '9',
                            'ORIGINAL_DTI_RATIO',
                            'ORIGINAL_UPB',
                            'ORIGINAL_LTV',
                            'ORIGINAL_INTEREST_RATE',
                            'CHANNEL',
                            '15',
                            'PRODUCT_TYPE',
                            'PROPERTY_STATE',
                            'PROPERTY_TYPE',
                            '19',
                            'LOAN_SQ_NUMBER',
                            'LOAN_PURPOSE',
                            'ORIGINAL_LOAN_TERM',
                            'NUMBER_OF_BORROWERS',
                            '24','25','26'#,'27'#data from every year may have different column number
                        #2004-2007: 27 2008: 26 2009: 27
                    ],
                    usecols=[
                        'CREDIT_SCORE',
                        'FIRST_TIME_HOMEBUYER_FLAG',
                        'NUMBER_OF_UNITS',
                        'OCCUPANCY_STATUS',
                        'ORIGINAL_DTI_RATIO',
                        'ORIGINAL_UPB',
                        'ORIGINAL_LTV',
                        'ORIGINAL_INTEREST_RATE',
                        'CHANNEL',
                        'PROPERTY_TYPE',
                        'LOAN_SQ_NUMBER',
                        'LOAN_PURPOSE',
                        'ORIGINAL_LOAN_TERM',
                        'NUMBER_OF_BORROWERS'
                    ],
                    dtype={'CREDIT_SCORE':np.float_, 
                           'FIRST_TIME_HOMEBUYER_FLAG':np.str, 
                           'NUMBER_OF_UNITS':np.int_, 
                           'OCCUPANCY_STATUS':np.str,
                           'ORIGINAL_DTI_RATIO':np.float_,
                           'ORIGINAL_UPB':np.float_,
                           'ORIGINAL_LTV':np.float_,
                           'ORIGINAL_INTEREST_RATE':np.float_,
                           'CHANNEL':np.str,
                           'PROPERTY_TYPE':np.str,
                           'LOAN_SQ_NUMBER':np.str,
                           'LOAN_PURPOSE':np.str,
                           'ORIGINAL_LOAN_TERM':np.int_,
                           'NUMBER_OF_BORROWERS':np.int_},
                    low_memory=False
                    )
    )
data = pd.concat(data_list)<jupyter_output><empty_output><jupyter_text>Static and Visualise<jupyter_code>print(PRODUCT_TYPE.value_counts())<jupyter_output>FRM    398439
Name: 15, dtype: int64<jupyter_text>- **Credit Score**<jupyter_code>CREDIT_clean = CREDIT_SCORE[CREDIT_SCORE != 9999]
bin_size = CREDIT_clean.max()-CREDIT_clean.min()
CREDIT_SCORE_UNKNOWN_RATIO = (CREDIT_SCORE.size-CREDIT_clean.size)/CREDIT_SCORE.size
plt.figure()
figure = plt.hist(CREDIT_clean,bins = bin_size, weights=np.ones(CREDIT_clean.size) / CREDIT_clean.size)
plt.show
print(CREDIT_SCORE_UNKNOWN_RATIO)<jupyter_output>0.00025599903623892237<jupyter_text>- **First Time Homebuyer Flag**<jupyter_code>FIRST_TIME_HOMEBUYER_FLAG_Y = (FIRST_TIME_HOMEBUYER_FLAG[FIRST_TIME_HOMEBUYER_FLAG=='Y'].size)/FIRST_TIME_HOMEBUYER_FLAG.size
FIRST_TIME_HOMEBUYER_FLAG_N = (FIRST_TIME_HOMEBUYER_FLAG[FIRST_TIME_HOMEBUYER_FLAG=='N'].size)/FIRST_TIME_HOMEBUYER_FLAG.size
FIRST_TIME_HOMEBUYER_FLAG_NG = (FIRST_TIME_HOMEBUYER_FLAG[FIRST_TIME_HOMEBUYER_FLAG=='9'].size)/FIRST_TIME_HOMEBUYER_FLAG.size

print('FIRST_TIME_HOMEBUYER_FLAG_Y = {}'.format(FIRST_TIME_HOMEBUYER_FLAG_Y))
print('FIRST_TIME_HOMEBUYER_FLAG_N = {}'.format(FIRST_TIME_HOMEBUYER_FLAG_N))
print('FIRST_TIME_HOMEBUYER_FLAG_NG = {}'.format(FIRST_TIME_HOMEBUYER_FLAG_NG))<jupyter_output>FIRST_TIME_HOMEBUYER_FLAG_Y = 0.06664508243419946
FIRST_TIME_HOMEBUYER_FLAG_N = 0.7890266766054528
FIRST_TIME_HOMEBUYER_FLAG_NG = 0.14432824096034777<jupyter_text>- **Number of Units**<jupyter_code>NUMBER_OF_UNITS_clean = NUMBER_OF_UNITS[NUMBER_OF_UNITS!=99]
plt.bar(NUMBER_OF_UNITS_clean.value_counts().index,NUMBER_OF_UNITS_clean.value_counts().array)<jupyter_output><empty_output><jupyter_text>- **State**<jupyter_code>plt.figure(figsize=(50,10))
# dif = np.diff(np.unique(data[16])).min()
# left_of_first_bin = data[16].min() - float(dif)/2
# right_of_last_bin = data[16].max() + float(dif)/2
plt.bar(data[16].value_counts().index,data[16].value_counts().array)
#print(data[16].value_counts()) #value_counts() shows how many different value in a series use size to see the total number
#lable = np.asarray(pd.get_dummies(data[2]))<jupyter_output><empty_output><jupyter_text>3. ImputationUse Mean substitution approach<jupyter_code>data[0] = data[0].apply(lambda x : CREDIT_clean.mean() if x == 9999 else x)
data[9] = data[9].apply(lambda x : CREDIT_clean.mean() if x == 999 else x)
data[11] = data[11].apply(lambda x : CREDIT_clean.mean() if x == 999 else x)<jupyter_output><empty_output><jupyter_text>- **Prepare input data for Linear Rgression**<jupyter_code>data.shape
data_c = data[[0,9,10,11,21]]
#data_d = data[[2,6,7,13,15,17,20,22]]

lable_2 = np.asarray(pd.get_dummies(data[2]))
lable_6 = np.asarray(pd.get_dummies(data[6]))
lable_7 = np.asarray(pd.get_dummies(data[7]))
lable_13 = np.asarray(pd.get_dummies(data[13]))
lable_15 = np.asarray(pd.get_dummies(data[15]))
lable_17 = np.asarray(pd.get_dummies(data[17]))
lable_20 = np.asarray(pd.get_dummies(data[20]))
lable_22 = np.asarray(pd.get_dummies(data[22]))

input_array = np.c_[data_c.to_numpy(),lable_2,lable_6,lable_7,lable_13,lable_15,lable_17,lable_20,lable_22] 
output_array = data[12].to_numpy()
print(input_array.shape)

# Split the data into training/testing sets
X_train = input_array[:-80000]
X_test = input_array[-80000:]

# Split the targets into training/testing sets
y_train = output_array[:-80000]
y_test = output_array[-80000:]

# Create linear regression object
regr = linear_model.LinearRegression()

# Train the model using the training sets
regr.fit(X_train, y_train)

# Make predictions using the testing set
y_pred = regr.predict(X_test)

# The coefficients
print('Coefficients: \n', regr.coef_)
# The mean squared error
print('Mean squared error: %.2f'
      % mean_squared_error(y_test, y_pred))
# The coefficient of determination: 1 is perfect prediction
print('Coefficient of determination: %.2f'
      % r2_score(y_test, y_pred))

# Plot outputs
axis = np.arange(80000)
size = np.linspace(0.5,0.5,80000)
plt.scatter(axis, y_test,s=size, color='red',marker='x')
plt.scatter(axis, y_pred, s = size, color='blue')

plt.show()

#try NN
X_train, X_test, y_train, y_test = train_test_split(input_array, output_array, test_size=0.2,
                                                    random_state=0)

print("Training MLPRegressor...")
tic = time()
est = make_pipeline(QuantileTransformer(),
                    MLPRegressor(hidden_layer_sizes=(50, 50),
                                 learning_rate_init=0.01,
                                 early_stopping=True))
est.fit(X_train, y_train)
print("done in {:.3f}s".format(time() - tic))
print("Test R2 score: {:.2f}".format(est.score(X_test, y_test)))<jupyter_output>Training MLPRegressor...
done in 115.203s
Test R2 score: 0.44
{'fraction_non_alphanumeric': 0.11024033437826541, 'fraction_numerical': 0.03625914315569488, 'mean_word_length': 2.760707269155206, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '6482', 'license': 'MIT', 'n_tokens_mistral': 3525, 'n_tokens_neox': 3210, 'n_words': 496, 'path': 'static.ipynb', 'repo_name': 'Shuhao99/2020-21-UNNC-CS-Final-Year-Project', 'size': '9570'}