<jupyter_start><jupyter_text>Import libraries<jupyter_code>import os

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import matplotlib as mpl
import seaborn as sns

%matplotlib inline

import sklearn
from sklearn.preprocessing import MinMaxScaler, LabelEncoder
from sklearn.metrics import (confusion_matrix, 
                             roc_auc_score, 
                             average_precision_score)
from sklearn.model_selection import cross_val_score

from imblearn.under_sampling import RandomUnderSampler

import tensorflow as tf
from tensorflow import keras

from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Activation
from tensorflow.keras.constraints import max_norm
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.models import load_model

import lightgbm as lgb
import xgboost as xgb
from sklearn.model_selection import GridSearchCV
from sklearn.linear_model import LogisticRegression
import joblib
import pickle

# import shap
import warnings
import time
warnings.filterwarnings("ignore")

import json
from collections import defaultdict

from datetime import datetime
import pytz
from datetime import date

# from bayes_opt import BayesianOptimization

# trained_model_folder_path = "/content/drive/MyDrive/Colab Notebooks/Kaggle/LendingClub/trained_models/"
def extract_windows(array, clearing_time_index, max_time, sub_window_size):
    examples = []
    # start = clearing_time_index + 1 - sub_window_size + 1
    start = clearing_time_index - sub_window_size + 1

    for i in range(max_time+1):
        if start+sub_window_size+i<sub_window_size:
            example = np.zeros((sub_window_size, array.shape[1])) ## zero padding
            example[-(start+sub_window_size+i):] = array[:start+sub_window_size+i]
        else:
            example = array[start+i:start+sub_window_size+i]

        examples.append(np.expand_dims(example, 0))
    
    return np.vstack(examples)
def shift_yr_qtr(yr_qtr):
    year = int(yr_qtr[:2])
    q = int(yr_qtr[-1])
    q_sum = (year*4+q-1)-1
    year = q_sum//4
    q = q_sum%4+1
    return str(year)+'Q'+str(q)

def shift_yr_mth(yr_mth):
    year = int(yr_mth[:4])
    month = int(yr_mth[4:])
    mth_sum = (year*12+month-1)-1
    year = mth_sum//12
    month = mth_sum%12+1
    if month<10:
        return str(year)+'0'+str(month)
    else:
        return str(year)+str(month)<jupyter_output><empty_output><jupyter_text>Load data  Load data: orig, performance<jupyter_code>dir_data = '/content/drive/MyDrive/Colab Notebooks/FreddieMac/FreddieMac/dataset_full/v3/'
dir_data_orig = os.path.join(dir_data, 'data_orig_2011_2019_pre_chg_20210803_2235.csv')
dir_data_monthly = os.path.join(dir_data, 'data_monthly_2011_2019_pre_chg_20210803_2235.csv') 

print('dir_data_orig', dir_data_orig)
print('dir_data_monthly', dir_data_monthly)
df_orig = pd.read_csv(dir_data_orig, index_col=0)
df_monthly = pd.read_csv(dir_data_monthly, index_col=0)

print("df_orig.shape", df_orig.shape)
print("df_monthly.shape", df_monthly.shape)
df_orig['o_yr_qtr'] = df_orig['loan_id'].apply(lambda x: x[1:5])
## shift year and quarter since economic factors can have a quarter lag
df_orig['o_yr_qtr'] = df_orig['o_yr_qtr'].apply(shift_yr_qtr) 

df_orig['o_st_yr_qtr'] = df_orig['o_prop_st']+'_'+df_orig['o_yr_qtr']
df_orig.head()
df_orig['o_yr_qtr_shifted'] = df_orig['o_yr_qtr'].apply(shift_yr_qtr)
df_orig.sample(100).head()
us_states = set(sorted(df_orig['o_prop_st'].unique().tolist()))
## merge
df_combined = pd.merge(df_monthly, df_orig, on='loan_id')
df_combined.sort_values(by=['loan_id', 'loan_age'], inplace=True)

df_combined['rep_period'] = df_combined['rep_period'].astype(str)
## shift rep_period 
df_combined['shifted_rep_period'] = df_combined['rep_period'].apply(shift_yr_mth)

## all date-related information are shifted by one month
## so that the lags of economic factors are considered, i.e., one-month or one-quarter lag
df_combined['o_rep_period'] = df_combined.groupby('loan_id').transform('first')['shifted_rep_period']
df_combined['o_st_rep_period'] = df_combined['o_prop_st']+'_'+df_combined['o_rep_period'].astype(str)

df_combined['st_rep_period'] = df_combined['o_prop_st']+'_'+df_combined['shifted_rep_period'].astype(str)
df_combined['rep_yr_qtr'] = df_combined['rep_period'].astype(str).apply(lambda x: x[2:4]+'Q'+str(((int(x[-2:])-1)//3)+1))
df_combined['rep_yr_qtr'] = df_combined['rep_yr_qtr'].apply(shift_yr_qtr)
df_combined['st_rep_yr_qtr'] = df_combined['o_prop_st'] + '_' + df_combined['rep_yr_qtr']

df_combined.head()<jupyter_output><empty_output><jupyter_text>Load data: house price index<jupyter_code>dir_hpi = '/content/drive/MyDrive/Colab Notebooks/FreddieMac/FreddieMac/economic_factors/house_index_history/HPI_PO_state.xls'
df_hpi = pd.read_excel(dir_hpi)
df_hpi['st_rep_yr_qtr'] = df_hpi['state']+'_'+df_hpi['yr'].apply(lambda x: str(x)[-2:])+'Q'+df_hpi['qtr'].apply(lambda x: str(x))
df_hpi.drop(columns=['Warning', 'index_nsa'], inplace=True)
df_hpi.rename(columns={'index_sa': 'hpi'}, inplace=True)
df_hpi.reset_index(inplace=True, drop=True)
df_hpi.sort_values(by=['state', 'yr', 'qtr'], inplace=True)
df_hpi.drop(columns=['state', 'yr', 'qtr'], inplace=True)

df_hpi<jupyter_output><empty_output><jupyter_text>Load data: unemployment rate history<jupyter_code>startyears = [2000, 2010, 2020]
endyears = [2009, 2019, 2021]
# dir_ue = '/content/drive/MyDrive/Colab Notebooks/FreddieMac/FreddieMac/economic_factors/monthly_state_ue_rate_2011_2020.csv'
df_ue_list = []
for startyear, endyear in zip(startyears, endyears):
    dir_ue = '/content/drive/MyDrive/Colab Notebooks/FreddieMac/FreddieMac/economic_factors/monthly_state_ue_rate_{startyear}_{endyear}.csv'.format(startyear=startyear, 
                                                                                                                                                    endyear=endyear)
    print(dir_ue)
    df_ue = pd.read_csv(dir_ue)
    df_ue_list.append(df_ue)

df_ue = pd.concat(df_ue_list, ignore_index=True)  
df_ue.drop(columns=['series_id', 'year', 'month', 'st', 'rep_period'], inplace=True)
df_ue.head()<jupyter_output>/content/drive/MyDrive/Colab Notebooks/FreddieMac/FreddieMac/economic_factors/monthly_state_ue_rate_2000_2009.csv
/content/drive/MyDrive/Colab Notebooks/FreddieMac/FreddieMac/economic_factors/monthly_state_ue_rate_2010_2019.csv
/content/drive/MyDrive/Colab Notebooks/FreddieMac/FreddieMac/economic_factors/monthly_state_ue_rate_2020_2021.csv<jupyter_text>Load data: Charge-Off Rate on Single Family Residential Mortgages, Booked in Domestic Offices, Top 100 Banks Ranked by Assetshttps://fred.stlouisfed.org/series/CORSFRMT100N<jupyter_code>dir_chgoff = '/content/drive/MyDrive/Colab Notebooks/FreddieMac/FreddieMac/economic_factors/FRED_mortgage_rate_data/Charge-Off Rate on Single Family Residential Mortgages, Booked in Domestic Offices, Top 100 Banks Ranked by Assets.xlsx'
df_chgoff = pd.read_excel(dir_chgoff)
df_chgoff['observation_date'] = pd.to_datetime(df_chgoff['observation_date'])
df_chgoff['rep_yr_qtr'] = df_chgoff['observation_date'].apply(lambda x: str(x.year)[-2:])+'Q'+df_chgoff['observation_date'].apply(lambda x: str(x.quarter))
df_chgoff.rename(columns={'CORSFRMT100S': 'charge_off_rate'}, inplace=True)
plt.plot(df_chgoff['observation_date'], df_chgoff['charge_off_rate'], label='charge_off_rate')
plt.legend()

df_chgoff.drop(columns=['observation_date'], inplace=True)
df_chgoff.head()<jupyter_output><empty_output><jupyter_text>Delinquency Rate on Single-Family Residential Mortgages, Booked in Domestic Offices, All Commercial Bankshttps://fred.stlouisfed.org/series/DRSFRMACBS<jupyter_code>dir_dlq = '/content/drive/MyDrive/Colab Notebooks/FreddieMac/FreddieMac/economic_factors/FRED_mortgage_rate_data/Delinquency Rate on Single-Family Residential Mortgages, Booked in Domestic Offices, All Commercial Banks.xlsx'
df_dlq = pd.read_excel(dir_dlq)
df_dlq['observation_date'] = pd.to_datetime(df_dlq['observation_date'])
df_dlq['rep_yr_qtr'] = df_dlq['observation_date'].apply(lambda x: str(x.year)[-2:])+'Q'+df_dlq['observation_date'].apply(lambda x: str(x.quarter))
df_dlq.rename(columns={'DRSFRMACBS': 'dlq_rate'}, inplace=True)
plt.plot(df_dlq['observation_date'], df_dlq['dlq_rate'], label='dlq_rate')
plt.legend()

df_dlq.drop(columns=['observation_date'], inplace=True)
df_dlq.head()<jupyter_output><empty_output><jupyter_text>30-Year Fixed Rate Mortgage Average in the United Stateshttps://fred.stlouisfed.org/series/MORTGAGE30US<jupyter_code>dir_mort_rate = '/content/drive/MyDrive/Colab Notebooks/FreddieMac/FreddieMac/economic_factors/FRED_mortgage_rate_data/30-Year Fixed Rate Mortgage Average in the United States.xlsx'
df_mort_rate = pd.read_excel(dir_mort_rate)
df_mort_rate['observation_date'] = pd.to_datetime(df_mort_rate['observation_date'])
df_mort_rate.sort_values(by='observation_date', inplace=True)
df_mort_rate['rep_period'] = df_mort_rate['observation_date'].dt.year.astype(str)+df_mort_rate['observation_date'].apply(lambda x: '0'+str(x.month) if x.month<10 else str(x.month))
df_mort_rate = df_mort_rate.groupby('rep_period').apply(lambda x: x.tail(1))
df_mort_rate.reset_index(inplace=True, drop=True)
df_mort_rate.rename(columns={'MORTGAGE30US': 'avg_frm'}, inplace=True)
plt.plot(df_mort_rate['observation_date'], df_mort_rate['avg_frm'], label='avg_frm')
plt.legend()

df_mort_rate.drop(columns=['observation_date'], inplace=True)
df_mort_rate.head()<jupyter_output><empty_output><jupyter_text>State Leve Mortgage Performance Statisticshttps://www.fhfa.gov/DataTools/Downloads/Pages/National-Mortgage-Database-Aggregate-Data.aspx<jupyter_code>dir_st_co_dlq = '/content/drive/MyDrive/Colab Notebooks/FreddieMac/FreddieMac/economic_factors/FRED_mortgage_rate_data/State_Level_Mortgage_Performance_Statistics.xlsx'
df_list = []
for st in us_states:
    try:
        df_st_co_dlq = pd.read_excel(dir_st_co_dlq, 
                                    sheet_name=st, 
                                    skiprows=4,
                                    header=1,
                                    skipfooter=2)
        df_st_co_dlq['st_rep_yr_qtr'] = st+'_'+df_st_co_dlq['Quarter'].astype(str).apply(lambda x: x[-4:])
        df_list.append(df_st_co_dlq)
        # break
    except:
        pass

df_st_co_dlq = pd.concat(df_list, ignore_index=True)  
df_st_co_dlq.drop(columns=['Quarter', 'Percent in Forbearance'], inplace=True)
df_st_co_dlq.rename(columns={'Percent 30 or 60 Days Past Due Date': 'pct_3060dpd', 
                             'Percent 90 to 180 Days Past Due Date': 'pct_90180dpd', 
                             'Percent in the Process of Foreclosure, Bankruptcy, or Deed in Lieu': 'pct_fcl'}, 
                    inplace=True)
df_st_co_dlq.dropna(axis=0, inplace=True)
df_st_co_dlq.head()<jupyter_output><empty_output><jupyter_text>Merge data  House index history data<jupyter_code>## hpi (quarterly updated) 
df_combined_hpi = pd.merge(df_combined, df_hpi, left_on='st_rep_yr_qtr', right_on='st_rep_yr_qtr')
df_combined_hpi.sort_values(by=['loan_id', 'loan_age'], inplace=True)

## hpi (origination date) 
df_combined_hpi['o_hpi'] = df_combined_hpi.groupby('loan_id').transform('first')['hpi']

df_combined_hpi.head()
del df_combined<jupyter_output><empty_output><jupyter_text>Unemployment rate<jupyter_code>## monthly unemployment rate
df_combined_ue = pd.merge(df_combined_hpi, 
                          df_ue, 
                          left_on='st_rep_period', 
                          right_on='st_rep_period')
df_combined_ue.sort_values(by=['loan_id', 'loan_age'], inplace=True)

## unemployment rate on origination date
df_combined_ue['o_ue_rate'] = df_combined_ue.groupby('loan_id').transform('first')['ue_rate']
df_combined_ue[['ue_rate', 'o_ue_rate']].describe()
del df_combined_hpi<jupyter_output><empty_output><jupyter_text>Charge-Off Rate on Single Family Residential Mortgages, Booked in Domestic Offices, Top 100 Banks Ranked by Assets<jupyter_code>df_combined_co = pd.merge(df_combined_ue, 
                          df_chgoff, 
                          left_on='rep_yr_qtr', 
                          right_on='rep_yr_qtr')
df_combined_co.sort_values(by=['loan_id', 'loan_age'], inplace=True)
df_combined_co.head()
del df_combined_ue<jupyter_output><empty_output><jupyter_text>Delinquency Rate on Single-Family Residential Mortgages, Booked in Domestic Offices, All Commercial Banks<jupyter_code>df_combined_dlq = pd.merge(df_combined_co, 
                           df_dlq, 
                           left_on='rep_yr_qtr', 
                           right_on='rep_yr_qtr')
df_combined_dlq.sort_values(by=['loan_id', 'loan_age'], inplace=True)
df_combined_dlq.head()
del df_combined_co<jupyter_output><empty_output><jupyter_text>30-Year Fixed Rate Mortgage Average in the United States<jupyter_code>df_combined_frm = pd.merge(df_combined_dlq, 
                           df_mort_rate, 
                           left_on='rep_period', 
                           right_on='rep_period')
df_combined_frm.sort_values(by=['loan_id', 'loan_age'], inplace=True)
df_combined_frm['o_avg_frm'] = df_combined_frm.groupby('loan_id').transform('first')['avg_frm']

df_combined_frm.head()
del df_combined_dlq<jupyter_output><empty_output><jupyter_text>State Level Mortgage Performance Statistics<jupyter_code>df_combined_co_dlq = pd.merge(df_combined_frm, 
                           df_st_co_dlq, 
                           left_on='st_rep_yr_qtr', 
                           right_on='st_rep_yr_qtr')
df_combined_co_dlq.sort_values(by=['loan_id', 'loan_age'], inplace=True)
df_combined_co_dlq['o_pct_fcl'] = df_combined_co_dlq.groupby('loan_id').transform('first')['pct_fcl']

df_combined_co_dlq.head()
del df_combined_frm<jupyter_output><empty_output><jupyter_text>Preprocess  Copy data<jupyter_code>df = df_combined_co_dlq.drop(columns=['o_yr_qtr', 'o_st_yr_qtr', 'o_yr_qtr_shifted',
                                    'shifted_rep_period', 'o_rep_period',
                                    'o_st_rep_period', 'st_rep_period', 
                                    'rep_yr_qtr', 'st_rep_yr_qtr',])
df.sort_values(by=['loan_id', 'loan_age'], inplace=True)
df.head()
del df_combined_co_dlq<jupyter_output><empty_output><jupyter_text>Create monthly features since est_ltv from the dataset have many missing values, the following estimation is used: $$est\_ltv = \dfrac{upb}{o\_upb}*\dfrac{o\_hpi}{hpi}*o\_ltv$$<jupyter_code>try:
    df['est_ltv'] = (df['actual_upb']/df['o_upb'])*(df['o_hpi']/df['hpi'])*df['o_ltv']
    df['upb_pct'] = df['actual_upb'] / df['o_upb']
    df['age_pct'] = df['loan_age'] / df['o_term']
    df.drop(columns=['o_term', 'o_pt_val_md'], inplace=True, errors='ignored') ## all mortgage are 30-year
except:
    pass

df.shape
df[['upb_pct', 'est_ltv', 'age_pct']].corr()<jupyter_output><empty_output><jupyter_text>Encoding categorical variables<jupyter_code>df.columns
df = pd.get_dummies(df, columns=['o_chan', 'o_purp', 
                                'o_first_flag', 
                                'o_num_brwrs', 'o_units', 
                                'o_occ_stat', 'o_prop_st',
                                'o_prop_type'])
df.shape
# df_file_name = "/content/drive/MyDrive/Colab Notebooks/FreddieMac/FreddieMac/dataset_full/v3/df_combined_2011_2019"
# df.to_csv(df_file_name+'.csv')
# print(df_file_name+'.csv')<jupyter_output><empty_output><jupyter_text>Origination and monthly features<jupyter_code>monthly_features = ['age_pct', 'loan_status', 
                    'upb_pct', 'est_ltv', 'delq_by_disaster',
                    'hpi', 'ue_rate', 'charge_off_rate', 'dlq_rate',
                    'avg_frm', 'pct_3060dpd', 'pct_90180dpd', 'pct_fcl', 
                    ]

orig_features = [col for col in df.columns if col.startswith('o_')]

target_col = ['zero_bal_code']
# ## removed columns
# for col in df.columns: 
#     if not col in monthly_features and not col in orig_features and not col in target_col:
#         print(col)<jupyter_output><empty_output><jupyter_text>Scale data<jupyter_code>df_scaled = df.copy()
df_scaled.sort_values(by=['loan_id', 'loan_age'], inplace=True)
df_scaled.drop(columns=[
                        # 'rep_period', 
                        'actual_upb', 'loan_age', 
                        'remaining_months', 'int_rate', 'o_init_pay_d'], 
               inplace=True)

monthly_scaler = MinMaxScaler()
orig_scaler = MinMaxScaler()

df_scaled.loc[:, monthly_features] = monthly_scaler.fit_transform(df.loc[:, monthly_features])
df_scaled.loc[:, orig_features] = orig_scaler.fit_transform(df.loc[:, orig_features])
df_scaled.head()
del df<jupyter_output><empty_output><jupyter_text>Create windowed dataset: monthly and origination features<jupyter_code>loan_idxes = df_scaled['loan_id'].unique().tolist()
# loan_idxes = np.random.choice(df_scaled['loan_id'].unique().tolist(), 10, replace=False)
# loan_idxes
assert len(loan_idxes)==len(set(loan_idxes))
print("total loans:", len(loan_idxes))
%%time
clearing_time_index = 0
sub_window_size = 24

list_start_windowed_monthly_arrs = defaultdict(list)
list_start_windowed_orig_arrs = defaultdict(list)
list_start_windowed_target_arrs = defaultdict(list)
list_start_windowed_loan_id_arrs = defaultdict(list)

list_end_windowed_monthly_arrs = defaultdict(list)
list_end_windowed_orig_arrs = defaultdict(list)
list_end_windowed_target_arrs = defaultdict(list)
list_end_windowed_loan_id_arrs = defaultdict(list)

loan_idxes_short_seq = set()

for i, loan_id in enumerate(loan_idxes):
    # print("loan_id", loan_id)
    if i%2000==0:
        print('{} loans have been processed'.format(i))

    loan_year = loan_id[1:3]
    # print('loan_year', loan_year, loan_id)

    df_m = df_scaled.loc[df_scaled['loan_id']==loan_id, monthly_features]
    df_o = df_scaled.loc[df_scaled['loan_id']==loan_id, orig_features]
    ## loans that ends in two months
    if len(df_m)<2:
        loan_idxes_short_seq.add(loan_id)
        continue

    zero_bal_code = df_scaled.loc[df_scaled['loan_id']==loan_id, target_col].values[-1][0]
    # print("zero_bal_code", zero_bal_code)
    loan_end_rep_period = df_scaled.loc[df_scaled['loan_id']==loan_id, ['rep_period']].values[-1][0]
    loan_end_year = loan_end_rep_period[2:4]
    # print("loan_end_rep_period", loan_end_year, loan_end_rep_period)
    if zero_bal_code==1.0: ## paid off
        target = 0
    elif zero_bal_code in set([2.0, 3.0, 9.0]): ## third-party sale, charged off, REO Disposition
        target = 1
    else: 
        raise ValueError("Wrong zero balance code!")
    
    array = df_m.values
    orig_array = df_o.values
    # assert np.isclose(orig_array, orig_array[0]).all()
    windowed_monthly_arr = extract_windows(array, 
                                           clearing_time_index=clearing_time_index, 
                                           max_time=len(array)-2, 
                                           sub_window_size=sub_window_size)
    windowed_orig_arr = orig_array[:windowed_monthly_arr.shape[0]]
    target_arr = np.ones(windowed_monthly_arr.shape[0]) * target
    loan_id_arr = np.repeat(np.array([loan_id]), windowed_monthly_arr.shape[0])

    # print('windowed_monthly_arr.shape', windowed_monthly_arr.shape)
    # print('windowed_orig_arr.shape', windowed_orig_arr.shape)
    # print('target_arr.shape', target_arr.shape)
    # print('loan_id_arr.shape', loan_id_arr.shape)

    ## check batch size
    assert windowed_monthly_arr.shape[0]==windowed_orig_arr.shape[0]
    assert windowed_monthly_arr.shape[0]==target_arr.shape[0]
    assert windowed_monthly_arr.shape[0]==loan_id_arr.shape[0]
    ## check window size 
    assert windowed_monthly_arr.shape[1]==sub_window_size    

    list_start_windowed_monthly_arrs[loan_year].append(windowed_monthly_arr)
    list_start_windowed_orig_arrs[loan_year].append(windowed_orig_arr)
    list_start_windowed_target_arrs[loan_year].append(target_arr)
    list_start_windowed_loan_id_arrs[loan_year].append(loan_id_arr)

    list_end_windowed_monthly_arrs[loan_end_year].append(windowed_monthly_arr)
    list_end_windowed_orig_arrs[loan_end_year].append(windowed_orig_arr)
    list_end_windowed_target_arrs[loan_end_year].append(target_arr)
    list_end_windowed_loan_id_arrs[loan_end_year].append(loan_id_arr)<jupyter_output>0 loans have been processed
2000 loans have been processed
4000 loans have been processed
6000 loans have been processed
8000 loans have been processed
10000 loans have been processed
12000 loans have been processed
14000 loans have been processed
16000 loans have been processed
18000 loans have been processed
20000 loans have been processed
CPU times: user 1h 13min 10s, sys: 18.5 s, total: 1h 13min 29s
Wall time: 1h 13min 16s<jupyter_text>Save all yearly loan data<jupyter_code>now = datetime.now()
# print(now)

utc = pytz.utc
utc_dt = datetime(now.year, now.month, now.day, now.hour, now.minute, now.second, tzinfo=utc)
eastern = pytz.timezone('US/Eastern')
loc_dt = utc_dt.astimezone(eastern)
fmt = '%Y%m%d_%H%M'
cur_datetime = loc_dt.strftime(fmt)
cur_datetime
save_path = '/content/drive/MyDrive/Colab Notebooks/FreddieMac/FreddieMac/dataset_full/v3/dataset_by_start'
list_start_windowed_monthly_arrs_file_name = os.path.join(save_path, 'list_start_windowed_monthly_arrs.pkl')
list_start_windowed_orig_arrs_file_name = os.path.join(save_path, 'list_start_windowed_orig_arrs.pkl')
list_start_windowed_target_arrs_file_name = os.path.join(save_path, 'list_start_windowed_target_arrs.pkl')
list_start_windowed_loan_id_arrs_file_name = os.path.join(save_path, 'list_start_windowed_loan_id_arrs.pkl')

print(list_start_windowed_monthly_arrs_file_name)
pickle.dump(list_start_windowed_monthly_arrs, open(list_start_windowed_monthly_arrs_file_name,'wb'))

print(list_start_windowed_orig_arrs_file_name)
pickle.dump(list_start_windowed_orig_arrs, open(list_start_windowed_orig_arrs_file_name,'wb'))

print(list_start_windowed_target_arrs_file_name)
pickle.dump(list_start_windowed_target_arrs, open(list_start_windowed_target_arrs_file_name,'wb'))

print(list_start_windowed_loan_id_arrs_file_name)
pickle.dump(list_start_windowed_loan_id_arrs, open(list_start_windowed_loan_id_arrs_file_name,'wb'))

save_path = '/content/drive/MyDrive/Colab Notebooks/FreddieMac/FreddieMac/dataset_full/v3/dataset_by_end'
list_end_windowed_monthly_arrs_file_name = os.path.join(save_path, 'list_end_windowed_monthly_arrs.pkl')
list_end_windowed_orig_arrs_file_name = os.path.join(save_path, 'list_end_windowed_orig_arrs.pkl')
list_end_windowed_target_arrs_file_name = os.path.join(save_path, 'list_end_windowed_target_arrs.pkl')
list_end_windowed_loan_id_arrs_file_name = os.path.join(save_path, 'list_end_windowed_loan_id_arrs.pkl')

print(list_end_windowed_monthly_arrs_file_name)
pickle.dump(list_end_windowed_monthly_arrs, open(list_end_windowed_monthly_arrs_file_name,'wb'))

print(list_end_windowed_orig_arrs_file_name)
pickle.dump(list_end_windowed_orig_arrs, open(list_end_windowed_orig_arrs_file_name,'wb'))

print(list_end_windowed_target_arrs_file_name)
pickle.dump(list_end_windowed_target_arrs, open(list_end_windowed_target_arrs_file_name,'wb'))

print(list_end_windowed_loan_id_arrs_file_name)
pickle.dump(list_end_windowed_loan_id_arrs, open(list_end_windowed_loan_id_arrs_file_name,'wb'))<jupyter_output>/content/drive/MyDrive/Colab Notebooks/FreddieMac/FreddieMac/dataset_full/v3/dataset_by_end/list_end_windowed_monthly_arrs.pkl
/content/drive/MyDrive/Colab Notebooks/FreddieMac/FreddieMac/dataset_full/v3/dataset_by_end/list_end_windowed_orig_arrs.pkl
/content/drive/MyDrive/Colab Notebooks/FreddieMac/FreddieMac/dataset_full/v3/dataset_by_end/list_end_windowed_target_arrs.pkl
/content/drive/MyDrive/Colab Notebooks/FreddieMac/FreddieMac/dataset_full/v3/dataset_by_end/list_end_windowed_loan_id_arrs.pkl<jupyter_text>Train/test split based on loan termination date To enusre no overlap between train and test data, the data are separated by the loan termination date. *   Train: mortgages that ends before 2018, i.e., before 12/31/2017*   Test: mortgages that orginated after 2018, i.e., after 01/01/2018<jupyter_code>train_end_year = 17
test_start_year = 18<jupyter_output><empty_output><jupyter_text>Train data<jupyter_code>dir_ = "/content/drive/MyDrive/Colab Notebooks/FreddieMac/FreddieMac/dataset_full/v3/dataset_by_end"
files = [
         "list_end_windowed_loan_id_arrs.pkl",
         "list_end_windowed_monthly_arrs.pkl",
         "list_end_windowed_orig_arrs.pkl",
         "list_end_windowed_target_arrs.pkl" 
        ]
object_list = []
for file_name in files:
    with open(os.path.join(dir_, file_name),'rb') as file:
        object_list.append(pickle.load(file))

(list_end_windowed_loan_id_arrs, 
 list_end_windowed_monthly_arrs, 
 list_end_windowed_orig_arrs, 
 list_end_windowed_target_arrs) = object_list

list_end_windowed_loan_id_arrs.keys()
n_train_loans = 0
train_orig_list = []
train_monthly_list = []
train_target_list = []
train_loan_id_list = []
for year in list_end_windowed_loan_id_arrs:
    if int(year)<=train_end_year:
        n_train_loans += len(list_end_windowed_loan_id_arrs[year])
        train_monthly_list.extend(list_end_windowed_monthly_arrs[year])
        train_orig_list.extend(list_end_windowed_orig_arrs[year])
        train_target_list.extend(list_end_windowed_target_arrs[year])
        train_loan_id_list.extend(list_end_windowed_loan_id_arrs[year])
print("n_train_loans", n_train_loans)
assert n_train_loans==len(train_monthly_list)
assert n_train_loans==len(train_orig_list)
assert n_train_loans==len(train_target_list)
assert n_train_loans==len(train_loan_id_list)


train_monthly_arr = np.concatenate(train_monthly_list)
train_orig_arr = np.concatenate(train_orig_list)
train_target_arr = np.concatenate(train_target_list)
train_loan_id_arr = np.concatenate(train_loan_id_list)

print("train_monthly_arr.shape", train_monthly_arr.shape, "(n_samples, time_steps, feature_size)")
print("train_orig_arr.shape", train_orig_arr.shape, "(n_samples, feature_size)")
print("train_target_arr.shape", train_target_arr.shape)
print("train_loan_id_arr.shape", train_loan_id_arr.shape)<jupyter_output>train_monthly_arr.shape (324443, 24, 13) (n_samples, time_steps, feature_size)
train_orig_arr.shape (324443, 86) (n_samples, feature_size)
train_target_arr.shape (324443,)
train_loan_id_arr.shape (324443,)<jupyter_text>Test data<jupyter_code>dir_ = "/content/drive/MyDrive/Colab Notebooks/FreddieMac/FreddieMac/dataset_full/v3/dataset_by_start"
files = [
         "list_start_windowed_loan_id_arrs.pkl",
         "list_start_windowed_monthly_arrs.pkl",
         "list_start_windowed_orig_arrs.pkl",
         "list_start_windowed_target_arrs.pkl" 
        ]
object_list = []
for file_name in files:
    with open(os.path.join(dir_, file_name),'rb') as file:
        object_list.append(pickle.load(file))

(list_start_windowed_loan_id_arrs, 
 list_start_windowed_monthly_arrs, 
 list_start_windowed_orig_arrs, 
 list_start_windowed_target_arrs) = object_list

list_start_windowed_loan_id_arrs.keys()
n_test_loans = 0
test_orig_list = []
test_monthly_list = []
test_target_list = []
test_loan_id_list = []
for year in list_start_windowed_loan_id_arrs:
    if int(year)>=test_start_year:
        n_test_loans += len(list_start_windowed_loan_id_arrs[year])
        test_monthly_list.extend(list_start_windowed_monthly_arrs[year])
        test_orig_list.extend(list_start_windowed_orig_arrs[year])
        test_target_list.extend(list_start_windowed_target_arrs[year])
        test_loan_id_list.extend(list_start_windowed_loan_id_arrs[year])
print("n_test_loans", n_test_loans)
assert n_test_loans==len(test_monthly_list)
assert n_test_loans==len(test_orig_list)
assert n_test_loans==len(test_target_list)
assert n_test_loans==len(test_loan_id_list)


test_monthly_arr = np.concatenate(test_monthly_list)
test_orig_arr = np.concatenate(test_orig_list)
test_target_arr = np.concatenate(test_target_list)
test_loan_id_arr = np.concatenate(test_loan_id_list)

print("test_monthly_arr.shape", test_monthly_arr.shape, "(n_samples, time_steps, feature_size)")
print("test_orig_arr.shape", test_orig_arr.shape, "(n_samples, feature_size)")
print("test_target_arr.shape", test_target_arr.shape)
print("test_loan_id_arr.shape", test_loan_id_arr.shape)<jupyter_output>test_monthly_arr.shape (10503, 24, 13) (n_samples, time_steps, feature_size)
test_orig_arr.shape (10503, 86) (n_samples, feature_size)
test_target_arr.shape (10503,)
test_loan_id_arr.shape (10503,)<jupyter_text>Save train/test data<jupyter_code>save_dir = "/content/drive/MyDrive/Colab Notebooks/FreddieMac/FreddieMac/dataset_full/v3/train_test_data"
train_data_names = [[train_monthly_arr, 'train_monthly_arr'], 
                    [train_orig_arr, 'train_orig_arr'], 
                    [train_target_arr, 'train_target_arr'], 
                    [train_loan_id_arr, 'train_loan_id_arr']]

test_data_names = [[test_monthly_arr, 'test_monthly_arr'], 
                    [test_orig_arr, 'test_orig_arr'], 
                    [test_target_arr, 'test_target_arr'], 
                    [test_loan_id_arr, 'test_loan_id_arr']]

for arr, file_name in train_data_names+test_data_names:
    dir_path = os.path.join(save_dir, file_name+'.npy')
    np.save(dir_path, arr)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11343537414965986, 'fraction_numerical': 0.014081632653061225, 'mean_word_length': 5.355598789450929, 'pattern_counts': {'":': 0, '<': 101, '<?xml version=': 0, '>': 97, 'https://': 4, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 21, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '25', 'dir': 'jupyter-structured-clean-dedup', 'id': '666048', 'license': 'MIT', 'n_tokens_mistral': 11547, 'n_tokens_neox': 10908, 'n_words': 1455, 'path': 'step2_0_create_windowed_dataset.ipynb', 'repo_name': 'Mengzhe/FreddieMacMortgageProject', 'size': '29400'}