<jupyter_start><jupyter_text>Libs<jupyter_code>import os
import ast
import pandas as pd
import numpy as np
from matplotlib import pyplot
import matplotlib.patches as mpatches
import seaborn as sn
from tqdm.std import tqdm
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
import tensorflow as tf
from tensorflow import keras

import logging
logging.basicConfig(level='INFO')


logging.info(msg=f"Using tensorflow version : {tf.__version__}")
import warnings
warnings.filterwarnings(action="ignore")<jupyter_output><empty_output><jupyter_text>Config & Helpers funcs<jupyter_code>class Config:
    data_dir = '../data/'
    working_dir = '../src/'
    models_dir = '../models/'
    submissions_dir = '../submissions'


def preprocess_data(df:pd.DataFrame, skip_cols:list=None)->pd.DataFrame:
    """
        Parameters
        ----------
            df : Pandas dataframe
                Dataframe to be preprocessed.
                
            skip_cols : list
                list of columns to skip during preprocessing
                
        Returns
        -------
            df : Pandas dataframe
                Preprocessed dataframe.
        
        Examples
        --------
        >>> dataset = pd.read_csv(filepath_or_buffer='<path_to_data.csv>') 
        >>> new_df = preprocess_data(df=dataset) 
        
    """
    
    df = df.copy(deep=True)
    cols = [col for col in df.columns[1:] if col not in skip_cols]
    cat_cols = [col for col in cols if df[col].dtype == 'O' and 'Date' not in col]
    date_cols = [col for col in cols if 'Date' in col]
    
    # convert cat col to category type
    # print("Categorical cols", cat_cols)
    for col in tqdm(cat_cols, desc='Processing categorical columns \t'):
        df[col] = df[col].astype('category')
        
    # print("Date cols", date_cols)
    # convert date col to datetime type
    for col in tqdm(date_cols, desc='Processing datetime columns \t'):
        df[col] = pd.to_datetime(df[col]).dt.tz_localize(None)
    
    # fill missing values
    df.UpsellDate.fillna(value='None', inplace=True)
    df.Region.fillna(value=df.Region.mode()[0], inplace=True)
    df.Age.fillna(value=round(df.Age.mean()), inplace=True)
    
    for col in cat_cols:            
        try:
            df[col] = df[col].cat.codes
        except Exception as ex:
            logging.error(msg=f"Column issue ({col}) : {ex}")

        
    return df


def extract_features(df:pd.DataFrame)->pd.DataFrame:
    df = df.copy()
    df['RegisteredInLeapYear'] = df.RegistrationDate.dt.is_leap_year.astype('float')
    df['RegisteredAtMonthStart'] = df.RegistrationDate.dt.is_month_start.astype('float')
    df['RegisteredAtMonthEnd'] = df.RegistrationDate.dt.is_month_end.astype('float')
  #  df['LastPaymentMonth'] = df.LastPaymentDate.dt.month
    df['FirstPaymentMonth'] = df.FirstPaymentDate.dt.month
    df['FirstPaymentYear'] = df.FirstPaymentDate.dt.year
    
    lastReceivedPayment = []
    accessory_rate = []
    upsell_date = []
    for r in df.copy().iterrows():
        r = r[1]
        lastReceivedPayment.append(ast.literal_eval(r.TransactionDates)[-1])
        if r.AccessoryRate :
          r.AccessoryRate = 1
        accessory_rate.append(r.AccessoryRate)
        if r.UpsellDate != None :
          upsell_date.append(1)
        else :
          upsell_date.append(0)

    print(lastReceivedPayment)
    df['LastReceivedPayment'] = lastReceivedPayment
    df['LastReceivedPayment'] = pd.to_datetime(df['LastReceivedPayment']).dt.tz_localize(None)
    df['LastReceivedMonth'] = df.LastReceivedPayment.dt.month
    df['LastReceivedYear'] = df.LastReceivedPayment.dt.year
    df['LeftTime'] = (df['ExpectedTermDate'].dt.year-df['LastReceivedPayment'].dt.year)*12 + df['ExpectedTermDate'].dt.month -df['LastReceivedPayment'].dt.month 
    df['AccessoryRate'] = accessory_rate
    df['UpsellDate'] = upsell_date

    return df.drop(columns=["TransactionDates", "PaymentsHistory", "m1", "m2", "m3", "m4", "m5", "m6"], inplace=False)


def pad_history(df:pd.DataFrame, max_len:int=41):
    
    df = df.copy()
    
    padded_payments = []
    
    for r in df.copy().iterrows():
        r = r[1]
        
        if len(ast.literal_eval(r.PaymentsHistory)) > max_len:
            padded_payments.append(ast.literal_eval(r.PaymentsHistory)[:max_len])
            
        else:
            padding_len = abs(max_len - len(ast.literal_eval(r.PaymentsHistory)))
            padded_payments.append(ast.literal_eval(r.PaymentsHistory) + padding_len*[0.])
            
    
    df["PaymentsHistory"] = padded_payments
    
    del padded_payments
    
    return df, max_len

def pad_history_most_recent(df:pd.DataFrame, max_len:int=6):
    
    df = df.copy()
    
    padded_payments_1 = []
    padded_payments_2 = []
    padded_payments_3 = []
    padded_payments_4 = []
    padded_payments_5 = []
    padded_payments_6 = []
    remaining_payment = []
    RemainingPaymentPerUnitTimeLeft = []
    
    for r in df.copy().iterrows():
        r = r[1]
        padded_payments_1.append(ast.literal_eval(r.PaymentsHistory)[-1])
        padded_payments_2.append(ast.literal_eval(r.PaymentsHistory)[-2])
        padded_payments_3.append(ast.literal_eval(r.PaymentsHistory)[-3])
        padded_payments_4.append(ast.literal_eval(r.PaymentsHistory)[-4])
        padded_payments_5.append(ast.literal_eval(r.PaymentsHistory)[-5])
        remaining_payment.append(r.TotalContractValue - np.sum(ast.literal_eval(r.PaymentsHistory)[:]))
   #     RemainingPaymentPerUnitTimeLeft.append((r.TotalContractValue - np.sum(ast.literal_eval(r.PaymentsHistory)[:]))/r.LeftTime)
    
    df["PaymentsHistory_1"] = padded_payments_1
    df["PaymentsHistory_2"] = padded_payments_2
    df["PaymentsHistory_3"] = padded_payments_3
    df["PaymentsHistory_4"] = padded_payments_4
    df["PaymentsHistory_5"] = padded_payments_5
    df["RemainingPayment"] = remaining_payment

    
    del padded_payments_1
    
    return df<jupyter_output><empty_output><jupyter_text>Dataset---1. Load files2. Exploratory Data Analysis (EDA)3. Preprocess data4. Extract features4. Train/Test split<jupyter_code>train = pd.read_csv( "Train.csv")
metadata = pd.read_csv( "metadata.csv")
test = pd.read_csv("Test.csv")
ss = pd.read_csv( "SampleSubmission.csv")<jupyter_output>C:\Users\<NAME>\.conda\envs\cropChallenge\lib\site-packages\IPython\core\interactiveshell.py:3441: DtypeWarning: Columns (3) have mixed types.Specify dtype option on import or set low_memory=False.
  exec(code_obj, self.user_global_ns, self.user_ns)<jupyter_text>EDA<jupyter_code>train.info()
train.describe(datetime_is_numeric=True).transpose()
train.head(n=2)
metadata.info()
metadata.drop(columns=["SupplierName", "LastPaymentDate", "PaymentMethod", "rateTypeEntity"], inplace=True)
metadata.describe(datetime_is_numeric=True).transpose()
metadata.head(n=2)
test.info()
test.describe(datetime_is_numeric=True).transpose()
test.head(n=2)
print("---------------- Missing values Report ----------------")
print(metadata.isna().sum())
merged = pd.merge(left=metadata, right=pd.concat(objs=[train, test]).fillna(value=0.), on="ID")<jupyter_output><empty_output><jupyter_text>Preprocessing<jupyter_code>merged_ = preprocess_data(df=merged, skip_cols=["PaymentsHistory", "TransactionDates"])
c= []
for i in range (0,len(merged_)):
    c.append(len(ast.literal_eval(merged_.TransactionDates[i])))
import matplotlib.pyplot as plt
plt.plot(c)
plt.show
merged_ = extract_features(df=merged_)
merged_.head(n=2)
plt.plot(merged_.LeftTime)
plt.show
t = 0 
for i in range (0,len(merged_)):
  if merged_.LeftTime[i] > 5:
    t= t+1
print(len(merged_) - t, len(merged_))
merged_.isna().sum()<jupyter_output><empty_output><jupyter_text>Features extraction<jupyter_code>#merged_ = extract_features(df=merged_)
merged_.head(n=20)
merged_['LeftTime'].unique()
merged_.head(2)
# RegistrationDate , UpsellDate , Region , Town , Occupation , ExpectedTermDate , FirstPaymentDate , LastReceivedPayment
for i in merged_.columns:
    print(type(merged_[i][1]),i)
selected_features = [c for c in merged_.columns[1:] if "Date" not in c]
selected_features
selected_features = [c for c in merged_.columns[1:] if "Date" not in c]
xtrain_loan = merged_[selected_features]
ytrain_loan = merged_.Target
## Split
xtrain_loan, xval_loan, ytrain_loan, yval_loan = train_test_split(
    xtrain_loan, 
    ytrain_loan, 
    test_size=.3, 
    random_state=21, 
    shuffle=True
)
scaler = StandardScaler()
##Fit scaler
scaler.fit(X=xtrain_loan.values)

## Scale data
xtrain_loan = scaler.transform(X=xtrain_loan.values)
xval_loan = scaler.transform(X=xval_loan.values)

print(f"> Training inputs shape : {xtrain_loan.shape}, Training targets shape : {ytrain_loan.shape}")
print(f"> Validation inputs shape : {xval_loan.shape}, Training targets shape : {yval_loan.shape}")<jupyter_output><empty_output><jupyter_text>Train/Val split<jupyter_code>#train = pad_history_most_recent(df=train, max_len=5)
train.head()
merged_train = pd.merge(left=train, right=pd.concat(objs=[merged_]).fillna(value=0.), on="ID")
merged_train = pad_history_most_recent(df=merged_train, max_len=5)
merged_train.head()
plt.plot(merged_train.RemainingPayment)
# func that returns a dummified DataFrame of significant dummies in a given column
def dum_sign(dummy_col, threshold=0.01):

    # removes the bind
    dummy_col = dummy_col.copy()

    # what is the ratio of a dummy in whole column
    count = pd.value_counts(dummy_col) / len(dummy_col)

    # cond whether the ratios is higher than the threshold
    mask = dummy_col.isin(count[count > threshold].index)

    # replace the ones which ratio is lower than the threshold by a special name
    dummy_col[~mask] = "others"

    return pd.get_dummies(dummy_col, prefix=dummy_col.name)

merged_train = pd.concat([merged_train, dum_sign(merged_train['DaysOnDeposit'])], axis=1)
merged_train = pd.concat([merged_train, dum_sign(merged_train['Occupation'])], axis=1)
merged_train = pd.concat([merged_train, dum_sign(merged_train['Region'])], axis=1)
merged_train.drop(columns=[ "DaysOnDeposit_others", "Occupation" ], inplace=True)
merged_train.head()
merged_train.drop(columns=["RegistrationDate" , "UpsellDate" , "Region" , "Town"  , "ExpectedTermDate" , "FirstPaymentDate" , "LastReceivedPayment", "TransactionDates", "PaymentsHistory"], inplace=True)
merged_train.head()
y_true_1 = merged_train.m1.values
y_true_2 = merged_train.m2.values
y_true_3 = merged_train.m3.values
y_true_4 = merged_train.m4.values
y_true_5 = merged_train.m5.values
y_true_6 = merged_train.m6.values
merged_train.drop(columns=["m1", "m2" ,"m3", "m4", "m5", "m6"], inplace=True)
merged_train.head()
merged_train.drop(columns=["RegisteredInLeapYear" ], inplace=True)
merged_train.head()
merged_train.drop(columns=["RegisteredAtMonthStart", "RegisteredAtMonthEnd" ], inplace=True)
merged_train.head()
merged_train['Age'] = np.log(merged_train['Age']+1)
merged_train['RemainingPayment'] = np.log(merged_train['RemainingPayment']+1)
plt.plot(merged_train['RemainingPayment'])
plt.plot(merged_train['Age'])
########################### Model params
#################################################################################
lgb_params = {
                    'boosting_type': 'gbdt',
                    'objective': 'tweedie',
                    'tweedie_variance_power': 1.1,
                    'metric': 'rmse',
                    'subsample': 0.5,
                    'subsample_freq': 1,
                    'learning_rate': 0.015,
                    'num_leaves': 2**8-1,
                    'min_data_in_leaf': 2**8-1,
                    'feature_fraction': 0.5,
                    'max_bin': 100,
                    'n_estimators': 1000,
                    'boost_from_average': False,
                    'verbose': 1,
                    'seed' : 1995, 
                    'random_state' : 21,
                    'is_unbalance': True,
                    'max_depth' : 7
                }
# Hyperparameter grid
param_grid = {
    'boosting_type': ['gbdt', 'dart'],
    'num_leaves': list(range(20, 2**8-1)),
    'learning_rate': list(np.logspace(np.log10(0.005), np.log10(0.5), base = 10, num = 1000)),
    'subsample_for_bin': list(range(20000, 300000, 20000)),
    'min_child_samples': list(range(20, 500, 5)),
    'reg_alpha': list(np.linspace(0, 1)),
    'reg_lambda': list(np.linspace(0, 1)),
    'colsample_bytree': list(np.linspace(0.6, 1, 10)),
    'subsample': list(np.linspace(0.5, 1, 100)),
    'is_unbalance': [True, False]
}
MAX_EVALS = 5
N_FOLDS = 5
grid_results = pd.DataFrame(columns = ['score', 'params', 'iteration'],
                              index = list(range(MAX_EVALS)))
com = 1
for x in param_grid.values():
    com *= len(x)
print('There are {} combinations'.format(com))
import itertools

def grid_search(param_grid, max_evals = MAX_EVALS):
    """Grid search algorithm (with limit on max evals)"""
    
    # Dataframe to store results
    results = pd.DataFrame(columns = ['score', 'params', 'iteration'],
                              index = list(range(MAX_EVALS)))
    
    # https://codereview.stackexchange.com/questions/171173/list-all-possible-permutations-from-a-python-dictionary-of-lists
    keys, values = zip(*param_grid.items())
    
    i = 0
    
    # Iterate through every possible combination of hyperparameters
    for v in itertools.product(*values):
        
        # Create a hyperparameter dictionary
        hyperparameters = dict(zip(keys, v))
        
        # Set the subsample ratio accounting for boosting type
        hyperparameters['subsample'] = 1.0 if hyperparameters['boosting_type'] == 'goss' else hyperparameters['subsample']
        
        # Evalute the hyperparameters
        eval_results = objective(hyperparameters, i)
        
        results.loc[i, :] = eval_results
        
        i += 1
        
        # Normally would not limit iterations
        if i > MAX_EVALS:
            break
       
    # Sort with best score on top
    results.sort_values('score', ascending = False, inplace = True)
    results.reset_index(inplace = True)
    
    return results  

def objective(hyperparameters, iteration):
    """Objective function for grid and random search. Returns
       the cross validation score from a set of hyperparameters."""
    
    # Number of estimators will be found using early stopping
    if 'n_estimators' in hyperparameters.keys():
        del hyperparameters['n_estimators']
    
     # Perform n_folds cross validation
    cv_results = lgb.cv(hyperparameters, lgbtrain_all, num_boost_round = 100000, nfold = N_FOLDS, 
                        early_stopping_rounds = 500, metrics = 'auc', seed = 42)
    
    # results to retun
    score = cv_results['auc-mean'][-1]
    estimators = len(cv_results['auc-mean'])
    hyperparameters['n_estimators'] = estimators 
    
    return [score, hyperparameters, iteration]
grid_results = grid_search(param_grid)

print('The best validation score was {:.5f}'.format(grid_results.loc[0, 'score']))
print('\nThe best hyperparameters were:')

import pprint
pprint.pprint(grid_results.loc[0, 'params'])
import lightgbm as lgb
# LightGBM dataset
y_true_train_1 = y_true_6[:-1000]
merged_train_1_train = merged_train[:27007]
lgbtrain_all = lgb.Dataset(data=merged_train_1_train.loc[:len(merged_train_1_train),merged_train_1_train.columns[1:]].values, 
                           label=y_true_train_1)
model = lgb.train(lgb_params, train_set = lgbtrain_all)
test_preds = model.predict(merged_train.loc[27007:,merged_train.columns[1:]].values)
test_preds[:50]
from sklearn.experimental import enable_hist_gradient_boosting  # noqa
from sklearn.ensemble import HistGradientBoostingRegressor
model = HistGradientBoostingRegressor(l2_regularization=1, learning_rate=0.0015,
                              loss='least_squares', max_bins=255,
                              max_depth=7, max_iter=10000, max_leaf_nodes=2**8-1,
                              min_samples_leaf=20, n_iter_no_change=None,
                              random_state=5, scoring=None, tol=1e-07,
                              validation_fraction=0.1, verbose=1,
                              warm_start=False)

y_true_train_1 = y_true_1[:-1000]
merged_train_1_train = merged_train[:27007]
model.fit(merged_train_1_train.loc[:,merged_train.columns[1:]], y_true_train_1)
test_preds_hist = model.predict(merged_train.loc[27007:,merged_train.columns[1:]].values)
test_preds_hist[:50]
from sklearn.ensemble import GradientBoostingRegressor
model = GradientBoostingRegressor(n_estimators=2000, max_depth=6, min_samples_leaf=1,
                                           min_samples_split=2, learning_rate=0.012, loss='huber',
                                           random_state=5, verbose = 1)
y_true_train_1 = y_true_1[:-1000]
merged_train_1_train = merged_train[:27007]
model.fit(merged_train_1_train.loc[:,merged_train.columns[1:]], y_true_train_1)
test_preds_gbr = model.predict(merged_train.loc[27007:,merged_train.columns[1:]].values)
test_preds_gbr[:50]
merged_train_1_val = merged_train[27007:]
merged_train_1_val = scaler.transform(merged_train_1_val.loc[:,merged_train.columns[1:]])
test_preds_gbr = model.predict(merged_train_1_val)
test_preds_gbr[:50]
import xgboost as xgb
y_true_train_1 = y_true_3[:-1000]
merged_train_1_train = merged_train[:27007]
xgbtrain = xgb.DMatrix(merged_train_1_train.loc[:len(merged_train_1_train),merged_train_1_train.columns[1:]].values, y_true_train_1)
param = {'max_depth':10, 
         'subsample':1,
         'min_child_weight':0.4,
         'eta':0.3, 
         'num_round':1000, 
         'seed':1995,
         'silent':0,
         'eval_metric':'rmse'} # random parameters
model = xgb.train(param, xgbtrain)
model
xgbpredict = xgb.DMatrix(merged_train.loc[27007:,merged_train.columns[1:]].values)
test_preds_xgb = model.predict(xgbpredict)
from sklearn.metrics import mean_squared_error
mean_squared_error(y_true_1[27007:], test_preds_gbr, squared=False)
best = ['gb - 1068, lgb - 1063.4','lgb - 376 , gb -390', ' ', '', '', 'gb - 1396']
for i in range(0,len(test_preds)):
    test_preds[i] = (test_preds_xgb[i] + test_preds[i])/2
mean_squared_error(y_true_3[27007:], test_preds, squared=False)
for i in range(0,len(test_preds)):
    test_preds[i] = (test_preds_xgb[i] + test_preds[i])/2
mean_squared_error(y_true_6[27007:], test_preds, squared=False)
for i in range (0,1000):
  if test_preds[i] >1200 :
    test_preds[i] = 1500
plt.plot(test_preds)
# LightGBM dataset
lgbtrain_all = lgb.Dataset(data=merged_train.loc[:,merged_train.columns].values, 
                           label=y_true_1, 
                           feature_name = merged_train.columns)
cols = merged_train.columns
merged_train.loc[:,cols].values
merged_test = pd.merge(left=test, right=pd.concat(objs=[merged_]).fillna(value=0.), on="ID")
merged_test = pad_history_most_recent(df=merged_test, max_len=5)
merged_test.drop(columns=["RegistrationDate" , "UpsellDate" , "Region" , "Town" , "Occupation" , "ExpectedTermDate" , "FirstPaymentDate" , "LastReceivedPayment", "TransactionDates", "PaymentsHistory"], inplace=True)
merged_test.drop(columns=["RegisteredInLeapYear" ], inplace=True)
merged_test.head()
merged_test.drop(columns=["RegisteredAtMonthStart", "RegisteredAtMonthEnd" ], inplace=True)
merged_test.head()
columnsTitles = merged_train.columns

merged_test = merged_test.reindex(columns=columnsTitles)
merged_test.head()
from sklearn.ensemble import GradientBoostingRegressor
model_1 = GradientBoostingRegressor(n_estimators=3000, max_depth=8, min_samples_leaf=1,
                                           min_samples_split=2, learning_rate=0.015, loss='huber',
                                           random_state=5, verbose = 1)
model_1.fit(merged_train.loc[:,merged_train.columns[1:]], y_true_1)

model_2 = GradientBoostingRegressor(n_estimators=3000, max_depth=8, min_samples_leaf=1,
                                           min_samples_split=2, learning_rate=0.015, loss='huber',
                                           random_state=5, verbose = 1)
model_2.fit(merged_train.loc[:,merged_train.columns[1:]], y_true_2)

model_3 = GradientBoostingRegressor(n_estimators=3000, max_depth=8, min_samples_leaf=1,
                                           min_samples_split=2, learning_rate=0.015, loss='huber',
                                           random_state=5, verbose = 1)
model_3.fit(merged_train.loc[:,merged_train.columns[1:]], y_true_3)

model_4 = GradientBoostingRegressor(n_estimators=3000, max_depth=8, min_samples_leaf=1,
                                           min_samples_split=2, learning_rate=0.015, loss='huber',
                                           random_state=5, verbose = 1)
model_4.fit(merged_train.loc[:,merged_train.columns[1:]], y_true_4)

model_5 = GradientBoostingRegressor(n_estimators=3000, max_depth=8, min_samples_leaf=1,
                                           min_samples_split=2, learning_rate=0.015, loss='huber',
                                           random_state=5, verbose = 1)
model_5.fit(merged_train.loc[:,merged_train.columns[1:]], y_true_5)

model_6 = GradientBoostingRegressor(n_estimators=3000, max_depth=8, min_samples_leaf=1,
                                           min_samples_split=2, learning_rate=0.015, loss='huber',
                                           random_state=5, verbose = 1)
model_6.fit(merged_train.loc[:,merged_train.columns[1:]], y_true_6)
test_preds_1 = model_1.predict(merged_test.loc[:,merged_test.columns[1:]].values)
test_preds_2 = model_2.predict(merged_test.loc[:,merged_test.columns[1:]].values)
test_preds_3 = model_3.predict(merged_test.loc[:,merged_test.columns[1:]].values)
test_preds_4 = model_4.predict(merged_test.loc[:,merged_test.columns[1:]].values)
test_preds_5 = model_5.predict(merged_test.loc[:,merged_test.columns[1:]].values)
test_preds_6 = model_6.predict(merged_test.loc[:,merged_test.columns[1:]].values)
merged_test.loc[:,merged_test.columns[1:]].values
np.mean(test_preds_1)
plt.plot(test_preds_1)
plt.plot(test_preds_2)
plt.plot(test_preds_3)
plt.plot(test_preds_4)
plt.plot(test_preds_5)
plt.plot(test_preds_6)

test_preds = []
for i in range (0,len(test_preds_1)):
    test_preds.append(test_preds_1[i])
    test_preds.append(test_preds_2[i])
    test_preds.append(test_preds_3[i])
    test_preds.append(test_preds_4[i])
    test_preds.append(test_preds_5[i])
    test_preds.append(test_preds_6[i])
len(test_preds)
ttt=0
for i in range(0,len(test_preds)):
    if test_preds[i] > 4000:
      test_preds[i] = min(test_preds[i],merged_test["PaymentsHistory_1"][int(i/6)])
      ttt = ttt+1
ttt
ss["Target"] = test_preds
ss.head(20)
#ss.to_csv('baseline_xsub.csv', index=False)
y_true_7 = np.concatenate((y_true_1,y_true_2,y_true_3,y_true_4,y_true_5,y_true_6))
y_true_7 = np.reshape(y_true_7, (6,len(merged_train))).T
y_true_7
y_true_7.shape
y_true_7 = np.append(y_true_1,y_true_2,y_true_3,y_true_4,y_true_5,y_true_6)
y_true_7 = np.append(y_true_7,y_true_3)
y_true_7 = np.append(y_true_7,y_true_4)
y_true_7 = np.append(y_true_7,y_true_5)
y_true_7 = np.append(y_true_7,y_true_6)
y_true_7
selected_features = [c for c in merged_train.columns[1:] if "Date" not in c]
xtrain_loan = merged_train[selected_features]
ytrain_loan = y_true_7
## Split
xtrain_loan, xval_loan, ytrain_loan, yval_loan = train_test_split(
    xtrain_loan, 
    ytrain_loan, 
    test_size=.3, 
    random_state=21, 
    shuffle=True
)
scaler = StandardScaler()
##Fit scaler
scaler.fit(X=xtrain_loan.values)

## Scale data
xtrain_loan = scaler.transform(X=xtrain_loan.values)
xval_loan = scaler.transform(X=xval_loan.values)

print(f"> Training inputs shape : {xtrain_loan.shape}, Training targets shape : {ytrain_loan.shape}")
print(f"> Validation inputs shape : {xval_loan.shape}, Training targets shape : {yval_loan.shape}")
train, max_len = pad_history(df=train, max_len=41)

xtrain = train.PaymentsHistory.values
ytrain = train[train.columns[-6:]].values

xs = np.array(xtrain.tolist()).reshape((xtrain.shape[0], 1, max_len))
ys = np.array(ytrain.tolist()).reshape((ytrain.shape[0], 1, 6))

train_ds = tf.data.Dataset.from_tensor_slices((xs, ys))
merged2 = pd.concat([train, metadata], axis=1, join="inner")
merged2 = pd.merge(left=train, right=pd.concat(objs=[metadata]).fillna(value=0.), on="ID")
merged2
xtrain = merged2.PaymentsHistory.values
deposits = merged2.Deposit.values
total_contract_value = merged2.TotalContractValue.values
for i in range (0,len(merged2)):
  xtrain[i].append(total_contract_value[i]-deposits[i]-np.sum(xtrain[i]))
xs = np.array(xtrain.tolist()).reshape((xtrain.shape[0], 1, max_len+1))
ys = np.array(ytrain.tolist()).reshape((ytrain.shape[0], 1, 6))
train_ds = tf.data.Dataset.from_tensor_slices((xs, ys))
xtrain[2][41] 
i=0
np.sum(xtrain[i])<jupyter_output><empty_output><jupyter_text>Modelling<jupyter_code>def build_model():
    
    payments_inp = keras.Input(shape=(41), name="payments_inputs")
    status_inp = keras.Input(shape=(xtrain_loan.shape[1]), name="status_inputs")
    
    x1 = keras.layers.Dense(35, activation="sigmoid", name="x1")(status_inp)
    out1 = keras.layers.Dense(1, name="loan_status_out")(x1)
    
    
    x2 = keras.layers.Dense(20, activation="sigmoid", name="x2")(payments_inp)
    x2_out = keras.layers.Dense(30, activation="sigmoid", name="payment_out")(x2)
    
    concat = keras.layers.Concatenate(name="concat_layer")([out1, x2_out])
    out2 = keras.layers.Dense(6, name="out")(concat)
    
    # Model
    model = keras.Model(inputs=[payments_inp, status_inp], outputs=[out1, out2])
    
    opt = keras.optimizers.Adam(learning_rate=0.01)
    wgt = {"out1":.7, "out2":.3}
    
    model.compile(
        loss="mean_squared_error",
        optimizer=opt,
        loss_weights=wgt
    )

    model.summary()
    
    return model


def build_simple_model():
    
    payments_inp = keras.Input(shape=(35), name="payments_inputs")

    x = keras.layers.Dense(30, activation="sigmoid", name="x2")(payments_inp)
    x2 = keras.layers.Dense(40, activation="sigmoid", name="x3")(x)
    x_out = keras.layers.Dense(30, activation="sigmoid", name="payment_out")(x2)
    
    out = keras.layers.Dense(6, name="out")(x_out)
    
    # Model
    model = keras.Model(inputs=[payments_inp], outputs=[out])
    
    opt = keras.optimizers.Adam(learning_rate=0.01)
    
    model.compile(
        loss="mean_squared_error",
        optimizer=opt,
    )

    model.summary()
    
    return model


def build_status_model():
    
    status_inp = keras.Input(shape=(xtrain_loan.shape[1]), name="status_inputs")

    x = keras.layers.Dense(20, activation="sigmoid", name="x2")(status_inp)
    x_out = keras.layers.Dense(30, activation="sigmoid", name="statut_out")(x)
    
    out = keras.layers.Dense(6, name="out")(x)
    
    # Model
    model = keras.Model(inputs=[status_inp], outputs=[out])
    
    opt = keras.optimizers.Adam(learning_rate=0.01)
    
    model.compile(
        loss="mean_squared_error",
        optimizer=opt,
    )

    model.summary()
    
    return model
model = build_simple_model()
tf.compat.v1.disable_eager_execution()

# model
#inputs = tf.keras.layers.Input((6*24*7, 13))
inputs = tf.keras.layers.Input((35,1), name="payments_inputs")
#inputs = tf.keras.layers.BatchNormalization(inputs),
conv1 = tf.keras.layers.Conv1D(200, kernel_size=6, strides=1, activation='relu')(inputs)
#conv1 = tf.keras.layers.BatchNormalization()(conv1),
trim1 = tf.keras.layers.Cropping1D((5, 0))(conv1)
conv2 = tf.keras.layers.Conv1D(50, kernel_size=6, strides=1, activation='relu')(trim1)
trim2 = tf.keras.layers.Cropping1D((1, 0))(conv2)
conv3 = tf.keras.layers.Conv1D(30, kernel_size=6, strides=1, activation='relu')(trim2)
trim3 = tf.keras.layers.Cropping1D((5, 0))(conv3)
conv4 = tf.keras.layers.Conv1D(30, kernel_size=6, strides=1, activation='relu')(trim3)
conv5 = tf.keras.layers.Conv1D(30, kernel_size=4, strides=1, activation='relu')(conv4)
dense = tf.keras.layers.Dense(50, activation='relu')(conv5)
output = tf.keras.layers.Flatten()(tf.keras.layers.Dense(6)(dense))
model = tf.keras.Model(inputs, output)
# Model
#model = tf.keras.Model(inputs=[payments_inp], outputs=[out])
    
model.compile(
        loss=tf.keras.losses.MeanSquaredError(),
        optimizer=tf.keras.optimizers.Adam(learning_rate=.001),
        metrics=[tf.keras.metrics.RootMeanSquaredError()],
    )

model.summary()
tf.keras.utils.plot_model(
        model,
        to_file="../model.png", 
        show_shapes=True
    )<jupyter_output><empty_output><jupyter_text>Learning phase<jupyter_code>model.fit(xtrain_loan.reshape((xtrain_loan.shape[0], xtrain_loan.shape[1], 1)), ytrain_loan, batch_size = 8, epochs=250, validation_data=(xval_loan.reshape((xval_loan.shape[0], xval_loan.shape[1], 1)), yval_loan))
preds = model.predict(xval_loan.reshape((xval_loan.shape[0], xval_loan.shape[1], 1)))
preds
yval_loan
xs = np.array(xtrain.tolist()).reshape((xtrain.shape[0], max_len+1,1))
ys = np.array(ytrain.tolist()).reshape((ytrain.shape[0],  6))
ys.shape
#model.compile(optimizer, loss)
model.fit(xs[:len(xs)], ys[:len(ys)], batch_size=32, epochs=20)
#model.fit(train_ds, batch_size=32, epochs=20)<jupyter_output><empty_output><jupyter_text>Eval/Inference time<jupyter_code>def predict(test:pd.DataFrame):
    test, max_len = pad_history(df=test, max_len=41)

    xtest = test.PaymentsHistory.values
    xtest = np.array(xtest.tolist()).reshape((xtest.shape[0], 1, max_len))

    merged_test2 = pd.merge(left=test, right=pd.concat(objs=[metadata]).fillna(value=0.), on="ID")
    xtest = merged_test2.PaymentsHistory.values
    
    deposits = merged_test2.Deposit.values
    total_contract_value = merged_test2.TotalContractValue.values

    w = []
    for i in range (0,len(merged_test2)):
        xtest[i].append(total_contract_value[i]-deposits[i]-np.sum(xtest[i]))
        w.append(xtest[i][-1])
        #print(111, w, xtest[i][-1])
    
    print(w)
    
    xts = np.array(xtest.tolist()).reshape((xtest.shape[0],  max_len+1, 1))
    print(xts)

    test_ds = tf.data.Dataset.from_tensor_slices(xts)
    preds = model.predict(xts)

    print(preds)
    
    assert preds.shape[0] == test.shape[0], f"Shape mismatch, {preds.shape[0]} predictions found while test set has {test.shape[0]} rows"
    
    return preds.flatten(),w
merged_test2 = pd.merge(left=test, right=pd.concat(objs=[metadata]).fillna(value=0.), on="ID")
merged_test2
xtest = merged_test2.PaymentsHistory.values
deposits = merged_test2.Deposit.values
total_contract_value = merged_test2.TotalContractValue.values
i=0
#total_contract_value[i]-deposits[i]-np.sum(xtest[i])
xtest = xtest.reshape((xtest.shape[0])).tolist()
#np.sum(xtest)
type(xtest[0])
preds,w = predict(test=test)
w[:10]
t = 0
for k in range(0,int(len(preds)/6)):
  sum = 0
  for i in range(0,6):
    sum = sum+preds[t]
    t = t+1
  for i in range(0,6):
    preds[t-6+i] = min(max(100,(w[k]*preds[t-6+i])/sum),1200)
preds = predict(test=test)<jupyter_output><empty_output><jupyter_text>Create submission<jupyter_code>ss["Target"] = preds
ss.to_csv('baseline_sub.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14111157435617844, 'fraction_numerical': 0.02758089862416215, 'mean_word_length': 3.936985433818873, 'pattern_counts': {'":': 2, '<': 46, '<?xml version=': 0, '>': 64, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '505405', 'license': 'MIT', 'n_tokens_mistral': 11682, 'n_tokens_neox': 10994, 'n_words': 1712, 'path': 'main.ipynb', 'repo_name': 'rishabh30/sfc_solar_pay_go', 'size': '31190'}