<jupyter_start><jupyter_text>Main Notebook<jupyter_code>import os
import gc
import pytz
import operator
import numpy as np
import pickle as pkl
import matplotlib as mtp
from time import sleep
from datetime import datetime
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from sklearn import preprocessing
from keras.wrappers.scikit_learn import KerasRegressor
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
import finn_data
import tensorflow as tf
from tensorflow import math
import keras
from keras import layers
from keras.models import Sequential
from keras.activations import relu, elu
from keras.layers import Dense, Dropout
from talos.model import early_stopper
from talos.utils.best_model import activate_model
from talos import Evaluate
import talos
from talos import Reporting
from talos import Deploy
from talos import Restore
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
import finn_data
import warnings
warnings.filterwarnings('ignore')
warnings.filterwarnings(action = 'ignore', category = FutureWarning)
warnings.filterwarnings(action = 'ignore', category = DeprecationWarning)

import pandas as pd
pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)

import sys
sys.path.append('..')

time = datetime.now(pytz.timezone('Europe/Oslo')).strftime('%m.%d.%Y_%H.%M.%S')
print(f'Notebook initialized execution at {time}.')
#import xgboost as xgb<jupyter_output>Notebook initialized execution at 04.21.2020_12.43.08.<jupyter_text>General Methods<jupyter_code>def memory_optimization(dfs):
    for df in dfs:
        del df
    gc.collect()<jupyter_output><empty_output><jupyter_text>Prepare Data<jupyter_code>import datasets

start_time = datetime.now()
scaler = MinMaxScaler()
train_x, train_y, validation_x, validation_y, test_x, test_y, scaler = datasets.load(f'../input/hele_norge.csv', scaler)<jupyter_output>cleaning removed 1.06 % of the original values<jupyter_text>Optimization Parameters<jupyter_code>parameters = {'activation_1':['relu', 'elu'],
     'activation_2':['relu', 'elu'],
     'activation_3':['relu', 'elu'],
     'optimizer': ['Adam', "RMSprop"],
     'loss-functions': ['mse'],
     'neurons_HL1': [50, 100, 200, 400],
     'neurons_HL2': [40, 80, 160, 320],
     'neurons_HL3': [40, 80, 160, 320, None],
     'dropout1': [0.1, 0.2, 0.3],
     'dropout2': [0.1, 0.2, 0.3],
     'batch_size': [100, 250, 500],
     'epochs': [400, 900]
}
def talolos(x_train, y_train, x_val, y_val, parameters):
    model = Sequential()

    model.add(Dense(parameters['neurons_HL1'], 
    input_shape=(train_x.shape[1],), 
    activation=parameters['activation_1'],use_bias=True))

    model.add(Dropout(parameters['dropout1']))

    model.add(Dense(parameters['neurons_HL2'], 
    activation=parameters['activation_2'], use_bias=True))

    model.add(Dropout(parameters['dropout1']))
    
    if parameters['neurons_HL3']:
        model.add(Dense(parameters['neurons_HL3'], 
        activation=parameters['activation_3'], use_bias=True))


    model.add(Dense(1, activation='relu'))

    model.compile(optimizer=parameters['optimizer'], loss=parameters['loss-functions'], 
    metrics=['mse', 'mae'])

    history = model.fit(x_train, y_train,
            batch_size=parameters['batch_size'],epochs=parameters['epochs'],
            verbose=0,validation_data=[x_val, y_val],
            callbacks=[early_stopper(epochs=parameters['epochs'], 
            mode='moderate',monitor='val_loss', patience=1)])
    
    return history, model<jupyter_output><empty_output><jupyter_text>Specify model<jupyter_code>t = talos.Scan(x=np.array(train_x),
               y=np.array(train_y),
               x_val=np.array(validation_x),
               y_val=np.array(validation_y),
               model=talolos,
               params=parameters,
               experiment_name='oloo',
             round_limit=2)
def inverse_transform(scaler, value):
    mat = np.zeros((1, scaler.scale_.shape[0]))
    mat[0, 0] = value
    return scaler.inverse_transform(mat)[:,0]<jupyter_output><empty_output><jupyter_text>Evaluate<jupyter_code>def evaluate(scan_model, test_x, test_y):
    eval_model = Evaluate(scan_model)
    results = eval_model.evaluate(np.array(test_x), np.array(test_y), task='continuous',folds=10, metric='loss')
    return np.array([inverse_transform(scaler,result) for result in results])<jupyter_output><empty_output><jupyter_text>Results<jupyter_code>#t.data
results = evaluate(t, test_x, test_y)
print(results)<jupyter_output><empty_output><jupyter_text>Retrieve old parameters (Last inn 2000-kjoring.csv fra drive)<jupyter_code>#Ligger på drive:
res = Reporting('../input/2000_kjoring.csv')
best = res.data.sort_values('val_mae', ascending=True).iloc[0]
best = pd.DataFrame(best)

def extract_param(df, key):
    return df.loc[key].values[0]
parameters = {'activation_1': extract_param(best,'activation_1'),
 'activation_2': extract_param(best,'activation_2'),
 'activation_3': extract_param(best,'activation_3'),
 'optimizer': extract_param(best,'optimizer'),
 'loss-functions': extract_param(best,'loss-functions'),
 'neurons_HL1': extract_param(best,'neurons_HL1'),
 'neurons_HL2': extract_param(best,'neurons_HL2'),
 'neurons_HL3': extract_param(best,'neurons_HL3'),
 'dropout1': extract_param(best,'dropout1'),
 'dropout2': extract_param(best,'dropout2'),
 'batch_size': extract_param(best,'batch_size'),
 'epochs': extract_param(best,'epochs')}
def best_model():
    parameters = {'activation_1': extract_param(best,'activation_1'),
     'activation_2': extract_param(best,'activation_2'),
     'activation_3': extract_param(best,'activation_3'),
     'optimizer': extract_param(best,'optimizer'),
     'loss-functions': extract_param(best,'loss-functions'),
     'neurons_HL1': extract_param(best,'neurons_HL1'),
     'neurons_HL2': extract_param(best,'neurons_HL2'),
     'neurons_HL3': extract_param(best,'neurons_HL3'),
     'dropout1': extract_param(best,'dropout1'),
     'dropout2': extract_param(best,'dropout2'),
     'batch_size': extract_param(best,'batch_size'),
     'epochs': extract_param(best,'epochs')
    }
    model = Sequential()

    model.add(Dense(parameters['neurons_HL1'], 
    input_shape=(train_x.shape[1],), 
    activation=parameters['activation_1'],use_bias=True))

    model.add(Dropout(parameters['dropout1']))

    model.add(Dense(parameters['neurons_HL2'], 
    activation=parameters['activation_2'], use_bias=True))

    model.add(Dropout(parameters['dropout1']))
    
    if parameters['neurons_HL3'] != 'None':
        model.add(Dense(parameters['neurons_HL3'], 
        activation=parameters['activation_3'], use_bias=True))


    model.add(Dense(1, activation='relu'))

    model.compile(optimizer=parameters['optimizer'], loss=parameters['loss-functions'], 
    metrics=['mse', 'mae'])
    
    return model<jupyter_output><empty_output><jupyter_text>Train retrieved model<jupyter_code>my_model = KerasRegressor(build_fn=best_model,epochs=300, 
    batch_size=parameters['batch_size'], verbose=2)    
my_model.fit(train_x, train_y)<jupyter_output>WARNING:tensorflow:From C:\Users\jonas\AppData\Roaming\Python\Python37\site-packages\keras\backend\tensorflow_backend.py:422: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.

Epoch 1/300
 - 1s - loss: 0.0186 - mse: 0.0186 - mae: 0.1073
Epoch 2/300
 - 0s - loss: 0.0103 - mse: 0.0103 - mae: 0.0780
Epoch 3/300
 - 0s - loss: 0.0079 - mse: 0.0079 - mae: 0.0659
Epoch 4/300
 - 0s - loss: 0.0067 - mse: 0.0067 - mae: 0.0584
Epoch 5/300
 - 1s - loss: 0.0057 - mse: 0.0057 - mae: 0.0534
Epoch 6/300
 - 0s - loss: 0.0054 - mse: 0.0054 - mae: 0.0511
Epoch 7/300
 - 0s - loss: 0.0051 - mse: 0.0051 - mae: 0.0489
Epoch 8/300
 - 0s - loss: 0.0049 - mse: 0.0049 - mae: 0.0475
Epoch 9/300
 - 0s - loss: 0.0047 - mse: 0.0047 - mae: 0.0460
Epoch 10/300
 - 0s - loss: 0.0045 - mse: 0.0045 - mae: 0.0453
Epoch 11/300
 - 0s - loss: 0.0045 - mse: 0.0045 - mae: 0.0445
Epoch 12/300
 - 1s - loss: 0.0044 - mse: 0.0044 - mae: 0.0442
Epoch 13/300
 - 0s - loss: 0.0043 - mse: 0.0[...]<jupyter_text>Find highest contributing features<jupyter_code>from eli5.sklearn import PermutationImportance
import eli5

perm = PermutationImportance(my_model, random_state=1).fit(train_x,train_y)
weight_df = eli5.explain_weights_df(perm, feature_names = train_x.columns.tolist(), )
largest_10 = weight_df.nlargest(10,['weight'])
largest_10['feature']
vals= np.array(weight_df['weight'])
model = best_model()
    history = model.fit(train_x, train_y,
            batch_size=parameters['batch_size'],epochs=parameters['epochs'],
            verbose=0,validation_data=[validation_x, validation_y],
            callbacks=[early_stopper(epochs=parameters['epochs'], 
            mode='moderate',monitor='val_loss', patience=25)])
def evaluate_prediction(predictions, test_y): 
    """ prints the importances of features 
    Args:
        predictions: predictions for tast set
        test_y:      test set labels
    Returns:
    """
    test_evaluation = {}
    test_evaluation['pred'] = predictions
    test_evaluation['benchmark'] = test_y.mean()
    test_evaluation['target'] = test_y.reset_index(drop=True)
    test_evaluation['difference'] = test_evaluation['pred'] - test_evaluation['target']
    test_evaluation['bench difference'] = test_evaluation['benchmark'] - test_evaluation['target']
    test_evaluation['abs difference'] = abs(test_evaluation['difference'])
    test_evaluation['abs bench difference'] = abs(test_evaluation['bench difference'])
    test_evaluation['difference %'] = (test_evaluation['pred'] / test_evaluation['target'] - 1) * 100
    test_evaluation['bench difference %'] = abs((test_evaluation['benchmark'] / test_evaluation['target'] - 1) * 100)
    
    mean = int(test_evaluation['abs difference'].mean())
    bench_mean = int(test_evaluation['abs bench difference'].mean())
    mean_perc = round(abs(test_evaluation['difference %']).mean(), 2)
    bench_mean_perc = round(abs(test_evaluation['bench difference %']).mean(), 2)
    print('Model evaluation campared to mean benchmark:', test_y.median())
    print(f'| mean abs.  difference | our model: {mean}  benchmark: {bench_mean}')
    print(f'| mean abs % difference | our model: {mean_perc} %  benchmark: {bench_mean_perc} %')
    print()
    
    return
from sklearn.externals import joblib

def inverse_transform_df(data):
    #Retrieve scaler:
    scaler = joblib.load('../talos_training/hele_norge.scaler')
    return np.array([inverse_transform(scaler,result) for result in data])
print(test_y)
print(predictions)
predictions = model.predict(validation_x)
    #Formatting:
predictions = pd.Series(np.ndarray.flatten(predictions))
prices = pd.Series(np.ndarray.flatten(inverse_transform_df(test_y)))
evaluate_prediction(predictions, prices)
features = ['boligtype_Leilighet','boligtype_Enebolig','bruksareal','boligtype_Tomannsbolig','postnummer','boligtype_Rekkehus', 
        'neighborhood_environment_demographics_housingage_10-30','neighborhood_environment_demographics_housingprices_0-2000000','neighborhood_environment_demographics_housingage_30-50',
        'eieform_Andel']<jupyter_output><empty_output><jupyter_text>10-feature model training (Ferdigtrent, lastes inn lenger ned)<jupyter_code>features = ['boligtype_Leilighet','boligtype_Enebolig','bruksareal','boligtype_Tomannsbolig','postnummer','boligtype_Rekkehus', 
'neighborhood_environment_demographics_housingage_10-30','neighborhood_environment_demographics_housingprices_0-2000000','neighborhood_environment_demographics_housingage_30-50',
'eieform_Andel']
train_x = train_x[features]
validation_x = validation_x[features]
test_x = test_x[features] 

parameters = {'activation_1':['relu', 'elu', 'sigmoid', 'tanh'],
 'activation_2':['relu', 'elu', 'sigmoid', 'tanh'],
 'activation_3':['relu', 'elu', 'sigmoid', 'tanh'],
 'optimizer': ['Adam', "RMSprop", 'sgd', 'Nadam'],
 'loss-functions': ['mse'],
 'neurons_HL1': [5, 10, 20, 50, 100],
 'neurons_HL2': [5, 10, 20, 50, 100],
 'neurons_HL3': [5, 10, 20, 50, 100, None],
 'dropout1': [0.1, 0.2, 0.3],
 'dropout2': [0.0, 0.1, 0.2, 0.3],
 'batch_size': [100, 250, 500],
 'epochs': [3000]
}
    



def talolos(x_train, y_train, x_val, y_val, parameters):
    model = Sequential()

    model.add(Dense(parameters['neurons_HL1'],
                    input_shape=(train_x.shape[1],),
                    activation=parameters['activation_1'], use_bias=True))

    model.add(Dropout(parameters['dropout1']))
    model.add(Dense(parameters['neurons_HL2'], activation=parameters['activation_2'], use_bias=True))
    model.add(Dropout(parameters['dropout1']))

    if parameters['neurons_HL3']:
        model.add(Dense(parameters['neurons_HL3'], activation=parameters['activation_3'], use_bias=True))

    model.add(Dense(1, activation='relu'))
    model.compile(optimizer=parameters['optimizer'], loss=parameters['loss-functions'], metrics=['mse', 'mae'])

    history = model.fit(x_train, y_train,
                        batch_size=parameters['batch_size'], epochs=parameters['epochs'],
                        verbose=0, validation_data=[x_val, y_val],
                        callbacks=[early_stopper(epochs=parameters['epochs'],
                                                 mode='moderate', monitor='val_loss', patience=40)])

    return history, model

t = talos.Scan(x=np.array(train_x),
               y=np.array(train_y),
               x_val=np.array(validation_x),
               y_val=np.array(validation_y),
               model=talolos,
               params=parameters,
               experiment_name='talos_training',
               round_limit=10)<jupyter_output><empty_output><jupyter_text>Best model<jupyter_code>best_model_10 = t.best_model(metric='loss', asc=True)
best_model_10.save(f'../talos_training/best_model_10.h5')<jupyter_output><empty_output><jupyter_text>Load 10-best model<jupyter_code>from keras.models import load_model
best_model_10 = load_model(f'../talos_training/best_model_10.h5')<jupyter_output><empty_output><jupyter_text>Predict<jupyter_code>predictions = pd.DataFrame()
predictions['pred'] = my_model.predict(test_x)
print(predictions['pred'])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14537166900420756, 'fraction_numerical': 0.0408835904628331, 'mean_word_length': 5.012225969645868, 'pattern_counts': {'":': 0, '<': 54, '<?xml version=': 0, '>': 54, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '406533', 'license': 'MIT', 'n_tokens_mistral': 5367, 'n_tokens_neox': 4907, 'n_words': 824, 'path': 'notebooks/main_keras.ipynb', 'repo_name': 'trygve55/Housing-Price-Estimator', 'size': '14260'}