<jupyter_start><jupyter_text>Community Learning> In diesem Notebook wird ein Ensemble von XGboost Modellen erstellt um die Prediction zu verbessern. Nachdem wir nun das Basismodel bestimmt haben. Werden wir die Daten möglichst nach Regionen aufteilen und für jede Region ein Basismodell laufen lassen. Anschliessend wird eine Vorhersage mit dem eigenen Model sowie mit dem der anderen Region erstellt um so die Vorhersagekraft weiter zu verbessern.   Aufteilung der DatenUm die Daten möglichst konsitent zu teilen, werden wir für jede Kundenid den ursprünglichen Wohnort ermitteln. Wenn nun ein Kunde die Region in dem Untersuchungszeitraum wechselt, so wird nur der ursprüngliche Wohnort ausgewertet. So können wir sicherstellen, dass wir keine Daten durch die Aufteilung velieren. In dem Bild weiter unten sind die verschiedenen Regionen zu sehen. Wir werden versuchen Spanien in nördliche und südliche Regionen aufzuteilen.![image.png](data/images/spain.png)<jupyter_code>#export
import pandas as pd
import numpy as np
import pickle
import xgboost as xgb

import community_learning.base_model as base_model

from fastscript import call_parse
from itertools import compress
from tqdm.notebook import tqdm
#export
region = {
"ALAVA":"north",
"ALBACETE":"south",
"ALICANTE":"south",
"ALMERIA":"south",
"ASTURIAS":"north",
"AVILA":"north",
"BADAJOZ":"south",
"BALEARS, ILLES":"north",
"BARCELONA":"north",
"BIZKAIA":"north",
"BURGOS":"north",
"CACERES":"north",
"CADIZ":"south",
"CANTABRIA":"north",
"CASTELLON":"north",
"CEUTA":"south",
"CIUDAD REAL":"south",
"CORDOBA":"south",
"CORUÑA, A":"north",
"CUENCA":"north",
"GIPUZKOA":"north",
"GIRONA":"north",
"GRANADA":"south",
"GUADALAJARA":"north",
"HUELVA":"south",
"HUESCA":"north",
"JAEN":"south",
"LEON":"north",
"LERIDA":"north",
"LUGO":"north",
"MADRID":"south",
"MALAGA":"south",
"MELILLA":"south",
"MURCIA":"south",
"NAVARRA":"north",
"OURENSE":"north",
"PALENCIA":"north",
"PALMAS, LAS":"north",
"PONTEVEDRA":"north",
"RIOJA, LA":"north",
"SALAMANCA":"north",
"<NAME>":"north",
"SEGOVIA":"north",
"SEVILLA":"south",
"SORIA":"north",
"TARRAGONA":"north",
"TERUEL":"north",
"TOLEDO":"north",
"UNKNOWN":"north",
"VALENCIA":"north",
"VALLADOLID":"north",
"ZAMORA":"north",
"ZARAGOZA":"north"
}
#export 
def load_provice_data(path:str='data/raw/train_ver2.csv'):
    """laden der Rohdaten für die Ermittlung der Aufteilung"""
    reader = pd.read_csv(path, chunksize=100000, header=0, usecols=['ncodpers', 'nomprov'])
    train = pd.concat([chunk for chunk in reader])
    return train

data = load_provice_data()
assert 'ncodpers' in data
assert 'nomprov' in data
#export
def add_region_to_nomprov(df:pd.DataFrame):
    """add a region column"""
    df = df.groupby(by='ncodpers').first()
    df['region'] = df['nomprov'].map(lambda x: region.get(x, '----'))
    return df
data1 = add_region_to_nomprov(data)
assert data['ncodpers'].unique().shape[0] == data1.shape[0]
data1.shape[0]
#export
def load_data(path_train='data/interim/03_train.csv',
              path_test='data/interim/03_test.csv'):
    """load data"""
    train = pd.read_csv(path_train)
    test = pd.read_csv(path_test)
    return (train, test)    
train_org, test = load_data()
#export 
def add_region_train_test(train:pd.DataFrame, 
                    test:pd.DataFrame,
                    region_df:pd.DataFrame):
    """split train and test data by region"""
    train = train.merge(region_df, left_on='id', right_on='ncodpers', how='left')
    test = test.merge(region_df, left_on='id', right_on='ncodpers', how='left')
    return (train, test)
pd.set_option('display.max_columns', None)
train, test = add_region_train_test(train_org, test, data1)
len(train_org) == len(train)
south_ids = train.loc[train.region=='south',]['id'].unique().tolist()
north_ids = train.loc[train.region=='north',]['id'].unique().tolist()
assert all([ True if south_id not in north_ids else False for south_id in tqdm(south_ids) ])
product_dict = base_model.get_product_dict(train_org)
product_reverse_dict = base_model.get_product_reverse_dict(train_org)    
train = base_model.encode_products(train_org)
region_df = load_provice_data()
region_df = add_region_to_nomprov(region_df)
train1, test1 = add_region_train_test(train, test, region_df)

train_X_south, train_y_south = base_model.x_y_split(train1.loc[train1.region == 'south',].copy().reset_index())
train_X_north, train_y_north = base_model.x_y_split(train1.loc[train1.region == 'north',].copy().reset_index())
assert len(train_X_south.groupby(by='region')) == 1
assert len(train_X_north.groupby(by='region')) == 1
assert len(train_X_south) + len(train_X_north) == len(train)
model_south = base_model.runXGB(train_X_south, train_y_south, base_model.feature_cols, use_gpu=True)
model_north = base_model.runXGB(train_X_north, train_y_north, base_model.feature_cols, use_gpu=True)
#export
def get_two_region_data(source_train:str='data/interim/03_train.csv',
                               source_test:str='data/interim/03_test.csv',
                               source_raw:str='data/raw/train_ver2.csv'):
    """returns X, y data for each region"""
    data = dict()
    
    print('load data')
    train_org, test = load_data(source_train, source_test)    
    print('prepare data')
    data['product_dict'] = base_model.get_product_dict(train_org)
    data['product_reverse_dict'] = base_model.get_product_reverse_dict(train_org)    
    train = base_model.encode_products(train_org)
    region_df = load_provice_data(source_raw)
    region_df = add_region_to_nomprov(region_df)
    
    data['feature_cols'], data['target_cols'] = base_model.feature_cols, base_model.target_cols
    
    train, test = add_region_train_test(train, test, region_df)
    data['train'], data['test'] = train, test
    data['train_south'] = train.loc[train.region == 'south',]
    data['train_north'] = train.loc[train.region == 'north',]
    
    data['train_X_south'], data['train_y_south'] = base_model.x_y_split(train.loc[train.region == 'south',])
    data['train_X_north'], data['train_y_north'] = base_model.x_y_split(train.loc[train.region == 'north',])
    data['train_X'], data['train_y'] = base_model.x_y_split(train)
    
    
    test_south = test.loc[test.region == 'south']
    data['test_south'] = test_south.reset_index(drop=True)
    
    test_north = test.loc[test.region == 'north']
    data['test_north'] = test_north.reset_index(drop=True)
    
    return data
data = get_two_region_data()
data.keys()<jupyter_output><empty_output><jupyter_text>Simple Ensemble<jupyter_code>#export
def get_two_region_base_models(data:dict,
                               dest_model_south:str='data/results/model_south.dat',
                               dest_model_north:str='data/results/model_north.dat',
                               use_gpu=False):
    """load data, assign region, and train model per region"""
    
    
    print('train model south')
    model_south = base_model.runXGB(
        data['train_X_south'], 
        data['train_y_south'],
        data['feature_cols'], 
        use_gpu=use_gpu)
    
    print('train model north')
    model_north = base_model.runXGB(
        data['train_X_north'], 
        data['train_y_north'], 
        data['feature_cols'], 
        use_gpu=use_gpu)
    
    print('train model all')
    model_all = base_model.runXGB(
        data['train_X'], 
        data['train_y'], 
        data['feature_cols'], 
        use_gpu=use_gpu)
    
    models = {}
    models['model_south'], models['model_north'], models['model_all'] = model_south, model_north, model_all
    #pickle.dump(model_south, open(dest_model_south, 'wb'))
    #pickle.dump(model_north, open(dest_model_north, 'wb'))
    
    return models
models = get_two_region_base_models(data, use_gpu=True)
#export
def get_prediction(model, data, feature_cols):
    """returns the results for two region model"""
    xgtest = xgb.DMatrix(data[feature_cols])    
    return model.predict(xgtest)
preds = get_prediction(models['model_south'], data['test_south'], data['feature_cols'])
assert data['test_south'][data['target_cols']].shape[0] == preds.shape[0]
#export
def evaluate_predictions(preds, test_data, target_cols, product_reverse_dict):
    """evaluates map metric"""
    preds = np.argsort(preds, axis=1)
    preds = np.fliplr(preds)[:,:7]
    preds = pd.DataFrame(preds)
    preds = preds.applymap(lambda x: product_reverse_dict[x])
    preds['added_products'] = preds.apply(lambda x: list(x.values), axis=1)
    preds = preds['added_products']
    
    test_data['added_products'] = preds
    test_data['truth_list'] = test_data[target_cols].apply(lambda x: list(compress(target_cols, x.values)), axis=1)
    test_data['apk'] = test_data.apply(lambda x: base_model.apk(x['truth_list'], x['added_products']),axis=1)
    #print(f"mean average precision = {test_data['apk'].mean()}")
    return test_data['apk'].mean()
evaluate_predictions(preds, data['test_south'], data['target_cols'], data['product_reverse_dict'])
data = get_two_region_data()
models = get_two_region_base_models(data, use_gpu=True)
data['test']
results = {'model_south':[1,1,1], 'model_north':[1,1,1], 'model_all':[1,1,1], 'model_south_north_combine':[1,1,1]} 
  
# Creates pandas DataFrame. 
results_df = pd.DataFrame(results, index = ['test_south', 'test_north', 'test']) 
results_df
#export
def fill_results(model_label, data_label, data, models, results_df):
    model = models[model_label]
    dat = data[data_label].copy()
    preds = get_prediction(model, dat, data['feature_cols'])
    result = evaluate_predictions(preds, dat, data['target_cols'], data['product_reverse_dict'])
    results_df.loc[data_label, model_label] = result
    return results_df
def fill_results_ensemble(model1_label, model2_label, data_label, data, models, results_df):
    model1 = models[model1_label]
    model2 = models[model2_label]
    dat = data[data_label].copy()
    preds1 = get_prediction(model1, dat, data['feature_cols'])
    preds2 = get_prediction(model2, dat, data['feature_cols'])
    preds = preds1 + preds2
    result = evaluate_predictions(preds, dat, data['target_cols'], data['product_reverse_dict'])
    results_df.loc[data_label, 'model_south_north_combine'] = result
    return results_df
results_df = fill_results_ensemble('model_south', 'model_north', 'test_south', data, models, results_df)
print(results_df)

results_df = fill_results_ensemble('model_south', 'model_north', 'test_north', data, models, results_df)
print(results_df)

results_df = fill_results_ensemble('model_south', 'model_north', 'test', data, models, results_df)
print(results_df)

results_df = fill_results('model_south', 'test_south', data, models, results_df)
print(results_df)
results_df = fill_results('model_south', 'test_north', data, models, results_df)
print(results_df)
results_df = fill_results('model_south', 'test', data, models, results_df)
print(results_df)


results_df = fill_results('model_north', 'test_south', data, models, results_df)
print(results_df)
results_df = fill_results('model_north', 'test_north', data, models, results_df)
print(results_df)
results_df = fill_results('model_north', 'test', data, models, results_df)
print(results_df)


results_df = fill_results('model_all', 'test_south', data, models, results_df)
print(results_df)
results_df = fill_results('model_all', 'test_north', data, models, results_df)
print(results_df)
results_df = fill_results('model_all', 'test', data, models, results_df)
print(results_df)
results_df['distributed_lgbm_training'] = [0.026443885421329187, 0.017835593289980797, 0.022793803380779395]
results_df.to_csv('data/final/results_df.csv')
results_df<jupyter_output><empty_output><jupyter_text>Export<jupyter_code>from nbdev.export import *
notebook2script()<jupyter_output>Converted 01_data_preprocess.ipynb.
Converted 02_data_Cleaning.ipynb.
Converted 03_features.ipynb.
Converted 04_base_model - Versuch CCA.ipynb.
Converted 04_base_model.ipynb.
Converted 05_xgboost_simple_ensemble.ipynb.
This cell doesn't have an export destination and was ignored:
e
Converted 06_Distributed_ML.ipynb.
Converted index.ipynb.
time: 116 ms
{'fraction_non_alphanumeric': 0.1416075259943885, 'fraction_numerical': 0.011388017824723552, 'mean_word_length': 5.582835415535035, 'pattern_counts': {'":': 53, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '484586', 'license': 'Apache-2.0', 'n_tokens_mistral': 4499, 'n_tokens_neox': 4261, 'n_words': 804, 'path': '05_xgboost_simple_ensemble.ipynb', 'repo_name': 'SubmitCode/CommunityLearning', 'size': '12134'}