<jupyter_start><jupyter_text>4. EmsembleMeankaggle score: 0.99357  Run name<jupyter_code>import time

project_name = 'DigitRecognizer'
step_name = 'EmsembleMean'
date_str = time.strftime("%Y%m%d", time.localtime())
time_str = time.strftime("%Y%m%d_%H%M%S", time.localtime())
run_name = '%s_%s_%s' % (project_name, step_name, time_str)
print('run_name: %s' % run_name)
t0 = time.time()<jupyter_output>run_name: DigitRecognizer_EmsembleMean_20180417_211659<jupyter_text>Import PKGs<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
%matplotlib inline
from IPython.display import display

import os
import gc
import math
import shutil
import zipfile
import pickle
import h5py
from PIL import Image

from tqdm import tqdm
from multiprocessing import cpu_count

from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, accuracy_score<jupyter_output><empty_output><jupyter_text>Basic folders<jupyter_code>cwd = os.getcwd()
input_folder = os.path.join(cwd, 'input')
log_folder = os.path.join(cwd, 'log')
model_folder = os.path.join(cwd, 'model')
output_folder = os.path.join(cwd, 'output')
print('input_folder: \t\t%s' % input_folder)
print('log_folder: \t\t%s' % log_folder)
print('model_folder: \t\t%s' % model_folder)
print('output_folder: \t\t%s'% output_folder)

train_csv_file = os.path.join(input_folder, 'train.csv')
test_csv_file = os.path.join(input_folder, 'test.csv')
print('\ntrain_csv_file: \t%s' % train_csv_file)
print('test_csv_file: \t\t%s' % test_csv_file)

processed_data_file = os.path.join(input_folder, '%s_%s.p' % (project_name, step_name))
print('processed_data_file: \t%s' % processed_data_file)<jupyter_output>input_folder: 		D:\Kaggle\digit-recognizer\input
log_folder: 		D:\Kaggle\digit-recognizer\log
model_folder: 		D:\Kaggle\digit-recognizer\model
output_folder: 		D:\Kaggle\digit-recognizer\output

train_csv_file: 	D:\Kaggle\digit-recognizer\input\train.csv
test_csv_file: 		D:\Kaggle\digit-recognizer\input\test.csv
processed_data_file: 	D:\Kaggle\digit-recognizer\input\DigitRecognizer_EmsembleMean.p<jupyter_text>Basic functions<jupyter_code>import sys
def describe(arr):
    print(arr.shape, arr.min(), arr.max(), sys.getsizeof(arr))
def show_data_images(rows, fig_column, y_data, *args):
    columns = len(args)
    figs, axes = plt.subplots(rows, columns, figsize=(rows, fig_column*columns))
    print(axes.shape)  
    for i, ax in enumerate(axes):
        y_data_str = ''
        if type(y_data) != type(None):
            y_data_str =  '_' + str(y_data[i])
        ax[0].set_title('28x28' + y_data_str)
        for j, arg in enumerate(args):
            ax[j].imshow(arg[i])<jupyter_output><empty_output><jupyter_text>Load predict probability files<jupyter_code>def is_npy(s):
    return s.startswith('DigitRecognizer_MoreStructure_20180417') and s.endswith('.p')

def is_larger_than(x):
    splits = x[:-2].split('_')
    acc = int(splits[-1])
#     print(acc)
    if acc >= 9990:
        return True
    return False

print(is_larger_than('DigitRecognizer_MoreStructure_20180417_162556_9990.p'))

file_names = os.listdir(model_folder)
file_names = list(filter(is_npy, file_names))
print(len(file_names))

file_names = list(filter(is_larger_than, file_names))
print(len(file_names))

print(file_names)
def load_proba(file_name):
    with h5py.File(file_name, 'r') as h:
        y_data_proba = np.array(h['y_data_proba'])
        y_data = np.array(h['y_data'])
        y_test_proba = np.array(h['y_test_proba'])
    print('File loaded: \t%s' % file_name)
    
    return y_data_proba, y_data, y_test_proba


y_proba_file = os.path.join(model_folder, file_names[0])
y_data_proba, y_data, y_test_proba = load_proba(y_proba_file)

print(y_data_proba.shape)
print(y_data.shape)
print(y_test_proba.shape)<jupyter_output>File loaded: 	D:\Kaggle\digit-recognizer\model\DigitRecognizer_MoreStructure_20180417_003649_9990.p
(42000, 10)
(42000, 10)
(28000, 10)<jupyter_text>Mean<jupyter_code>from sklearn.metrics import accuracy_score

y_proba_file = os.path.join(model_folder, file_names[0])
y_data_probas, y_datas, y_test_probas = load_proba(y_proba_file)

print(y_data_probas.shape)
print(y_datas.shape)
print(y_test_probas.shape)

for file_name in file_names[1:]:
    file_path = os.path.join(model_folder, file_name)
    y_data_proba, y_data, y_test_proba = load_proba(file_path)
    y_data_probas = y_data_probas + y_data_proba
    y_test_probas = y_test_probas + y_test_proba
    
y_data_probas = y_data_probas / len(file_names)
y_test_probas = y_test_probas / len(file_names)

print(len(y_data_probas))
print(y_data_probas[:2])<jupyter_output>File loaded: 	D:\Kaggle\digit-recognizer\model\DigitRecognizer_MoreStructure_20180417_003649_9990.p
(42000, 10)
(42000, 10)
(28000, 10)
File loaded: 	D:\Kaggle\digit-recognizer\model\DigitRecognizer_MoreStructure_20180417_011627_9990.p
File loaded: 	D:\Kaggle\digit-recognizer\model\DigitRecognizer_MoreStructure_20180417_043418_9990.p
File loaded: 	D:\Kaggle\digit-recognizer\model\DigitRecognizer_MoreStructure_20180417_063232_9990.p
File loaded: 	D:\Kaggle\digit-recognizer\model\DigitRecognizer_MoreStructure_20180417_071204_9990.p
File loaded: 	D:\Kaggle\digit-recognizer\model\DigitRecognizer_MoreStructure_20180417_095031_9990.p
File loaded: 	D:\Kaggle\digit-recognizer\model\DigitRecognizer_MoreStructure_20180417_103009_10000.p
File loaded: 	D:\Kaggle\digit-recognizer\model\DigitRecognizer_MoreStructure_20180417_122854_9990.p
File loaded: 	D:\Kaggle\digit-recognizer\model\DigitRecognizer_MoreStructure_20180417_130832_9990.p
File loaded: 	D:\Kaggle\digit-recognizer\model\DigitRecognizer_[...]<jupyter_text>Predict<jupyter_code>y_data_pred = np.argmax(y_data_probas, axis=1)
print('y_test_pred.shape: ', y_data_pred.shape)
y_data = np.argmax(y_data, axis=1)

data_acc = accuracy_score(y_data, y_data_pred)
run_name_acc = run_name + '_' + str(int(data_acc*10000)).zfill(4)
print(run_name_acc)

if not os.path.exists(output_folder):
    os.mkdir(output_folder)
pred_file = os.path.join(output_folder, run_name_acc + '.csv')
print(pred_file)
y_test_pred = np.argmax(y_test_proba, axis=1)
print('y_test_pred.shape: ', y_test_pred.shape)

with open(pred_file, 'w') as f:
    f.write('ImageId,Label\n')
    for i in range(len(y_test_pred)) :
        f.write("".join([str(i+1),',',str(y_test_pred[i]),'\n']))
print(run_name_acc)

t1 = time.time()
print('time cost: %.2f s' % (t1-t0))
print('Done!')<jupyter_output>DigitRecognizer_EmsembleMean_20180417_211659_9993
time cost: 2.67 s
Done!
{'fraction_non_alphanumeric': 0.1258741258741259, 'fraction_numerical': 0.04910307084220128, 'mean_word_length': 6.31, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '406362', 'license': 'MIT', 'n_tokens_mistral': 2901, 'n_tokens_neox': 2652, 'n_words': 384, 'path': 'digit-recognizer/4. EmsembleMean.ipynb', 'repo_name': 'StudyExchange/Kaggle', 'size': '6578'}