<jupyter_start><jupyter_text>MNIST Data Exploration<jupyter_code>import numpy as np
import pandas as pd
import csv
from sklearn.model_selection import train_test_split
import xgboost
from sklearn import metrics

#all the data is under the same folder
import os
print(os.listdir("../mnist_data_exploration"))

df_train = pd.read_csv('../mnist_data_exploration/train.csv')
y = df_train.label.values
X = df_train.drop('label', axis=1).values
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.1)
#df_train.head()
#print(X_train)
dtrain = xgboost.DMatrix(X_train, label=y_train)
dval = xgboost.DMatrix(X_val, label=y_val)

params = {
    'max_depth': 5,                 # the maximum depth of each tree
    'eta': 0.5,                     # the training step for each iteration
    'eta_decay': 0.9,
    'min_eta': 0.05,
    'silent': 0,                    # logging mode - quiet
    'objective': 'multi:softmax',   # multiclass classification using the softmax objective
    'num_class': 10,                 # the number of classes that exist in this datset
    'early_stopping_rounds': 10,
    'n_estimators': 1000
}  

def eval_acc(preds, dtrain):
    labels = dtrain.get_label()
    return 'acc', metrics.accuracy_score(labels, preds)

results = {}
model = xgboost.train(params, dtrain, num_boost_round=100, evals=[(dval, 'val')], evals_result=results, feval=eval_acc)
df_test = pd.read_csv('../mnist_data_exploration/test.csv')
#df_test.head()
x_test = df_test.values
dtest = xgboost.DMatrix(x_test)
pred_values = model.predict(dtest)
print(pred_values.shape)
pred_values[:2]
df_sub = pd.read_csv('../mnist_data_exploration/sample_submission.csv')
df_sub.head()
df_sub.Label = pred_values.astype(np.int32)
df_sub.head()
df_sub.to_csv('submission.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12362030905077263, 'fraction_numerical': 0.01434878587196468, 'mean_word_length': 4.33235294117647, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '205523', 'license': 'Apache-2.0', 'n_tokens_mistral': 679, 'n_tokens_neox': 651, 'n_words': 141, 'path': 'MNIST_EXPLORATION_WITH_XGBOOST.ipynb', 'repo_name': 'pedrostrepeckes/mnist_data_exploration', 'size': '1812'}