<jupyter_start><jupyter_text>ETLEDA를 통해 데이터 분석이 어느정도 완료되면, ETL를 통해 모델링 하기 위한 데이터 셋을 만듭니다.<jupyter_code>from utils import *

sdate = get_env_sdate(default = "2018070108")
path_base = get_env_path_base(default = "/root/mnt/dfs/notebooks-skp/mnist")
path_data = get_env_path_date(default = "/root/mnt/dfs/data/mnist")

print("sdate: {}".format(sdate))
print("path_base: {}".format(path_base))
print("path_data: {}".format(path_data))
# 데이터 로드
import os
from sklearn.externals import joblib 

path_raw = os.path.join(path_data, "raw")
path_raw_sdate = os.path.join(path_raw, sdate)

path_train_xs = os.path.join(path_raw_sdate, "train_xs.pkl")
path_train_ys = os.path.join(path_raw_sdate, "train_ys.pkl")
np_train_xs = joblib.load(path_train_xs)
np_train_ys = joblib.load(path_train_ys)
print(np_train_xs.shape, np_train_ys.shape)

path_test_xs = os.path.join(path_raw_sdate, "test_xs.pkl")
path_test_ys = os.path.join(path_raw_sdate, "test_ys.pkl")
np_test_xs = joblib.load(path_test_xs)
np_test_ys = joblib.load(path_test_ys)
print(np_test_xs.shape, np_test_ys.shape)
# 전처리 - 28x28 2D 이미지를 DNN 모델에 적합한 784 1D 로 변환합니다. 
dim_x = np_train_xs.shape[1] * np_train_xs.shape[2]
dim_y = 10
np_train_xs = np_train_xs.reshape(-1, dim_x).astype(float)
np_test_xs = np_test_xs.reshape(-1, dim_x).astype(float)
print(np_train_xs.shape)
print(np_test_xs.shape)
# 전처리 - 0~1 사이로 데이터 Min Max Scaling
from sklearn import preprocessing

scaler = preprocessing.MinMaxScaler(feature_range=(0, 1))
np_train_xs = scaler.fit_transform(np_train_xs)

path_etl = os.path.join(path_data, "etl")
path_etl_sdate = os.path.join(path_etl, sdate)
os.makedirs(path_etl_sdate, exist_ok=True)

path_scaler = os.path.join(path_etl_sdate, "rf-scaler.pkl")
joblib.dump(scaler, path_scaler)

scaler = joblib.load(path_scaler)
np_test_xs = scaler.transform(np_test_xs)
# 데이터 저장
from sklearn.externals import joblib 

path_train_xs = os.path.join(path_etl_sdate, "rf-train_xs.pkl")
path_train_ys = os.path.join(path_etl_sdate, "rf-train_ys.pkl")
joblib.dump(np_train_xs, path_train_xs)
joblib.dump(np_train_ys, path_train_ys)

path_test_xs = os.path.join(path_etl_sdate, "rf-test_xs.pkl")
path_test_ys = os.path.join(path_etl_sdate, "rf-test_ys.pkl")
joblib.dump(np_test_xs, path_test_xs)
joblib.dump(np_test_ys, path_test_ys)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12975778546712802, 'fraction_numerical': 0.012543252595155709, 'mean_word_length': 6.8941979522184305, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '233825', 'license': 'Apache-2.0', 'n_tokens_mistral': 1116, 'n_tokens_neox': 1133, 'n_words': 152, 'path': 'mnist/src/backend/etl-rf.ipynb', 'repo_name': 'comafire/notebooks-skp', 'size': '2312'}