<jupyter_start><jupyter_text>[Module 2.4] SageMaker 내장 XGBoost 훈련 및 추론이 노트북은 아래와 같은 작업을 합니다.- 훈련, 검증 데이터 로딩- 훈련, 검증 데이터의 헤더 없이 S3에 업로딩- 내장 알고리즘 XGBoost 컨테이너 로딩- XGBoost 훈련- 엔드포인트 생성- 테스트 데이터 로딩 및 추론- 모델 평가<jupyter_code>import pandas as pd
import numpy as np
from IPython.display import display
%store -r te_pre_train_file
%store -r te_pre_test_file<jupyter_output><empty_output><jupyter_text>데이터 세트 로딩<jupyter_code>preproc_train_data = pd.read_csv(te_pre_train_file)
print(f'preproc_train_data: {preproc_train_data.shape}')
preproc_test_data = pd.read_csv(te_pre_test_file)
display(preproc_train_data.head())<jupyter_output><empty_output><jupyter_text>훈련, 검증 데이터의 헤더없이 S3에 업로드<jupyter_code>import os

def save_local_noheader(train_data, test_data, preproc_folder):
    train_df = pd.concat([train_data['classes'], train_data.drop(['classes'], axis=1)], axis=1)
    train_file_name = os.path.join(preproc_folder, 'train.csv')
    train_df.to_csv(train_file_name, index=False, header=False)
    print(f'{train_file_name} is saved')


    test_df = pd.concat([test_data['classes'], test_data.drop(['classes'], axis=1)], axis=1)
    test_file_name = os.path.join(preproc_folder, 'test.csv')
    test_df.to_csv(test_file_name, index=False)
    print(f'{test_file_name} is saved')        
    
    return train_file_name, test_file_name

preproc_folder = 'preproc_data/feature_engineering/no_header/'
os.makedirs(preproc_folder, exist_ok=True)    
pre_train_file, pre_test_file = save_local_noheader(preproc_train_data, 
                                                                 preproc_test_data, 
                                                                 preproc_folder)

import sagemaker

sagemaker_session = sagemaker.Session()
bucket = sagemaker.Session().default_bucket()  # replace with an existing bucket if needed
prefix = 'sagemaker/delivery_time'           # prefix used for all data stored within the bucket


# Define IAM role
import boto3
from sagemaker import get_execution_role

role = get_execution_role()
from p_utils import upload_s3
s3_train_path = upload_s3(bucket, pre_train_file, prefix)
# s3_val_path = upload_s3(bucket, pre_val_file, prefix)
print("s3_train_path: ", s3_train_path)
# print("s3_val_path: ", s3_val_path)
s3_input_train = sagemaker.inputs.TrainingInput(s3_data= s3_train_path, content_type='csv')
# s3_input_validation = sagemaker.inputs.TrainingInput(s3_data= s3_val_path, content_type='csv')<jupyter_output><empty_output><jupyter_text>내장 알고리즘 XGBoost 컨테이너 로딩<jupyter_code>from sagemaker import image_uris, session
container = image_uris.retrieve("xgboost", session.Session().boto_region_name, version="latest")
print("xgb_image: ", container)<jupyter_output>xgb_image:  306986355934.dkr.ecr.ap-northeast-2.amazonaws.com/xgboost:latest<jupyter_text>XGBoost 파라미터- https://github.com/dmlc/xgboost/blob/master/doc/parameter.rst- https://docs.aws.amazon.com/sagemaker/latest/dg/xgboost_hyperparameters.html  XGBoost 훈련<jupyter_code>%%time 
sess = sagemaker.Session()

xgb = sagemaker.estimator.Estimator(container,
                                    role, 
                                    train_instance_count=1, 
                                    train_instance_type='ml.m4.xlarge',
                                    output_path='s3://{}/{}/output'.format(bucket, prefix),
                                    sagemaker_session=sess)
xgb.set_hyperparameters(max_depth=6, # default: 6
                        eta=0.3, # learning_rate, default : 0.3
                        alpha = 10, # L1 regularization, default : 0.3
                        gamma=0, # regularization, default : 0
                        colsample_bytree = 0.3,                         
                        min_child_weight=1, # regularization, default: 1, possible: 6
                        subsample=0.8, # default: 1
                        silent=0,
                        num_class = 5,
                        objective='multi:softmax',
                        num_round=100,
                        seed = 1000
                       )

# xgb.fit({'train': s3_input_train, 'validation': s3_input_validation})
xgb.fit({'train': s3_input_train})<jupyter_output>train_instance_count has been renamed in sagemaker>=2.
See: https://sagemaker.readthedocs.io/en/stable/v2.html for details.
train_instance_type has been renamed in sagemaker>=2.
See: https://sagemaker.readthedocs.io/en/stable/v2.html for details.<jupyter_text>엔드 포인트 생성<jupyter_code>%%time 
xgb_predictor = xgb.deploy(initial_instance_count=1,
                           instance_type='ml.m5.xlarge')<jupyter_output>-----------!CPU times: user 198 ms, sys: 5.91 ms, total: 204 ms
Wall time: 5min 31s<jupyter_text>테스트 데이터 로딩 및 추론<jupyter_code>from sagemaker.serializers import CSVSerializer
CSVSerializer.ACCEPT = 'text/csv'
xgb_predictor.serializer = CSVSerializer()
test_df = pd.read_csv(te_pre_test_file, header=None,skiprows=1)
test_df.head()
X_test, y_test = test_df.iloc[:,1:], test_df.iloc[:,0]
def predict(data, rows=500):
    split_array = np.array_split(data, int(data.shape[0] / float(rows) + 1))
    predictions = ''
    for array in split_array:
        predictions = ','.join([predictions, xgb_predictor.predict(array).decode('utf-8')])

    return np.fromstring(predictions[1:], sep=',')

prediction = predict(X_test.to_numpy())<jupyter_output><empty_output><jupyter_text>모델 평가<jupyter_code>from sklearn.metrics import accuracy_score, f1_score
accuracy = accuracy_score(y_test, prediction)
f1_score = f1_score(y_test, prediction, average='macro')
print("accuracy: %.2f%%" % accuracy)
print("f1_score: %.2f%%" % f1_score)
from p_utils import plot_conf_mat


from sklearn.metrics import confusion_matrix
import matplotlib.pyplot as plt
%matplotlib inline
%config InlineBackend.figure_format='retina'

cm = confusion_matrix(y_true= y_test, y_pred= prediction)

plt.figure()
fig, ax = plt.subplots(figsize=(10,5))
plot_conf_mat(cm, 
              classes=['0','1', '2', '3', '4'], 
              title='Confusion Matrix')
plt.show()<jupyter_output>[[ 955  423  301   24    2]
 [ 345 1273 1720  731   93]
 [ 194  413 2395 1119  414]
 [  61  183  593 1102  229]
 [  23   51  271  274  247]]
{'fraction_non_alphanumeric': 0.11208010335917312, 'fraction_numerical': 0.030523255813953487, 'mean_word_length': 3.3217027215631543, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 31, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '303298', 'license': 'MIT', 'n_tokens_mistral': 2407, 'n_tokens_neox': 2377, 'n_words': 398, 'path': 'brazil_ecommerce/3.4.SM_XGBoost.ipynb', 'repo_name': 'gonsoomoon-ml/predict-delivery-time', 'size': '6192'}