<jupyter_start><jupyter_text>Train XGBoost Model using SageMaker Training + Serve (Host) the Model as a SageMaker Endpoint  1. Imports<jupyter_code>import pandas as pd
import numpy as np
import json
import os

from sagemaker.serializers import CSVSerializer
from sagemaker.inputs import TrainingInput
from sagemaker.predictor import Predictor
from sagemaker import get_execution_role
import sagemaker
import boto3
sagemaker.__version__<jupyter_output><empty_output><jupyter_text>2. Essentials<jupyter_code># Get the Execution Role for this Notebook and AWS Session to create Clients
sagemaker_execution_role = get_execution_role()
print('Role = {}'.format(sagemaker_execution_role))
session = boto3.Session()

# Clients and Resources
s3 = session.resource('s3')
sagemaker_session = sagemaker.Session()

BUCKET = sagemaker_session.default_bucket()
PREFIX = 'clf'<jupyter_output>Role = arn:aws:iam::892313895307:role/service-role/AmazonSageMaker-ExecutionRole-20200629T123070<jupyter_text>Upload Train & Test Sets to S3 and Create Pointers to Data<jupyter_code>s3.create_bucket(Bucket=BUCKET)
s3.Bucket(BUCKET).Object(os.path.join(PREFIX, 'train/train.csv')).upload_file('./DATA/train/train.csv')
s3.Bucket(BUCKET).Object(os.path.join(PREFIX, 'test/test.csv')).upload_file('./DATA/test/test.csv')
s3.Bucket(BUCKET).Object(os.path.join(PREFIX, 'batch_test/batch_test.csv')).upload_file('./DATA/batch_test/batch_test.csv')<jupyter_output><empty_output><jupyter_text>Create Pointers to the uploaded files<jupyter_code>train_set_location = 's3://{}/{}/train/'.format(BUCKET, PREFIX)
test_set_location = 's3://{}/{}/test/'.format(BUCKET, PREFIX)
print(train_set_location)
print(test_set_location)
train_set_pointer = TrainingInput(s3_data=train_set_location, content_type='csv')
test_set_pointer = TrainingInput(s3_data=test_set_location, content_type='csv')
print(json.dumps(train_set_pointer.__dict__, indent=2))<jupyter_output>{
  "config": {
    "DataSource": {
      "S3DataSource": {
        "S3DataType": "S3Prefix",
        "S3Uri": "s3://sagemaker-us-east-1-892313895307/clf/train/",
        "S3DataDistributionType": "FullyReplicated"
      }
    },
    "ContentType": "csv"
  }
}<jupyter_text>3. Train a Model using SageMaker + Builtin XgBoost Algorithm<jupyter_code>container_uri = sagemaker.image_uris.retrieve(region=session.region_name, 
                                              framework='xgboost', 
                                              version='1.0-1', 
                                              image_scope='training')
xgb = sagemaker.estimator.Estimator(image_uri=container_uri,
                                    role=sagemaker_execution_role, 
                                    instance_count=2, 
                                    instance_type='ml.m5.large',
                                    output_path='s3://{}/{}/model-artifacts'.format(BUCKET, PREFIX),
                                    sagemaker_session=sagemaker_session,
                                    base_job_name='classifier')

xgb.set_hyperparameters(objective='binary:logistic',
                        num_round=100)
xgb.fit({'train': train_set_pointer, 'validation': test_set_pointer})<jupyter_output><empty_output><jupyter_text>4. Host the Trained Model as a SageMaker Endpoint (using Estimator object)<jupyter_code>xgb_predictor = xgb.deploy(initial_instance_count=2,
                           instance_type='ml.m5.large')<jupyter_output>---------------!<jupyter_text>5. Real Time Inference from the Endpoint<jupyter_code>csv_serializer = CSVSerializer()
predictor = Predictor(endpoint_name=xgb_predictor.endpoint_name, 
                      serializer=csv_serializer)
test_df = pd.read_csv('./DATA/test/test.csv', names=['class', 'bmi', 'diastolic_bp_change', 'systolic_bp_change', 'respiratory_rate'])
test_df.head()
X = test_df.sample(1)
X
X = X.values[0]
X[1:]
payload = X[1:]
%%time

predicted_class_prob = predictor.predict(payload).decode('utf-8')
if float(predicted_class_prob) < 0.5:
    print('PREDICTION = NOT DIABETIC')
else:
    print('PREDICTION = DIABETIC')
print()<jupyter_output>PREDICTION = NOT DIABETIC

CPU times: user 12.6 ms, sys: 171 µs, total: 12.8 ms
Wall time: 176 ms<jupyter_text>5. Evaluate Hosted Model for Accuracy<jupyter_code>predictions = []
expected = []
correct = 0
for row in test_df.values:
    expected_class = row[0]
    payload = row[1:]
    predicted_class_prob = predictor.predict(payload).decode('utf-8')
    predicted_class = 1
    if float(predicted_class_prob) < 0.5:
        predicted_class = 0  
    if predicted_class == expected_class:
        correct += 1
    predictions.append(predicted_class)
    expected.append(expected_class)
print('Accuracy = {:.2f}%'.format(correct/len(predictions) * 100))<jupyter_output>Accuracy = 77.72%<jupyter_text>Confusion Matrix<jupyter_code>expected = pd.Series(np.array(expected))
predictions = pd.Series(np.array(predictions))
pd.crosstab(expected, predictions, rownames=['Actual'], colnames=['Predicted'], margins=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12269938650306748, 'fraction_numerical': 0.022165050465070255, 'mean_word_length': 3.954901960784314, 'pattern_counts': {'":': 7, '<': 34, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '444431', 'license': 'Apache-2.0', 'n_tokens_mistral': 1760, 'n_tokens_neox': 1667, 'n_words': 269, 'path': '2. Built-in Algorithm XGBoost.ipynb', 'repo_name': 'arunprsh/SageMaker-Studio-Examples', 'size': '5053'}