<jupyter_start><jupyter_text>Reliance Test predict the risk of hospital readmission for patients   Let's configure our environment<jupyter_code>import sys
!{sys.executable} -m pip install sagemaker==2.42.0 -U
!{sys.executable} -m pip install sagemaker-experiments
!{sys.executable} -m pip install xgboost==1.3.3
#!pip freeze | grep sagemaker
#!pip freeze | grep xgboost
import pandas as pd
import boto3
import sagemaker

sess = boto3.Session()
region = sess.region_name
sm = sess.client('sagemaker')
role = sagemaker.get_execution_role()

region = sess.region_name
account_id = sess.client('sts', region_name=region).get_caller_identity()["Account"]
bucket = 'sagemaker-studio-{}-{}'.format(sess.region_name, account_id)
prefix = 'xgboost-readmission-predictor'

try:
    if sess.region_name == "us-east-1":
        sess.client('s3').create_bucket(Bucket=bucket)
    else:
        sess.client('s3').create_bucket(Bucket=bucket, 
                                        CreateBucketConfiguration={'LocationConstraint': sess.region_name})
except Exception as e:
    print("Looks like you already have a bucket of this name. That's good!")

framework_version = '1.2-2'
docker_image_name = sagemaker.image_uris.retrieve(framework='xgboost', region=region, version=framework_version)

# Workaround while versions are not updated in SM SDK
framework_version = '1.3-1'
docker_image_name = docker_image_name[:-5] + framework_version

print("Setting some useful environment variables (bucket, prefix, region, docker_image_name)...")
%store bucket
%store prefix
%store region
%store docker_image_name
%store framework_version<jupyter_output>Setting some useful environment variables (bucket, prefix, region, docker_image_name)...
Stored 'bucket' (str)
Stored 'prefix' (str)
Stored 'region' (str)
Stored 'docker_image_name' (str)
Stored 'framework_version' (str)<jupyter_text>--- Let's upload the data to S3<jupyter_code>local_raw_path = "readmission.csv"
raw_dir = f"{prefix}/data/raw"
s3uri_raw = sagemaker.s3.S3Uploader.upload(local_raw_path, f's3://{bucket}/{raw_dir}')
s3uri_raw<jupyter_output><empty_output><jupyter_text>Store the raw data S3 URI for later:<jupyter_code>import sagemaker

sess = sagemaker.Session()
bucket = sess.default_bucket()
prefix = "Reliance/xgboost-risk-predictor"

# Define IAM role
import boto3
import re
from sagemaker import get_execution_role

role = get_execution_role()
role
import io
import os
import sys
import time
import json

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

from IPython.display import display as dis
from time import strftime, gmtime
from sagemaker.inputs import TrainingInput
from sagemaker.serializers import CSVSerializer
from IPython import display 
import pandas as pd
import seaborn as sns

#loading Dataset
df = pd.read_csv("readmission.csv")
#displaying first 10 rows of data
df.head(10).T
#checking shape of the dataset
df.shape
#Checking data types of each variable
num_vars = df.columns[df.dtypes != 'object']
cat_vars = df.columns[df.dtypes == 'object']

print(num_vars, cat_vars)
#Checking for missing values in dataset
df[num_vars].isnull().sum()
# Distribution of Readmission 
sns.countplot(df['Readmission.Status']).set_title('Distrinution of Readmission')
!pip install imbalanced-learn
f = sns.countplot(x='Readmission.Status', data=df, hue='Gender')
plt.legend(['Female', 'Male'])
f.set_title("readmission by gender")
f.set_xticklabels(['0', '1'])
plt.xlabel("");
heat_map = sns.heatmap(df.corr(method='pearson'), annot=True, fmt='.2f', linewidths=2)
heat_map.set_xticklabels(heat_map.get_xticklabels(), rotation=45);
plt.subplot(121)
df[df['Readmission.Status']==1].groupby('Age_Category')['Age'].count().plot(kind='bar')
plt.title('Age Distribution of Patients readmitted')

plt.subplot(122)
df[df['Readmission.Status']==0].groupby('Age_Category')['Age'].count().plot(kind='bar')
plt.title('Age Distribution of Patients not readmitted')<jupyter_output><empty_output><jupyter_text>From the above visulaization, we have more elderly generally in the datset.<jupyter_code>f = sns.countplot(x='HCC.Riskscore', data=df, hue='Readmission.Status')
f.set_title('Readmitted by Risk score')
plt.ylabel('')
plt.xlabel('Readmitted.status')
plt.legend(['Not readmitted', 'Readmitted']);<jupyter_output><empty_output><jupyter_text>Let us One Hot-encode our categorical variables.<jupyter_code>categorical_columns = ["Gender", "Race", "DRG.class", "DRG.Complication"]
for column in categorical_columns:
    tempdf = pd.get_dummies(df[column], prefix=column)
    df = pd.merge(
        left=df,
        right=tempdf,
        left_index=True,
        right_index=True,
    )
    df = df.drop(columns=column)
print(df)<jupyter_output>ID.Codes  Readmission.Status  ER  LOS  Age  HCC.Riskscore  \
0      0005EC02F1DC3A15                   0   0    3   71          1.631   
1      0005EC02F1DC3A15                   0   0    2   71          1.631   
2      0006C4369B0D7B5F                   0   0    2   68          0.585   
3      000960C9E1C43BE4                   0   0   12   79          1.660   
4      00098CF15837D918                   0   0    4   91          1.768   
...                 ...                 ...  ..  ...  ...            ...   
66777  FFFC678D34A09F24                   0   0    4   72          3.675   
66778  FFFD086FCB5E0153                   0   0    9   76          1.839   
66779  FFFD086FCB5E0153                   0   0    4   77          1.711   
66780  FFFEC53980B44371                   0   0    9   52          0.655   
66781  FFFEC53980B44371                   0   0    5   53          2.128   

       Gender_F  Gender_M  Race_Black  Race_Hispanic  Race_Other  Race_White  \
0      [...]<jupyter_text>From the Data exploration, there were no missing values and the features were well distributed. But target variable was imbalanced. Let's drop the ID.codes column.<jupyter_code>#drop the ID.Codes column.
df.drop(df.columns[[0]], axis=1, inplace=True)
print(df)<jupyter_output>Readmission.Status  ER  LOS  Age  HCC.Riskscore  Gender_F  Gender_M  \
0                       0   0    3   71          1.631         0         1   
1                       0   0    2   71          1.631         0         1   
2                       0   0    2   68          0.585         0         1   
3                       0   0   12   79          1.660         1         0   
4                       0   0    4   91          1.768         0         1   
...                   ...  ..  ...  ...            ...       ...       ...   
66777                   0   0    4   72          3.675         0         1   
66778                   0   0    9   76          1.839         1         0   
66779                   0   0    4   77          1.711         1         0   
66780                   0   0    9   52          0.655         1         0   
66781                   0   0    5   53          2.128         1         0   

       Race_Black  Race_Hispanic  Race_Other  Race_White  DRG.c[...]<jupyter_text>Let's split the dataset<jupyter_code>train_data, validation_data, test_data = np.split(
    df.sample(frac=1, random_state=1729),
    [int(0.7 * len(df)), int(0.9 * len(df))],
)
train_data.shape, validation_data.shape, test_data.shape
train_data.head(2)
validation_data.head(2)
test_data.head(2)
df.shape<jupyter_output><empty_output><jupyter_text>Create CSV files for the split datasets<jupyter_code>train_file_name = "train.csv"
validation_file_name = "validation.csv"
test_file_name = "test.csv"

train_data.to_csv(train_file_name , header=False, index=False)
validation_data.to_csv(validation_file_name, header=False, index=False)
test_data.to_csv(test_file_name, header=False, index=False)<jupyter_output><empty_output><jupyter_text>Upload files to s3<jupyter_code>train_dir = f"{prefix}/data/train"
val_dir = f"{prefix}/data/validation"
test_dir = f"{prefix}/data/test"

s3uri_train = sagemaker.s3.S3Uploader.upload(train_file_name, f's3://{bucket}/{train_dir}')
s3uri_validation = sagemaker.s3.S3Uploader.upload(validation_file_name, f's3://{bucket}/{val_dir}')
s3uri_test = sagemaker.s3.S3Uploader.upload(test_file_name, f's3://{bucket}/{test_dir}')
s3uri_train, s3uri_validation, s3uri_test<jupyter_output><empty_output><jupyter_text>Let us Model the data<jupyter_code>#Supress default INFO logging
import logging
logger = logging.getLogger()
logger.setLevel(logging.ERROR)
import pprint
from time import strftime, gmtime

import boto3
import sagemaker
from sagemaker import get_execution_role
from sagemaker.debugger import rule_configs, Rule, DebuggerHookConfig

from smexperiments.experiment import Experiment
from smexperiments.trial import Trial
from smexperiments.trial_component import TrialComponent<jupyter_output><empty_output><jupyter_text>--- TrainWe'll use the XGBoost library to train a class of models known as gradient boosted decision trees on the data that we just uploaded.<jupyter_code>from sagemaker.inputs import TrainingInput
s3_input_train = TrainingInput(s3_data=s3uri_train, content_type='csv')
s3_input_validation = TrainingInput(s3_data=s3uri_validation, content_type='csv')
pprint.pprint(s3_input_train.config)
pprint.pprint(s3_input_validation.config)<jupyter_output>{'ContentType': 'csv',
 'DataSource': {'S3DataSource': {'S3DataDistributionType': 'FullyReplicated',
                                 'S3DataType': 'S3Prefix',
                                 'S3Uri': 's3://sagemaker-us-east-1-713782020867/Reliance/xgboost-risk-predictor/data/validation/validation.csv'}}}<jupyter_text>Amazon SageMaker Experiments allows me to keep track of my model training<jupyter_code># Helper to create timestamps
create_date = lambda: strftime("%Y-%m-%d-%H-%M-%S", gmtime())
readmission_predictor_experiment = Experiment.create(experiment_name=f"readmission-predictor-xgboost-{create_date()}", 
                                              description="Using xgboost to predict customer churn", 
                                              sagemaker_boto_client=boto3.client('sagemaker'))<jupyter_output><empty_output><jupyter_text>HyperparametersNow we can specify our XGBoost hyperparameters.<jupyter_code>hyperparams = {"max_depth":5,
               "subsample":0.8,
               "num_round":600,
               "eta":0.2,
               "gamma":4,
               "min_child_weight":6,
               "objective":'binary:logistic',
               "verbosity": 0
              }
%%writefile xgboost_readmission_predictor.py
import argparse
import json
import os
import pickle
import random
import tempfile
import urllib.request

import xgboost
from smdebug import SaveConfig
from smdebug.xgboost import Hook


def parse_args():

    parser = argparse.ArgumentParser()

    parser.add_argument("--max_depth", type=int, default=5)
    parser.add_argument("--eta", type=float, default=0.2)
    parser.add_argument("--gamma", type=int, default=4)
    parser.add_argument("--min_child_weight", type=int, default=6)
    parser.add_argument("--subsample", type=float, default=0.8)
    parser.add_argument("--verbosity", type=int, default=0)
    parser.add_argument("--objective", type=str, default="binary:logistic")
    parser.add_argument("--num_round", type=int, default=50)
    parser.add_argument("--smdebug_path", type=str, default=None)
    parser.add_argument("--smdebug_frequency", type=int, default=1)
    parser.add_argument("--smdebug_collections", type=str, default='metrics')
    parser.add_argument("--output_uri", type=str, default="/opt/ml/output/tensors",
                        help="S3 URI of the bucket where tensor data will be stored.")

    parser.add_argument('--train', type=str, default=os.environ.get('SM_CHANNEL_TRAIN'))
    parser.add_argument('--validation', type=str, default=os.environ.get('SM_CHANNEL_VALIDATION'))
    parser.add_argument('--model-dir', type=str, default=os.environ['SM_MODEL_DIR'])
    
    args = parser.parse_args()

    return args


def create_smdebug_hook(out_dir, train_data=None, validation_data=None, frequency=1, collections=None,):

    save_config = SaveConfig(save_interval=frequency)
    hook = Hook(
        out_dir=out_dir,
        train_data=train_data,
        validation_data=validation_data,
        save_config=save_config,
        include_collections=collections,
    )

    return hook


def main():
    
    args = parse_args()

    train, validation = args.train, args.validation
    parse_csv = "?format=csv&label_column=0"
    dtrain = xgboost.DMatrix(train+parse_csv)
    dval = xgboost.DMatrix(validation+parse_csv)

    watchlist = [(dtrain, "train"), (dval, "validation")]

    params = {
        "max_depth": args.max_depth,
        "eta": args.eta,
        "gamma": args.gamma,
        "min_child_weight": args.min_child_weight,
        "subsample": args.subsample,
        "verbosity": args.verbosity,
        "objective": args.objective}

    # The output_uri is a the URI for the s3 bucket where the metrics will be
    # saved.
    output_uri = (
        args.smdebug_path
        if args.smdebug_path is not None
        else args.output_uri
    )

    collections = (
        args.smdebug_collections.split(',')
        if args.smdebug_collections is not None
        else None
    )

    hook = create_smdebug_hook(
        out_dir=output_uri,
        frequency=args.smdebug_frequency,
        collections=collections,
        train_data=dtrain,
        validation_data=dval,
    )

    bst = xgboost.train(
        params=params,
        dtrain=dtrain,
        evals=watchlist,
        num_boost_round=args.num_round,
        callbacks=[hook])
    
    if not os.path.exists(args.model_dir):
        os.makedirs(args.model_dir)

    model_location = os.path.join(args.model_dir, 'xgboost-model')
    pickle.dump(bst, open(model_location, 'wb'))


if __name__ == "__main__":

    main()


def model_fn(model_dir):
    """Load a model. For XGBoost Framework, a default function to load a model is not provided.
    Users should provide customized model_fn() in script.
    Args:
        model_dir: a directory where model is saved.
    Returns:
        A XGBoost model.
        XGBoost model format type.
    """
    model_files = (file for file in os.listdir(model_dir) if os.path.isfile(os.path.join(model_dir, file)))
    model_file = next(model_files)
    try:
        booster = pickle.load(open(os.path.join(model_dir, model_file), 'rb'))
        format = 'pkl_format'
    except Exception as exp_pkl:
        try:
            booster = xgboost.Booster()
            booster.load_model(os.path.join(model_dir, model_file))
            format = 'xgb_format'
        except Exception as exp_xgb:
            raise ModelLoadInferenceError("Unable to load model: {} {}".format(str(exp_pkl), str(exp_xgb)))
    booster.set_param('nthread', 1)
    return booster, format


def predict_fn(input_object, model):
    """
    Perform prediction on the deserialized object, with the loaded model.
    """
    X_test = xgboost.DMatrix(input_object.values)
    predictions_probs = model.predict(X_test)
    predictions = predictions_probs.round()
    return {"predictions": predictions}


def input_fn(request_body, content_type):
    """
    Perform preprocessing task on inference dataset.
    """
    if content_type == "text/csv":
        df = pd.read_csv(StringIO(request_body), header=None)
        return df
    else:
        raise ValueError("{} not supported by script!".format(content_type))
from sagemaker.xgboost.estimator import XGBoost
role = sagemaker.get_execution_role()
sm_sess = sagemaker.session.Session()
train_script_name = 'xgboost_readmission_predictor.py'

trial = Trial.create(trial_name=f'framework-mode-trial-{create_date()}', 
                     experiment_name=readmission_predictor_experiment.experiment_name,
                     sagemaker_boto_client=boto3.client('sagemaker'))<jupyter_output><empty_output><jupyter_text>Setting up Amazon SageMaker Debugger<jupyter_code>debug_rules = [Rule.sagemaker(rule_configs.loss_not_decreasing()),
               Rule.sagemaker(rule_configs.overtraining()),
               Rule.sagemaker(rule_configs.overfit())
              ]
framework_xgb = XGBoost(image_uri=docker_image_name,
                        entry_point=train_script_name,
                        role=role,
                        framework_version=framework_version,
                        py_version="py3",
                        hyperparameters=hyperparams,
                        instance_count=1, 
                        instance_type='ml.m4.xlarge',
                        output_path=f's3://{bucket}/{prefix}/output',
                        base_job_name='readmission-predictor-experiment',
                        sagemaker_session=sm_sess,
                        rules=debug_rules
                        )


framework_xgb.fit(inputs={
                      'train': s3_input_train,
                      'validation': s3_input_validation
                         },
                  experiment_config={
                      'ExperimentName': readmission_predictor_experiment.experiment_name, 
                      'TrialName': trial.trial_name,
                      'TrialComponentDisplayName': 'Training'
                  }
                 )<jupyter_output>INFO:sagemaker.image_uris:Defaulting to the only supported framework/algorithm version: latest.
INFO:sagemaker.image_uris:Ignoring unnecessary instance type: None.
INFO:sagemaker:Creating training-job with name: readmission-predictor-experiment-2022-03-01-17-04-49-538<jupyter_text>Save training Job<jupyter_code>training_job_name = framework_xgb.latest_training_job.job_name
training_job_name
%store training_job_name<jupyter_output>Stored 'training_job_name' (str)<jupyter_text>Store the S3 URI and training script name where our training script was saved<jupyter_code>framework_xgb.hyperparameters()
s3_modeling_code_uri = eval(framework_xgb.hyperparameters()["sagemaker_submit_directory"])
s3_modeling_code_uri
%store s3_modeling_code_uri
%store train_script_name<jupyter_output>Stored 'train_script_name' (str)<jupyter_text>Evaluation<jupyter_code>from sagemaker.s3 import S3Uploader, S3Downloader

# Get name of training job and other variables
%store -r training_job_name

training_job_name
estimator = sagemaker.estimator.Estimator.attach(training_job_name)
s3uri_model = estimator.model_data
print("\ns3uri_model =",s3uri_model)

S3Downloader.download(s3uri_model, ".")
import json
import os
import tarfile
import logging
import pickle

import pandas as pd
import xgboost

from sklearn.metrics import classification_report, roc_auc_score, accuracy_score


model_path = "model.tar.gz"
with tarfile.open(model_path) as tar:
    tar.extractall(path=".")

print("Loading xgboost model.")
model = pickle.load(open("xgboost-model", "rb"))
model
print("Loading test input data")
test_path = "test.csv"
df = pd.read_csv(test_path, header=None)
df
print("Reading test data. We should get an `DMatrix` object...")
y_test = df.iloc[:, 0].to_numpy()
df.drop(df.columns[0], axis=1, inplace=True)
X_test = xgboost.DMatrix(df.values)
X_test
print("Performing predictions against test data.")
predictions_probs = model.predict(X_test)
predictions = predictions_probs.round()

print("Creating classification evaluation report")
acc = accuracy_score(y_test, predictions)
auc = roc_auc_score(y_test, predictions_probs)

print("Accuracy =", acc)
print("AUC =", auc)<jupyter_output>Performing predictions against test data.
Creating classification evaluation report
Accuracy = 0.8736337775116035
AUC = 0.7154107424180318<jupyter_text>Creating a classification report<jupyter_code>import pprint
# The metrics reported can change based on the model used - check the link for the documentation 
report_dict = {
    "binary_classification_metrics": {
        "accuracy": {
            "value": acc,
            "standard_deviation": "NaN",
        },
        "auc": {"value": auc, "standard_deviation": "NaN"},
    },
}

print("Classification report:")
pprint.pprint(report_dict)
evaluation_output_path = os.path.join(
    ".", "evaluation.json"
)
print("Saving classification report to {}".format(evaluation_output_path))

with open(evaluation_output_path, "w") as f:
    f.write(json.dumps(report_dict))
%%writefile evaluate.py
"""Evaluation script for measuring model accuracy."""

import json
import os
import tarfile
import logging
import pickle

import pandas as pd
import xgboost

logger = logging.getLogger()
logger.setLevel(logging.INFO)
logger.addHandler(logging.StreamHandler())

# May need to import additional metrics depending on what you are measuring.
# See https://docs.aws.amazon.com/sagemaker/latest/dg/model-monitor-model-quality-metrics.html
from sklearn.metrics import classification_report, roc_auc_score, accuracy_score

def get_dataset(dir_path, dataset_name) -> pd.DataFrame:
    files = [ os.path.join(dir_path, file) for file in os.listdir(dir_path) ]
    if len(files) == 0:
        raise ValueError(('There are no files in {}.\n' +
                          'This usually indicates that the channel ({}) was incorrectly specified,\n' +
                          'the data specification in S3 was incorrectly specified or the role specified\n' +
                          'does not have permission to access the data.').format(files, dataset_name))
    raw_data = [ pd.read_csv(file, header=None) for file in files ]
    df = pd.concat(raw_data)
    return df

if __name__ == "__main__":
    model_path = "/opt/ml/processing/model/model.tar.gz"
    with tarfile.open(model_path) as tar:
        tar.extractall(path="..")

    logger.debug("Loading xgboost model.")
    model = pickle.load(open("xgboost-model", "rb"))

    logger.info("Loading test input data")
    test_path = "/opt/ml/processing/test"
    df = get_dataset(test_path, "test_set")

    logger.debug("Reading test data.")
    y_test = df.iloc[:, 0].to_numpy()
    df.drop(df.columns[0], axis=1, inplace=True)
    X_test = xgboost.DMatrix(df.values)

    logger.info("Performing predictions against test data.")
    predictions_probs = model.predict(X_test)
    predictions = predictions_probs.round()

    logger.info("Creating classification evaluation report")
    acc = accuracy_score(y_test, predictions)
    auc = roc_auc_score(y_test, predictions_probs)

    # The metrics reported can change based on the model used, but it must be a specific name per (https://docs.aws.amazon.com/sagemaker/latest/dg/model-monitor-model-quality-metrics.html)
    report_dict = {
        "binary_classification_metrics": {
            "accuracy": {
                "value": acc,
                "standard_deviation": "NaN",
            },
            "auc": {"value": auc, "standard_deviation": "NaN"},
        },
    }

    logger.info("Classification report:\n{}".format(report_dict))

    evaluation_output_path = os.path.join(
        "/opt/ml/processing/evaluation", "evaluation.json"
    )
    logger.info("Saving classification report to {}".format(evaluation_output_path))

    with open(evaluation_output_path, "w") as f:
        f.write(json.dumps(report_dict))<jupyter_output>Writing evaluate.py<jupyter_text>Let us run the above script<jupyter_code>from sagemaker.processing import (
    ProcessingInput,
    ProcessingOutput,
    ScriptProcessor,
)
# Processing step for evaluation
processor = ScriptProcessor(
    image_uri=docker_image_name,
    command=["python3"],
    instance_type="ml.m5.xlarge",
    instance_count=1,
    base_job_name="readmission/eval-script",
    sagemaker_session=sm_sess,
    role=role,
)
entrypoint = "evaluate.py"
from time import strftime, gmtime
# Helper to create timestamps
create_date = lambda: strftime("%Y-%m-%d-%H-%M-%S", gmtime())
processor.run(
    code=entrypoint,
    inputs=[
        sagemaker.processing.ProcessingInput(
            source=s3uri_model,
            destination="/opt/ml/processing/model",
        ),
        sagemaker.processing.ProcessingInput(
            source=s3uri_test,
            destination="/opt/ml/processing/test",
        ),
    ],
    outputs=[
        sagemaker.processing.ProcessingOutput(
            output_name="evaluation", source="/opt/ml/processing/evaluation"
        ),
    ],
    job_name=f"ReadmissionEval-{create_date()}"
)
for proc_in in processor.latest_job.inputs:
    if proc_in.input_name == "code":
        s3_evaluation_code_uri = proc_in.source 
        
s3_evaluation_code_uri
%store s3_evaluation_code_uri<jupyter_output>Stored 's3_evaluation_code_uri' (str)<jupyter_text>Let's check our evaluation report<jupyter_code>out_s3_report_uri = processor.latest_job.outputs[0].destination
out_s3_report_uri
reports_list = S3Downloader.list(out_s3_report_uri)
reports_list
report = S3Downloader.read_file(reports_list[0])

print("=====Model Report====")
print(json.dumps(json.loads(report.split('\n')[0]), indent=2))
model_name="xgboost-readmission-1"

model = estimator.create_model(name=model_name)
container_def = model.prepare_container_def()
sm_sess.create_model(model_name,
                     role,
                     container_def)
from sagemaker import clarify
clarify_processor = clarify.SageMakerClarifyProcessor(role=role,
                                                      instance_count=1,
                                                      instance_type='ml.m5.xlarge',
                                                      sagemaker_session=sm_sess)
columns_headers = ['Readmission.Status', 'ER', 'LOS', 'Age', 'HCC.Riskscore	',
       'Gender_F', 'Gender_M', 'Race_Black', 'Race_Hispanic', 'Race_Other',
       'Race_White', 'DRG.class_MED', 'DRG.class_SURG', 'DRG.class_UNGROUP', 'DRG.Complication_MedicalMCC.CC',
       'DRG.Complication_MedicalNoC', 'DRG.Complication_Other', 'DRG.Complication_SurgMCC.CC', 'DRG.Complication_SurgNoC']<jupyter_output><empty_output><jupyter_text>Writing ModelConfig and Bias config<jupyter_code>model_config = clarify.ModelConfig(model_name=model_name,
                                   instance_type='ml.m5.xlarge',
                                   instance_count=1,
                                   accept_type='text/csv',
                                   content_type='text/csv')<jupyter_output><empty_output><jupyter_text>Explaining Predictions using SHAP<jupyter_code>df.columns = columns_headers[1:]
df
shap_config = clarify.SHAPConfig(baseline=[df.iloc[0].values.tolist()],
                                 num_samples=20,
                                 agg_method='mean_abs',
                                 save_local_shap_values=False)

explainability_output_path = 's3://{}/{}/clarify-explainability'.format(bucket, prefix)

explainability_data_config = clarify.DataConfig(s3_data_input_path=s3uri_train,
                                s3_output_path=explainability_output_path,
                                label='Readmission.Status',
                                headers=columns_headers,
                                dataset_type='text/csv')

shap_config = clarify.SHAPConfig(
    baseline=[test_features.iloc[0].values.tolist()],
    num_samples=15,
    agg_method="mean_abs",
    save_local_shap_values=True,
)


clarify_processor.run_explainability(data_config=explainability_data_config,
                                     model_config=model_config,
                                     explainability_config=shap_config)<jupyter_output>INFO:sagemaker:Creating processing-job with name Clarify-Explainability-2022-03-01-17-53-32-465
{'fraction_non_alphanumeric': 0.10637367442877446, 'fraction_numerical': 0.02270325425458256, 'mean_word_length': 3.068495181616012, 'pattern_counts': {'":': 35, '<': 78, '<?xml version=': 0, '>': 79, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 70, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '22', 'dir': 'jupyter-structured-clean-dedup', 'id': '514447', 'license': 'MIT', 'n_tokens_mistral': 9067, 'n_tokens_neox': 8571, 'n_words': 1745, 'path': 'Reliance_Test.ipynb', 'repo_name': 'Ayo-faks/reliance-health-test', 'size': '27441'}