<jupyter_start><jupyter_text>Amazon SageMaker inference deployment to CPUs, GPUs, and EIThis example demonstrates Amazon SageMaker inference deployment using SageMaker SDKThis example was tested on Amazon SageMaker Studio NotebookRun this notebook using the following Amazon SageMaker Studio conda environment:`TensorFlow 2 CPU Optimized`<jupyter_code># !pip install --upgrade pip -q
# !pip install --upgrade sagemaker -q
import tarfile
import sagemaker
import tensorflow as tf
import tensorflow.keras as keras
import shutil
import os
import time
from tensorflow.keras.applications.resnet50 import ResNet50

role = sagemaker.get_execution_role()
sess = sagemaker.Session()
region = sess.boto_region_name
bucket = sess.default_bucket()
print('sagemaker version: '+sagemaker.__version__)
print('tensorflow version: '+tf.__version__)
def load_save_resnet50_model(model_path):
    model = ResNet50(weights='imagenet')
    shutil.rmtree(model_path, ignore_errors=True)
    model.save(model_path, include_optimizer=False, save_format='tf')

saved_model_dir = 'resnet50_saved_model' 
model_ver = '1'
model_path = os.path.join(saved_model_dir, model_ver)

# load_save_resnet50_model(model_path)
shutil.rmtree('model.tar.gz', ignore_errors=True)
!tar cvfz model.tar.gz -C resnet50_saved_model .
from sagemaker.tensorflow.model import TensorFlowModel, TensorFlowPredictor

prefix = 'keras_models'
s3_model_path = sess.upload_data(path='model.tar.gz', key_prefix=prefix)

model = TensorFlowModel(model_data=s3_model_path, 
                        framework_version='1.15',
                        role=role,
                        predictor_cls = TensorFlowPredictor,
                        sagemaker_session=sess)<jupyter_output><empty_output><jupyter_text>Deploy to CPU instance<jupyter_code>predictor_cpu = model.deploy(initial_instance_count=1, 
                             instance_type='ml.c5.xlarge')<jupyter_output>update_endpoint is a no-op in sagemaker>=2.
See: https://sagemaker.readthedocs.io/en/stable/v2.html for details.<jupyter_text>Deploy using EI<jupyter_code>predictor_ei = model.deploy(initial_instance_count=1, 
                            instance_type='ml.c5.xlarge',
                            accelerator_type='ml.eia2.large')<jupyter_output>update_endpoint is a no-op in sagemaker>=2.
See: https://sagemaker.readthedocs.io/en/stable/v2.html for details.<jupyter_text>Deploy to GPU instance<jupyter_code>predictor_gpu = model.deploy(initial_instance_count=1, 
                         instance_type='ml.g4dn.xlarge')<jupyter_output>update_endpoint is a no-op in sagemaker>=2.
See: https://sagemaker.readthedocs.io/en/stable/v2.html for details.<jupyter_text>Test endpoint<jupyter_code>## If you have an existing endpoint, create a predictor using the endpoint name

# from sagemaker.tensorflow.model import TensorFlowPredictor
# predictor = TensorFlowPredictor('ENDPOINT_NAME',
#                                sagemaker_session=sess)
def image_preprocess(img, reps=1):
    img = np.asarray(img.resize((224, 224)))
    img = np.stack([img]*reps)
    img = tf.keras.applications.resnet50.preprocess_input(img)
    return img
from PIL import Image 
import numpy as np
import json

img= Image.open('kitten.jpg')
img = image_preprocess(img, 5)<jupyter_output><empty_output><jupyter_text>Invoke CPU endpoint<jupyter_code>response = predictor_cpu.predict(data=img)
probs = np.array(response['predictions'][0])
tf.keras.applications.resnet.decode_predictions(np.expand_dims(probs, axis=0), top=5)<jupyter_output><empty_output><jupyter_text>Invoke CPU Instance + EI endpoint<jupyter_code>response = predictor_ei.predict(data=img)
probs = np.array(response['predictions'][0])
tf.keras.applications.resnet.decode_predictions(np.expand_dims(probs, axis=0), top=5)<jupyter_output><empty_output><jupyter_text>Invoke G4 GPU Instance with NVIDIA T4 endpoint<jupyter_code>response = predictor_gpu.predict(data=img)
probs = np.array(response['predictions'][0])
tf.keras.applications.resnet.decode_predictions(np.expand_dims(probs, axis=0), top=5)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10179053225410842, 'fraction_numerical': 0.013490311503556536, 'mean_word_length': 5.113943028485757, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 33, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '507421', 'license': 'Apache-2.0', 'n_tokens_mistral': 1403, 'n_tokens_neox': 1340, 'n_words': 247, 'path': 'sagemaker-tf-cpu-gpu-ei-resnet50.ipynb', 'repo_name': 'manchann/ai-accelerators-examples', 'size': '4077'}