<jupyter_start><jupyter_text>Prepare datasets<jupyter_code>data.head()
data = data.drop(['neckline', 'occasion', 'theme'], axis=1)
data.head()
data = data[data['length'].notnull()]
data = data[data['dress_types'].notnull()]
data = data[data['color'].notnull()]
data = data[data['material'].notnull()]
data.head()
data.shape<jupyter_output><empty_output><jupyter_text>Data encoding<jupyter_code>from sklearn.preprocessing import LabelEncoder


def encode_data(data_frame, target_attr):
    encoder = LabelEncoder()
    encoder.fit(data_frame[target_attr])

    encode_row = lambda x: encoder.transform(x) if x.name == target_attr else x
    data_frame = data_frame.apply(encode_row)

    return data_frame, encoder.classes_.tolist()
tmp_df = data
tmp_df, encoding_length_dict = encode_data(tmp_df,'length')
tmp_df, encoding_dress_types_dict = encode_data(tmp_df,'dress_types')
tmp_df, encoding_color_dict = encode_data(tmp_df,'color')
tmp_df, encoding_material_dict = encode_data(tmp_df,'material')
prepared_df = tmp_df
prepared_df.head()
target_attributes = ['length','dress_types','color','material']
dict = {'length': encoding_length_dict,
       'dress_types': encoding_dress_types_dict,
       'color': encoding_color_dict,
       'material': encoding_material_dict}
for attr in target_attributes:
    print('Encoding {}:\n {}\n'.format(attr, dict[attr]))<jupyter_output>Encoding length:
 ['high low', 'knee length', 'long', 'short']

Encoding dress_types:
 ['a-line', 'fit & flare', 'gown', 'halter', 'high waist', 'maxi', 'sheath', 'shift', 'tank', 'tunic', 'wrap']

Encoding color:
 ['black', 'blue', 'brown', 'gold', 'gray', 'green', 'khaki', 'multi', 'navy', 'olive', 'orange', 'pink', 'red', 'silver', 'white', 'yellow']

Encoding material:
 ['cotton', 'elastan', 'jersey', 'lace', 'modal', 'nylon', 'polyester', 'rayon', 'spandex', 'viscose']<jupyter_text>Prepare ML manifestAWS Documents<jupyter_code>prepared_df['source-ref'] = prepared_df.image_path.apply(lambda x:'s3://{}/visual-attribution/{}'.format(ml_data_bucket,x))
prepared_df.head()
from sklearn.model_selection import train_test_split
train_df, validate_df = train_test_split(prepared_df, test_size=0.25)
train_df.head()
validate_df.head()
from io import StringIO
import boto3

def saveDFasJsonToS3(df, key):
    buffer = StringIO()
    df.to_json(buffer, orient='records', lines=True)
    s3_resource = boto3.resource('s3')
    s3_resource.Object(ml_data_bucket, key).put(Body=buffer.getvalue())
target_columns = ['source-ref', 'length','dress_types','color','material']
target_train_df = train_df[target_columns]
target_validation_df = validate_df[target_columns]
target_train_df.head()
target_validation_df.head()
train_key = 'visual-attribution/train-manifest.json'
validation_key = 'visual-attribution/validation-manifest.json'
saveDFasJsonToS3(target_train_df, train_key)
saveDFasJsonToS3(target_validation_df, validation_key)
print('1) Length')
print('num_training_samples = {}'.format(target_train_df.shape[0]))
print('num_classes = {}'.format(len(encoding_length_dict)))
print('\n')
print('2) Dress type')
print('num_training_samples = {}'.format(target_train_df.shape[0]))
print('num_classes = {}'.format(len(encoding_dress_types_dict)))
print('\n')
print('3) Color')
print('num_training_samples = {}'.format(target_train_df.shape[0]))
print('num_classes = {}'.format(len(encoding_color_dict)))
print('\n')
print('4) Material')
print('num_training_samples = {}'.format(target_train_df.shape[0]))
print('num_classes = {}'.format(len(encoding_material_dict)))<jupyter_output>1) Length
num_training_samples = 15342
num_classes = 4


2) Dress type
num_training_samples = 15342
num_classes = 11


3) Color
num_training_samples = 15342
num_classes = 16


4) Material
num_training_samples = 15342
num_classes = 10<jupyter_text>TrainingRun the training using Amazon sagemaker CreateTrainingJob API<jupyter_code>s3_train_manifest = 's3://adp-ml-datasets/visual-attribution/train-manifest.json'
s3_validation_manifest = 's3://adp-ml-datasets/visual-attribution/validation-manifest.json'
output_model_path = 's3://adp-ml-models/visual-attribution/{}/model'
def create_job_name(target_field):
    return 'visual-attributes-{}-train-job-{}'.format(target_field, 
                                                    time.strftime('%Y-%m-%d-%H%M%S', time.gmtime())).replace('_','-')
num_training_samples = target_train_df.shape[0]
#Hyper Parameters
def create_hyper_params(encoding_dict, num_training_samples):
    return {
        'beta_1': '0.9',
        'beta_2': '0.999',
        'checkpoint_frequency': '1',
        'early_stopping': 'false',
        'epochs': '25',
        'eps': '1e-8',
        'gamma': '0.9',
        'image_shape': '3,800,800',
        'learning_rate': '0.001',
        'lr_scheduler_factor': '0.1',
        'mini_batch_size': '12',
        'momentum': '0.9',
        'multi_label': '0',
        'num_classes': str(len(encoding_dict)),
        'num_layers': '34',
        'num_training_samples': str(num_training_samples),
        'optimizer': 'adam',
        'precision_dtype': 'float32',
        'use_pretrained_model': '1',
        'use_weighted_loss': '0',
        'weight_decay': '0.0001'
    }
def create_training_params(target_field, job_name, s3_train_manifest, s3_validation_manifest,
                           hyper_params, output_model_path):
    training_image = get_image_uri(boto3.Session().region_name, 'image-classification')
    role = get_execution_role()
    return \
{
    # specify the training docker image
    "AlgorithmSpecification": {
        "TrainingImage": training_image,
        "TrainingInputMode": "Pipe"
    },
    "RoleArn": role,
    "OutputDataConfig": {
        "S3OutputPath": output_model_path
    },
    "ResourceConfig": {
        "InstanceCount": 1,
        "InstanceType": "ml.p2.xlarge",
        "VolumeSizeInGB": 64
    },
    "TrainingJobName": job_name,
    "HyperParameters": {
        "beta_1": hyper_params['beta_1'],
        "beta_2": hyper_params['beta_2'],
        "checkpoint_frequency": hyper_params['checkpoint_frequency'],
        "early_stopping": hyper_params['early_stopping'],
        "epochs": hyper_params['epochs'],
        "eps": hyper_params['eps'],
        "gamma": hyper_params['gamma'],
        "image_shape": hyper_params['image_shape'],
        "learning_rate": hyper_params['learning_rate'],
        "lr_scheduler_factor": hyper_params['lr_scheduler_factor'],
        "mini_batch_size": hyper_params['mini_batch_size'],
        "momentum": hyper_params['momentum'],
        "multi_label": hyper_params['multi_label'],
        "num_classes": hyper_params['num_classes'],
        "num_layers": hyper_params['num_layers'],
        "num_training_samples": hyper_params['num_training_samples'],
        "optimizer": hyper_params['optimizer'],
        "precision_dtype": hyper_params['precision_dtype'],
        "use_pretrained_model": hyper_params['use_pretrained_model'],
        "use_weighted_loss": hyper_params['use_weighted_loss'],
        "weight_decay": hyper_params['weight_decay'],
        
    },
    "StoppingCondition": {
        "MaxRuntimeInSeconds": 172800
    },
    "InputDataConfig": [
        {
            "ChannelName": "train",
            "DataSource": {
                "S3DataSource": {
                    "S3DataType": "AugmentedManifestFile",
                    "AttributeNames": ['source-ref',target_field],
                    "S3Uri": s3_train_manifest,
                    "S3DataDistributionType": "FullyReplicated"
                }
            },
            "InputMode": "Pipe",
            "ContentType": "application/x-image",
            "CompressionType": "None",
            "RecordWrapperType": "RecordIO"
        },
        {
            "ChannelName": "validation",
            "DataSource": {
                "S3DataSource": {
                    "S3DataType": "AugmentedManifestFile",
                    "AttributeNames": ['source-ref',target_field],
                    "S3Uri": s3_validation_manifest,
                    "S3DataDistributionType": "FullyReplicated"
                }
            },
            "InputMode": "Pipe",
            "ContentType": "application/x-image",
            "CompressionType": "None",
            "RecordWrapperType": "RecordIO"
        }
    ]
}
config = {}
for attr in target_attributes:
    config[attr] = {}
    config[attr]['encoder'] = dict[attr]
    config[attr]['job_name'] = create_job_name(attr)
    config[attr]['hyper_params'] = create_hyper_params(config[attr]['encoder'], num_training_samples)
    config[attr]['output_model_path'] = output_model_path.format(attr)
    config[attr]['training_params'] = create_training_params(attr, 
                                                             config[attr]['job_name'],
                                                             s3_train_manifest, s3_validation_manifest,
                                                             config[attr]['hyper_params'], 
                                                             config[attr]['output_model_path'])
    
#print('Configuration:\n {}'.format(json.dumps(config, indent=2)))
for attr in target_attributes:
    print('Training job for {} with name: {}'.format(attr, config[attr]['job_name']))
def start_training_job(sagemaker, target_field, job_name, training_params):
    # create the Amazon SageMaker training job
    sagemaker.create_training_job(**training_params)
    # confirm that the training job has started
    status = sagemaker.describe_training_job(TrainingJobName=job_name)['TrainingJobStatus']
    print('Training job[{}] started with current status: {}'.format(target_field, status))
def wait_training_job(sagemaker, target_field, job_name):
    try:
        # wait for the job to finish and report the ending status
        sagemaker.get_waiter('training_job_completed_or_stopped').wait(TrainingJobName=job_name)
        training_info = sagemaker.describe_training_job(TrainingJobName=job_name)
        status = training_info['TrainingJobStatus']
        print("Training job[{}] ended with status: {}".format(target_field, status))
    except:
        print('Training job[{}] failed to start'.format(target_field))
         # if exception is raised, that means it has failed
        message = sagemaker.describe_training_job(TrainingJobName=job_name)['FailureReason']
        print('Training job[{}] failed with the following error: {}'.format(target_field, message))
sagemaker = boto3.client(service_name='sagemaker')
# Run train jobs for length
attr = 'length'
start_training_job(sagemaker, attr, config[attr]['job_name'], config[attr]['training_params'])
#wait_training_job(sagemaker, attr, config[attr]['job_name'])
# Run train jobs for color
attr = 'color'
start_training_job(sagemaker, attr, config[attr]['job_name'], config[attr]['training_params'])
#wait_training_job(sagemaker, attr, config[attr]['job_name'])
# Run train jobs for dress_types
attr = 'dress_types'
start_training_job(sagemaker, attr, config[attr]['job_name'], config[attr]['training_params'])
#wait_training_job(sagemaker, attr, config[attr]['job_name'])
# Run train jobs for material
attr = 'material'
start_training_job(sagemaker, attr, config[attr]['job_name'], config[attr]['training_params'])
#wait_training_job(sagemaker, attr, config[attr]['job_name'])
for attr in target_attributes:
    status = sagemaker.describe_training_job(TrainingJobName=config[attr]['job_name'])['TrainingJobStatus']
    print("Training job[{}] ended with status: {}".format(attr, status))<jupyter_output>Training job[length] ended with status: Completed
Training job[dress_types] ended with status: Completed
Training job[color] ended with status: Completed
Training job[material] ended with status: Completed<jupyter_text>Create ModelWe now create a SageMaker Model from the training output. Using the model we can create a Batch Transform Job or an Endpoint Configuration.<jupyter_code>def create_model(sage, target_field, job_name): 
    role = get_execution_role()
    model_name="visual-attributes-{}-model-{}".format(target_field, time.strftime('%Y-%m-%d-%H%M%S', time.gmtime())).replace('_','-')
    print('Model name: ' + model_name)
    info = sage.describe_training_job(TrainingJobName=job_name)
    model_data = info['ModelArtifacts']['S3ModelArtifacts']
    print(model_data)

    hosting_image = get_image_uri(boto3.Session().region_name, 'image-classification')

    primary_container = {
        'Image': hosting_image,
        'ModelDataUrl': model_data,
    }

    create_model_response = sage.create_model(ModelName = model_name,
                                              ExecutionRoleArn = role,
                                              PrimaryContainer = primary_container)
    print(create_model_response['ModelArn'])
    return model_name
for attr in target_attributes:
    model_name = create_model(sagemaker, attr, config[attr]['job_name'])
    config[attr]['model_name'] = model_name<jupyter_output>Model name: visual-attributes-length-model-2020-04-15-073912
s3://adp-ml-models/visual-attribution/length/model/visual-attributes-length-train-job-2020-04-11-171718/output/model.tar.gz
arn:aws:sagemaker:us-east-1:592068120969:model/visual-attributes-length-model-2020-04-15-073912
Model name: visual-attributes-dress-types-model-2020-04-15-073912
s3://adp-ml-models/visual-attribution/dress_types/model/visual-attributes-dress-types-train-job-2020-04-11-171718/output/model.tar.gz
arn:aws:sagemaker:us-east-1:592068120969:model/visual-attributes-dress-types-model-2020-04-15-073912
Model name: visual-attributes-color-model-2020-04-15-073914
s3://adp-ml-models/visual-attribution/color/model/visual-attributes-color-train-job-2020-04-11-171718/output/model.tar.gz
arn:aws:sagemaker:us-east-1:592068120969:model/visual-attributes-color-model-2020-04-15-073914
Model name: visual-attributes-material-model-2020-04-15-073916
s3://adp-ml-models/visual-attribution/material/model/visual-attributes-materia[...]<jupyter_text>Batch transformWe now create a SageMaker Batch Transform job using the model created above to perform batch prediction.<jupyter_code>target_field = 'color'
batch_job_name = "visual-attributes-{}-transform-job-{}".format(target_field, time.strftime('%Y-%m-%d-%H%M%S', time.gmtime()))
stage_bucket = 'adp-ml-stage'
batch_input = 's3://{}/visual-attribution/input/'.format(stage_bucket)
batch_output = 's3://{}/{}/output'.format(stage_bucket, batch_job_name)
request = \
{
    "TransformJobName": batch_job_name,
    "ModelName": model_name,
    "MaxConcurrentTransforms": 16,
    "MaxPayloadInMB": 6,
    "BatchStrategy": "SingleRecord",
    "TransformOutput": {
        "S3OutputPath": batch_output
    },
    "TransformInput": {
        "DataSource": {
            "S3DataSource": {
                "S3DataType": "S3Prefix",
                "S3Uri": batch_input
            }
        },
        "ContentType": "application/x-image",
        "SplitType": "None",
        "CompressionType": "None"
    },
    "TransformResources": {
            "InstanceType": "ml.p2.xlarge",
            "InstanceCount": 1
    }
}

print('Transform job name: {}'.format(batch_job_name))
sagemaker = boto3.client('sagemaker')
sagemaker.create_transform_job(**request)

print("Created Transform job[{}] with name: {}".format(target_field, batch_job_name))

while(True):
    response = sagemaker.describe_transform_job(TransformJobName=batch_job_name)
    status = response['TransformJobStatus']
    if status == 'Completed':
        print("Transform job[{}] ended with status: {}".format(target_field, status))
        break
    if status == 'Failed':
        message = response['FailureReason']
        print('Transform job[{}] failed with the following error: {}'.format(target_field, message))
        raise Exception('Transform job failed') 
    time.sleep(30)<jupyter_output>Created Transform job[color] with name: visual-attributes-color-transform-job-2020-04-15-074059
Transform job[color] ended with status: Completed<jupyter_text>Registr endpoint Create Endpoint Configuration<jupyter_code>from time import gmtime, strftime

def create_endpoint_config(sage, target_field, model_name):
    timestamp = time.strftime('%Y-%m-%d-%H%M%S', time.gmtime())
    endpoint_config_name = "visual-attributes-{}-endpoint-job-".format(target_field) + timestamp
    endpoint_config_name = endpoint_config_name.replace('_','-')
    endpoint_config_response = sage.create_endpoint_config(
        EndpointConfigName = endpoint_config_name,
        ProductionVariants=[{
            'InstanceType':'ml.m4.xlarge',
            'InitialInstanceCount':1,
            'ModelName':model_name,
            'VariantName':'AllTraffic'}])

    print('Endpoint configuration name: {}'.format(endpoint_config_name))
    print('Endpoint configuration arn:  {}'.format(endpoint_config_response['EndpointConfigArn']))
    return endpoint_config_name
for attr in target_attributes:
    config[attr]['endpoint_config'] = create_endpoint_config(sagemaker, attr, config[attr]['model_name'])<jupyter_output>Endpoint configuration name: visual-attributes-length-endpoint-job-2020-04-15-080306
Endpoint configuration arn:  arn:aws:sagemaker:us-east-1:592068120969:endpoint-config/visual-attributes-length-endpoint-job-2020-04-15-080306
Endpoint configuration name: visual-attributes-dress-types-endpoint-job-2020-04-15-080306
Endpoint configuration arn:  arn:aws:sagemaker:us-east-1:592068120969:endpoint-config/visual-attributes-dress-types-endpoint-job-2020-04-15-080306
Endpoint configuration name: visual-attributes-color-endpoint-job-2020-04-15-080311
Endpoint configuration arn:  arn:aws:sagemaker:us-east-1:592068120969:endpoint-config/visual-attributes-color-endpoint-job-2020-04-15-080311
Endpoint configuration name: visual-attributes-material-endpoint-job-2020-04-15-080311
Endpoint configuration arn:  arn:aws:sagemaker:us-east-1:592068120969:endpoint-config/visual-attributes-material-endpoint-job-2020-04-15-080311<jupyter_text>Create Endpoint<jupyter_code>import time

def create_endpoint(sage, target_field, endpoint_config_name):
    timestamp = time.strftime('%Y-%m-%d-%H-%M-%S', time.gmtime())
    endpoint_name = "visual-attributes-{}-endpoint-{}".format(target_field, timestamp)
    endpoint_name = endpoint_name.replace('_','-')
    print('Endpoint name: {}'.format(endpoint_name))

    endpoint_params = {
        'EndpointName': endpoint_name,
        'EndpointConfigName': endpoint_config_name,
    }
    endpoint_response = sage.create_endpoint(**endpoint_params)
    print('EndpointArn = {}'.format(endpoint_response['EndpointArn']))
    # get the status of the endpoint
    response = sage.describe_endpoint(EndpointName=endpoint_name)
    status = response['EndpointStatus']
    print('EndpointStatus = {}'.format(status))

    # wait until the status has changed
    sage.get_waiter('endpoint_in_service').wait(EndpointName=endpoint_name)

    # print the status of the endpoint
    endpoint_response = sage.describe_endpoint(EndpointName=endpoint_name)
    status = endpoint_response['EndpointStatus']
    print('Endpoint creation ended with EndpointStatus = {}'.format(status))

    if status != 'InService':
        raise Exception('Endpoint creation failed.')
    return endpoint_name
for attr in target_attributes:
    config[attr]['endpoint_name'] = create_endpoint(sagemaker, attr, config[attr]['endpoint_config'])
for attr in target_attributes:
    print('Endpoint {}: {}'.format(attr, config[attr]['endpoint_name']))<jupyter_output>Endpoint length: visual-attributes-length-endpoint-2020-04-15-08-03-19
Endpoint dress_types: visual-attributes-dress-types-endpoint-2020-04-15-08-11-51
Endpoint color: visual-attributes-color-endpoint-2020-04-15-08-21-23
Endpoint material: visual-attributes-material-endpoint-2020-04-15-08-29-55<jupyter_text>Call endpoint<jupyter_code>test_img_path = 'visual-attribution/input/test-3.jpg'
test_img = Image.open(read_object(test_img_path, stage_bucket))
test_img
def image_to_byte_array(image:Image):
  io_byte = io.BytesIO()
  image.save(io_byte, format=image.format)
  byte_arr = io_byte.getvalue()
  return byte_arr
target_attribute = 'color'
import json
import numpy as np
import boto3

runtime = boto3.Session().client(service_name='runtime.sagemaker') 
img_to_request = image_to_byte_array(test_img)
response = runtime.invoke_endpoint(EndpointName=config[target_attribute]['endpoint_name'], 
                                   ContentType='application/x-image', 
                                   Body=img_to_request)
result = json.loads(response['Body'].read())
index = np.argmax(result)
encoding_color_dict[index]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13694190490011127, 'fraction_numerical': 0.02771731243651139, 'mean_word_length': 4.256547165013984, 'pattern_counts': {'":': 76, '<': 30, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '182807', 'license': 'Apache-2.0', 'n_tokens_mistral': 6827, 'n_tokens_neox': 6345, 'n_words': 1058, 'path': 'notebooks/visual-attribution.ipynb', 'repo_name': 'AlexKbit/visual-attribution', 'size': '20673'}