<jupyter_start><jupyter_text>Train Model using Boto3- AWS Boto3: create_training_job    - https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_CreateTrainingJob.html- Step 5: Train a Model        - Amazon SageMaker Python SDK, AWS SDK for Python (Boto3) 의 두가지 방법을 통한 모델 훈련 방법    - https://docs.aws.amazon.com/sagemaker/latest/dg/ex1-train-model.html<jupyter_code>import sagemaker
from sagemaker import get_execution_role
import json
import os
import boto3
import datetime
import logging


# Project name
project_name = 'predict-deliver-time'
str_time = datetime.datetime.now.strftime('%Y-%m-%d-%H-%M-%S-%f')[:-3]

# Region
region = boto3.Session().region_name
# Bucket
bucket = sagemaker.Session().default_bucket() 
# role
role = get_execution_role()

from utils import upload_s3

# Upload train. test file
data_prefix = 'autogluon/predict-deliver-time/data'
train_file = 'auto_train.csv'
test_file = 'auto_test.csv'
s3_train_file_path = upload_s3(bucket, train_file, data_prefix)
s3_test_file_path = upload_s3(bucket, test_file, data_prefix)
print("s3_train_file_path: ", s3_train_file_path)
print("s3_test_file_path: ", s3_test_file_path)
from sagemaker.mxnet import MXNet

# Resource Type
instance_type = 'ml.c5.9xlarge'
instance_type = 'local'
training_image = '763104351884.dkr.ecr.{}.amazonaws.com/mxnet-training:1.6.0-cpu-py3'.format(region)

mx_estimator = MXNet('autogluon-tab-with-test.py',
                     image_uri = training_image,
                     role = role,
                     instance_type = instance_type,
                     instance_count = 1,
                    )

                     
                     
# from sagemaker.inputs import TrainingInput

# s3_train = TrainingInput(s3_train_file_path, distribution='FullyReplicated') 
# s3_test = TrainingInput(s3_test_file_path, distribution='FullyReplicated')
# mx_estimator.fit( {'training': s3_train_file_path, 'test': s3_test_file_path})<jupyter_output><empty_output><jupyter_text>Boto3<jupyter_code>from utils import upload_s3

# Upload train. test file
data_prefix = 'autogluon/predict-deliver-time/data'
train_file = 'auto_train.csv'
test_file = 'auto_test.csv'
s3_train_file_path = upload_s3(bucket, train_file, data_prefix)
s3_test_file_path = upload_s3(bucket, test_file, data_prefix)
print("s3_train_file_path: ", s3_train_file_path)
print("s3_test_file_path: ", s3_test_file_path)

## Hyperparameters
train_code_file = 'autogluon-tab-with-test.py'
train_file_name = 'auto_train.csv'

# Upload source tar file
source_prefix = 'autogluon/predict-deliver-time/source'
code_tar_file = 'sourcedir.tar.gz'

s3_code_tar_path = upload_s3(bucket, code_tar_file, source_prefix)
print("s3_code_tar_path: ", s3_code_tar_path)


# Ouput file
output_prefix = 'autogluon/predict-deliver-time/output'
s3_output_path = 's3://{}/{}/'.format(bucket, output_prefix)
print("s3_output_path: ", s3_output_path)

# Model Artifact
model_artifact_prefix = 'autogluon/predict-deliver-time/model'
s3_model_artifact_path = 's3://{}/{}/'.format(bucket, model_artifact_prefix)
print("model_artifact_prefix: ", model_artifact_prefix)


## AlgorithmSpecification
training_image = '763104351884.dkr.ecr.{}.amazonaws.com/mxnet-training:1.6.0-cpu-py3'.format(region)

training_job_params = {
    'TrainingJobName': project_name + '-autogluon-' + str_time,
    'HyperParameters': {
      'filename':json.dumps(train_file_name),
      'sagemaker_container_log_level': '20',
      'sagemaker_enable_cloudwatch_metrics': 'false',
      'sagemaker_program': train_code_file,
      'sagemaker_region': region,
      'sagemaker_submit_directory': s3_code_tar_path,
      's3-output': s3_output_path
    },
    'AlgorithmSpecification': {
      'TrainingImage': training_image,
      'TrainingInputMode': 'File',
      'EnableSageMakerMetricsTimeSeries': False
    },
    'RoleArn': role,
    'InputDataConfig': [
      {
        'ChannelName': 'training',
        'DataSource': {
          'S3DataSource': {
            'S3DataType': 'S3Prefix',
            'S3Uri': s3_train_file_path,
            'S3DataDistributionType': 'FullyReplicated'
          }
        },
        'CompressionType': 'None',
        'RecordWrapperType': 'None'
      }
    ],
    'OutputDataConfig': {
      'KmsKeyId': '',
      'S3OutputPath': s3_model_artifact_path
    },
    'ResourceConfig': {
      'InstanceType': instance_type,
      'InstanceCount': 1,
      'VolumeSizeInGB': 30
    },
    'StoppingCondition': {
      'MaxRuntimeInSeconds': 86400
    },
    'EnableNetworkIsolation': False,
    'EnableInterContainerTrafficEncryption': False,
    'EnableManagedSpotTraining': False,
}
  
print("training_job_params:\n", training_job_params)



sm = boto3.Session().client('sagemaker')
response = sm.create_training_job(**training_job_params)
import json
import os
import boto3
import datetime
from urllib.parse import unquote_plus
import logging
logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(message)s',
                    datefmt='%Y-%m-%d %H:%M:%S')

def lambda_handler(event, context):
  for record in event['Records']:
    bucket = record['s3']['bucket']['name']
    key = unquote_plus(record['s3']['object']['key'])
    tmpkey = key.replace('/', '')
  logging.info(key)
  logging.info(tmpkey)
  filename = key.split('/')[-1]
  print(filename) # auto_train.csv
  dataset = filename.split('_')[0]
  print(dataset) # auto
  
  now = datetime.datetime.now
  str_time = now().strftime('%Y-%m-%d-%H-%M-%S-%f')[:-3]
  sm = boto3.Session().client('sagemaker')
  training_job_params = {
    'TrainingJobName': dataset + '-autogluon-' + str_time,
    'HyperParameters': {
      'filename':json.dumps(filename),
      'sagemaker_container_log_level': '20',
      'sagemaker_enable_cloudwatch_metrics': 'false',
      'sagemaker_program': 'autogluon-tab-with-test.py',
      'sagemaker_region': os.environ['AWS_REGION'],
      'sagemaker_submit_directory': 's3://' + bucket + '/source/sourcedir.tar.gz',
      's3-output': os.environ['S3_OUTPUT_PATH']
    },
    'AlgorithmSpecification': {
      'TrainingImage': '763104351884.dkr.ecr.' + os.environ['AWS_REGION'] + '.amazonaws.com/mxnet-training:1.6.0-cpu-py3',
      'TrainingInputMode': 'File',
      'EnableSageMakerMetricsTimeSeries': False
    },
    'RoleArn': os.environ['SAGEMAKER_ROLE_ARN'],
    'InputDataConfig': [
      {
        'ChannelName': 'training',
        'DataSource': {
          'S3DataSource': {
            'S3DataType': 'S3Prefix',
            'S3Uri': os.environ['S3_TRIGGER_PATH'],
            'S3DataDistributionType': 'FullyReplicated'
          }
        },
        'CompressionType': 'None',
        'RecordWrapperType': 'None'
      }
    ],
    'OutputDataConfig': {
      'KmsKeyId': '',
      'S3OutputPath': os.environ['S3_OUTPUT_PATH']
    },
    'ResourceConfig': {
      'InstanceType': os.environ['AG_INSTANCE_TYPE'],
      'InstanceCount': 1,
      'VolumeSizeInGB': 30
    },
    'StoppingCondition': {
      'MaxRuntimeInSeconds': 86400
    },
    'EnableNetworkIsolation': False,
    'EnableInterContainerTrafficEncryption': False,
    'EnableManagedSpotTraining': False,
  }
  
  response = sm.create_training_job(**training_job_params)
  
  return {
    'statusCode': 200,
    'body': json.dumps(key)
  }<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13613996719518862, 'fraction_numerical': 0.02009294696555495, 'mean_word_length': 3.788612565445026, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '214035', 'license': 'MIT', 'n_tokens_mistral': 2634, 'n_tokens_neox': 2543, 'n_words': 380, 'path': 'brazil_ecommerce/working/3.9.Train_Boto3.ipynb', 'repo_name': 'gonsoomoon-ml/predict-delivery-time', 'size': '7316'}