<jupyter_start><jupyter_text>Setup Path<jupyter_code># location for train.csv, val.csv and labels.csv
DATA_PATH = Path("../sample_data/multi_label_toxic_comments/")   

# Location for storing training_config.json
CONFIG_PATH = DATA_PATH/'config'
CONFIG_PATH.mkdir(exist_ok=True)

# S3 bucket name
bucket = sagemaker_session.default_bucket()

# Prefix for S3 bucket for input and output
prefix = 'toxic_comments/input'
prefix_output = 'toxic_comments/output'<jupyter_output><empty_output><jupyter_text>Upload Data<jupyter_code># This is a helper feature to upload data
# from your local machine to S3 bucket.

s3_input = sagemaker_session.upload_data(DATA_PATH, bucket=bucket , key_prefix=prefix)
sagemaker_session.upload_data(str(DATA_PATH/'label/labels.csv'), bucket=bucket , key_prefix=prefix)
sagemaker_session.upload_data(str(DATA_PATH/'data/train_sample.csv'), bucket=bucket , key_prefix=prefix)
sagemaker_session.upload_data(str(DATA_PATH/'data/val_sample.csv'), bucket=bucket , key_prefix=prefix)<jupyter_output><empty_output><jupyter_text>Hyperparameters & Training Config<jupyter_code>hyperparameters = {
    "epochs": 10,
    "lr": 8e-5,
    "max_seq_length": 512,
    "train_batch_size": 16,
    "lr_schedule": "warmup_cosine",
    "warmup_steps": 1000,
    "optimizer_type": "adamw"
}
training_config = {
    "run_text": "toxic comments",
    "finetuned_model": None,
    "do_lower_case": "True",
    "train_file": "train_sample.csv",
    "val_file": "val_sample.csv",
    "label_file": "labels.csv",
    "text_col": "comment_text",
    "label_col": '["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]',
    "multi_label": "True",
    "grad_accumulation_steps": "1",
    "fp16_opt_level": "O1",
    "fp16": "True",
    "model_type": "roberta",
    "model_name": "roberta-base",
    "logging_steps": "300"
}

with open(CONFIG_PATH/'training_config.json', 'w') as f:
    json.dump(training_config, f)<jupyter_output><empty_output><jupyter_text>Create an Estimator and start training<jupyter_code>account = sagemaker_session.boto_session.client('sts').get_caller_identity()['Account']
region = sagemaker_session.boto_session.region_name

image = "{}.dkr.ecr.{}.amazonaws.com/fluent-sagemaker-fast-bert:1.0-gpu-py36".format(account, region)
print(image)
output_path = "s3://{}/{}".format(bucket, prefix_output)
estimator = sagemaker.estimator.Estimator(image, 
                                          role,
                                          train_instance_count=1, 
                                          train_instance_type='ml.p3.8xlarge', 
                                          output_path=output_path,
                                          base_job_name='toxic-comments',
                                          enable_sagemaker_metrics=True,
                                          hyperparameters=hyperparameters,
                                          sagemaker_session=sagemaker_session
                                         )
# This is throwing the following error with both py3.7 (original) and py3.6 (i changed the docker image to use this)
estimator.fit(s3_input)<jupyter_output>2020-01-31 18:49:05 Starting - Starting the training job...
2020-01-31 18:49:07 Starting - Launching requested ML instances......
2020-01-31 18:50:13 Starting - Preparing the instances for training......
2020-01-31 18:51:27 Downloading - Downloading input data
2020-01-31 18:51:27 Training - Downloading the training image...............
2020-01-31 18:54:03 Uploading - Uploading generated training model
2020-01-31 18:54:03 Failed - Training job failed
[34mStarting the training.[0m
[34m/opt/ml/input/data/training/config/training_config.json[0m
[34m{'run_text': 'toxic comments', 'finetuned_model': None, 'do_lower_case': 'True', 'train_file': 'train_sample.csv', 'val_file': 'val_sample.csv', 'label_file': 'labels.csv', 'text_col': 'comment_text', 'label_col': '["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]', 'multi_label': 'True', 'grad_accumulation_steps': '1', 'fp16_opt_level': 'O1', 'fp16': 'True', 'model_type': 'roberta', 'model_name': 'roberta-base', 'lo[...]<jupyter_text>Deploy the model to hosting service<jupyter_code>predictor = estimator.deploy(1, 
                             'ml.m5.large', 
                             endpoint_name='bert-toxic-comments', 
                             update_endpoint=True, 
                             serializer=json_serializer)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13432504440497336, 'fraction_numerical': 0.03396980461811723, 'mean_word_length': 3.331730769230769, 'pattern_counts': {'":': 22, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '248451', 'license': 'Apache-2.0', 'n_tokens_mistral': 1549, 'n_tokens_neox': 1408, 'n_words': 265, 'path': '07_train/archive/extras/bert/fast-bert/sample_notebooks/toxic_comments_sagemaker.ipynb', 'repo_name': 'MarcusFra/workshop', 'size': '4504'}