<jupyter_start><jupyter_text>Set up  1. Set  up  accounts and role<jupyter_code>#!pip install sagemaker==1.39.0
import sagemaker
import boto3

sagemaker_session = sagemaker.Session()
account_id =  boto3.client('sts').get_caller_identity().get('Account')
region = boto3.session.Session().region_name


#role = sagemaker.get_execution_role()
role="arn:aws:iam::{}:role/service-role/AmazonSageMaker-ExecutionRole-20190118T115449".format(account_id)
max_runs=1<jupyter_output><empty_output><jupyter_text>2. Setup image and instance type<jupyter_code>pytorch_custom_image_name="ppi-extractor:gpu-1.0.0-201910130520"
instance_type = "ml.p3.8xlarge" 
docker_repo = "{}.dkr.ecr.{}.amazonaws.com/{}".format(account_id, region, pytorch_custom_image_name)<jupyter_output><empty_output><jupyter_text>3. Configure train/ test and validation datasets<jupyter_code>bucket = "aegovan-data"
#trainfile = "s3://{}/aimed/AIMedtrain.json".format(bucket)
trainfile_plain = "s3://{}/aimed/AIMedFull.json".format(bucket)
trainfile="s3://{}/aimed/AIMedFull_preprocessed.json".format(bucket)

valfile="s3://{}/aimed/AIMedval_preprocessed.json".format(bucket)

pretrained_bert="s3://{}/embeddings/bert/".format(bucket)



train_split="s3://{}/aimed/AIMedtrain.json".format(bucket)
val_split="s3://{}/aimed/AIMedval.json".format(bucket)



s3_output_path= "s3://{}/results/".format(bucket)
s3_code_path= "s3://{}/aimed_bert_code".format(bucket)<jupyter_output><empty_output><jupyter_text>Start training<jupyter_code>commit_id = "21a539b999abfbad643c2727aa1<PASSWORD>"
pub_inputs = {
    "train" : trainfile,
    "PRETRAINED_BIOBERT" : pretrained_bert
}
plain_inputs = {
    "train" : trainfile_plain,
    "PRETRAINED_BIOBERT" : pretrained_bert
}
train_split_inputs = {
    "train" : train_split,
    "val" : val_split,
   # "val" :valfile,
    "PRETRAINED_BIOBERT" : pretrained_bert
}
BertNetworkFactoryhyperparameters = {
    "dataset":"PpiAimedDatasetPreprocessedFactory",
    "network" :"RelationExtractorBioBertFactory",
    "trainfile":trainfile.split("/")[-1],
    "batchsize": "8",
    "accumulation_steps" : "8",
    "epochs" : "1000",   
    "log-level" : "INFO",
    "learningrate":.00001,
    "earlystoppingpatience":20,
    "commit_id" : commit_id,
    "docidfieldname":"docid",
    "labelfieldname":"isValid"


}
BertNetworkFactoryhyperparameters_plain_split = {
    "dataset":"PpiAimedDatasetFactory",
    "network" :"RelationExtractorBioBertFactory",
    "trainfile":train_split.split("/")[-1],
    "valfile":val_split.split("/")[-1],
    "batchsize": "8",
    "accumulation_steps" : "8",
    "epochs" : "1000",   
    "log-level" : "INFO",
    "learningrate":.00001,
    "earlystoppingpatience":20,
    "commit_id" : commit_id,
    "docidfieldname":"docid",
    "labelfieldname":"isValid"


}
BertNetworkFactoryhyperparameters_overlap = {
    "dataset":"PpiAimedDatasetPreprocessedFactory",
    "network" :"RelationExtractorBioBertFactory",
    "trainfile":trainfile.split("/")[-1],
    "batchsize": "8",
    "accumulation_steps" : "8",
    "epochs" : "1000",   
    "log-level" : "INFO",
    "learningrate":.00001,
    "earlystoppingpatience":20,
    "commit_id" : commit_id,
    "labelfieldname":"isValid"


}
BertNetworkFactoryhyperparameters_plain = {
    "dataset":"PpiAimedDatasetFactory",
    "network" :"RelationExtractorBioBertFactory",
    "trainfile":trainfile_plain.split("/")[-1],
    
    "batchsize": "8",
    "accumulation_steps" : "8",
    "epochs" : "1000",   
    "log-level" : "INFO",
    "learningrate":.00001,
    "earlystoppingpatience":20,
    "commit_id" : commit_id,
    "docidfieldname":"docid",
    "labelfieldname":"isValid"
}
BertNetworkFactoryhyperparameters_plain_overlap = {
    "dataset":"PpiAimedDatasetFactory",
    "network" :"RelationExtractorBioBertFactory",
    "trainfile":trainfile_plain.split("/")[-1],
    "batchsize": "8",
    "accumulation_steps" : "8",
    "epochs" : "1000",   
    "log-level" : "INFO",
    "learningrate":.00001,
    "earlystoppingpatience":20,
    "commit_id" : commit_id,
    "labelfieldname":"isValid"
}
LstmBertNetworkFactoryhyperparameters_plain = {
    "dataset":"PpiAimedDatasetFactory",
    "network" :"RelationExtractorBertBiLstmNetworkNoPosFactory",
    "trainfile":trainfile_plain.split("/")[-1],
    "batchsize": "64",
    "accumulation_steps" : "1",
    "epochs" : "1000",   
    "log-level" : "INFO",
    "earlystoppingpatience":20,
    "commit_id" : commit_id,
    "learningrate":.001,
    "lstm_dropout":0.5,
    "lstm_num_layers" :1,
    "lstm_hidden_size":200,
    "fc_drop_out_rate":0.5,
    "input_dropout" :0.1,
    "docidfieldname":"docid",
    "labelfieldname":"isValid"
   
}
metric_definitions = [{"Name": "TrainLoss",
                     "Regex": "###score: train_loss### (\d*[.]?\d*)"}
                    ,{"Name": "ValidationLoss",
                     "Regex": "###score: val_loss### (\d*[.]?\d*)"}
                    ,{"Name": "TrainFScore",
                     "Regex": "###score: train_fscore### (\d*[.]?\d*)"}
                   ,{"Name": "ValidationFScore",
                     "Regex": "###score: val_fscore### (\d*[.]?\d*)"}
                    ]
!git log -1 | head -1
!git log -1 | head -5 | tail -1
hyperparameters = BertNetworkFactoryhyperparameters_plain_split # BertNetworkFactoryhyperparameters #BertNetworkFactoryhyperparameters_plain_overlap # BertNetworkFactoryhyperparameters_plain #  # LstmBertNetworkFactoryhyperparameters_plain 
inputs = train_split_inputs # pub_inputs #plain_inputs 
base_name =  "aimed-bert-plain-split"
BertNetworkFactoryhyperparameters_plain_split
git_config = {'repo': 'https://github.com/elangovana/PPI-typed-relation-extractor.git',
              'branch': 'master',
              'commit': hyperparameters["commit_id"]
             }
hyperparameters
inputs
from sagemaker.pytorch import PyTorch

estimator = PyTorch(
     entry_point='main_train_bert.py',
   # entry_point='main_train_bert_k_fold.py',
                    source_dir = 'source/algorithms',
                    dependencies =['source/algorithms', 'source/datasets', 'source/preprocessor', 'source/modelnetworks','source/trainpipelinesbuilders', 'source/metrics'],
                    role=role,
                    framework_version ="1.0.0",
                    py_version='py3',
                    git_config= git_config,
                    image_name= docker_repo,
                    train_instance_count=1,
                    train_instance_type=instance_type,
                    hyperparameters =hyperparameters,
                    output_path=s3_output_path,
                    metric_definitions=metric_definitions,
                    #train_use_spot_instances = True
                    train_volume_size=30,
                    code_location=s3_code_path,
                    train_max_run = 60 * 60 * 24 * 4,
                    base_job_name =base_name)

for i in range( 0, 6):
    estimator.fit(inputs, wait=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1523768490593135, 'fraction_numerical': 0.026712623869021974, 'mean_word_length': 3.6241699867197874, 'pattern_counts': {'":': 52, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '108579', 'license': 'MIT', 'n_tokens_mistral': 2530, 'n_tokens_neox': 2389, 'n_words': 304, 'path': 'Sagemaker-AIMed-Bert.ipynb', 'repo_name': 'elangovana/kegg-pathway-extractor', 'size': '6966'}