<jupyter_start><jupyter_text>Classifier Train<jupyter_code>estimator = TensorFlow(
    entry_point='./src/classifier_train.py',
    role=role,
    instance_count=1,
    instance_type='ml.g4dn.xlarge',
    framework_version='1.13', # SageMaker Neo を使うために keras 2.2.4 に合わせる
    py_version='py3',
    hyperparameters={
        'epochs':30,
        'increment':'False'
    }
)
%%time

print(train_X_uri[:-11]) # dir以下全て
estimator.fit({
    'train': train_X_uri[:-11],
})
classifier_model_uri = estimator.latest_training_job.describe()['ModelArtifacts']['S3ModelArtifacts']
print(classifier_model_uri)<jupyter_output><empty_output><jupyter_text>Neo Compile<jupyter_code>import boto3
sm_client = boto3.client('sagemaker')
classifier_output_s3_location = f's3://{sagemaker.session.Session().default_bucket()}/sagemaker/mnist/classifier/compilied_model'
print(classifier_output_s3_location)
classifier_compile_jobname = f'classifier-{str(uuid4())}'
print(classifier_compile_jobname)
response = sm_client.create_compilation_job(
    CompilationJobName=classifier_compile_jobname,
    RoleArn=role,
    InputConfig={
        'S3Uri': classifier_model_uri,
        'DataInputConfig': '{"input_1":[1,1,28,28]}',
        'Framework': 'KERAS',
    },
    OutputConfig={
        'S3OutputLocation': classifier_output_s3_location,
        'TargetDevice': 'ml_c5',
    },
    StoppingCondition={
        'MaxRuntimeInSeconds': 900,
        'MaxWaitTimeInSeconds': 900
    },
)
while True:
    response = sm_client.describe_compilation_job(CompilationJobName=classifier_compile_jobname)
    status = response['CompilationJobStatus']
    if status in ['COMPLETED','FAILED','STOPPED'] : 
        print('!')
        print(status)
        classifier_neo_model_uri = response['ModelArtifacts']['S3ModelArtifacts']
        break
    else:
        print('.',end='')
    sleep(5)
print(classifier_neo_model_uri)<jupyter_output><empty_output><jupyter_text>ファイルの配置<jupyter_code>publisher_dir = './src/ggv2/components/artifacts/com.example.Publisher/1.0.0'
subscriber_dir = './src/ggv2/components/artifacts/com.example.Subscriber/1.0.0'
iotpublisher_classifier_dir = './src/ggv2/components/artifacts/com.example.IoTPublisher/1.0.1/classifier'

# test_x の配置
!mkdir -p {publisher_dir}
!mv {work_dir}/test_X.npy {publisher_dir}/test_X.npy

# # classifier1.h5 のダウンロード・解凍・gz削除
!aws s3 cp {classifier_model_uri} {work_dir}/
!mkdir -p {subscriber_dir}
!tar zxvf {work_dir}/model.tar.gz -C {subscriber_dir}
!rm {work_dir}/model.tar.gz 

# Neo コンパイル済モデルのダウンロード・解凍・gz削除

!aws s3 cp {classifier_neo_model_uri} {work_dir}/
!mkdir -p {iotpublisher_classifier_dir}
!tar zxvf {work_dir}/model-ml_c5.tar.gz -C {iotpublisher_classifier_dir}
!rm {work_dir}/model-ml_c5.tar.gz<jupyter_output><empty_output><jupyter_text>動作確認 keras model<jupyter_code>classifier = keras.models.load_model(f'{subscriber_dir}/classifier.h5')
classifier.summary()
pred_y = classifier.predict(test_X[0:1,:,:,:])
plt.imshow(test_X[0,:,:,0],'gray')
np.argmax(pred_y[0,:])<jupyter_output><empty_output><jupyter_text>Neo Model<jupyter_code>import dlr
import numpy as np

classifier_neo = dlr.DLRModel(iotpublisher_classifier_dir, 'cpu', 0)
pred_y = classifier_neo.run(test_X[0,:,:,:].reshape(1,1,28,28))[0]
np.argmax(pred_y)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14420731707317072, 'fraction_numerical': 0.022560975609756097, 'mean_word_length': 5.79296066252588, 'pattern_counts': {'":': 1, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '363657', 'license': 'MIT-0', 'n_tokens_mistral': 1340, 'n_tokens_neox': 1300, 'n_words': 145, 'path': 'inference/edge/greengrass-ml-inference/build_mnist_classifier.ipynb', 'repo_name': 'mshin20220124/aws-ml-jp', 'size': '3280'}