<jupyter_start><jupyter_text>Step 0 and 1Import Data (only FD001) Download data from S3<jupyter_code>import boto3
import sagemaker

#role = sagemaker.get_execution_role()
role = '<your role>'
region = boto3.Session().region_name

print(region)
print(role)

# Test bucket name immersionday-sagemaker-test
bucket_name = 'iiot-book-data'
file_name_train = 'train_FD001.txt'
file_name_test = 'test_FD001.txt'

import boto3

s3 = boto3.resource('s3')
s3.Bucket(bucket_name).download_file(file_name_train, file_name_train)
s3.Bucket(bucket_name).download_file(file_name_test,file_name_test)<jupyter_output><empty_output><jupyter_text>Import Data (only FD001)<jupyter_code># step 1: read the dataset
columns = ['unitid', 'time', 'set_1','set_2','set_3']
columns.extend(['sensor_' + str(i) for i in range(1,22)])
df = pd.read_csv(file_name_train, delim_whitespace=True,names=columns)
df_test = pd.read_csv(file_name_test, delim_whitespace=True,names=columns)<jupyter_output><empty_output><jupyter_text>Prepare the data setRemove unusefull columns.<jupyter_code># only engine 1
i=1

# prepare model
columns_feature=['sensor_4','sensor_7']

file_name_train = 'train.csv'
file_name_test = 'test.csv'

dataset_train=df[(df.unitid ==i)]
dataset_test=df_test[(df_test.unitid ==i)]<jupyter_output><empty_output><jupyter_text>Save on local test<jupyter_code>print(dataset_train)
np.savetxt(os.path.join('./container/local_test/test_dir/input/data/training',file_name_train), dataset_train, delimiter=",",fmt='%s')<jupyter_output>unitid  time   set_1   set_2  set_3  sensor_1  sensor_2  sensor_3  \
0         1     1 -0.0007 -0.0004  100.0    518.67    641.82   1589.70   
1         1     2  0.0019 -0.0003  100.0    518.67    642.15   1591.82   
2         1     3 -0.0043  0.0003  100.0    518.67    642.35   1587.99   
3         1     4  0.0007  0.0000  100.0    518.67    642.35   1582.79   
4         1     5 -0.0019 -0.0002  100.0    518.67    642.37   1582.85   
5         1     6 -0.0043 -0.0001  100.0    518.67    642.10   1584.47   
6         1     7  0.0010  0.0001  100.0    518.67    642.48   1592.32   
7         1     8 -0.0034  0.0003  100.0    518.67    642.56   1582.96   
8         1     9  0.0008  0.0001  100.0    518.67    642.12   1590.98   
9         1    10 -0.0033  0.0001  100.0    518.67    641.71   1591.24   
10        1    11  0.0018 -0.0003  100.0    518.67    642.28   1581.75   
11        1    12  0.0016  0.0002  100.0    518.67    642.06   1583.41   
12        1    13 -0.0019  0.0004  100[...]<jupyter_text>--- Build and deploy Docker FileNow we can build our docker file, test locally and deploy on the cloud<jupyter_code>## Local Test
import os
import subprocess

def run_interactive(command, working_directory='.'):
    p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=working_directory)
    for line in p.stdout.readlines():
        print (line)
    retval = p.wait()
    
# build docker file    
run_interactive('docker build -t rul-estimator .','container')<jupyter_output>b'Sending build context to Docker daemon  205.3kB\r\r\n'
b'Step 1/18 : FROM tensorflow/tensorflow:latest-py3\n'
b' ---> 7f147470ab6f\n'
b'Step 2/18 : MAINTAINER IIOT-BOOK <<EMAIL>>\n'
b' ---> Using cache\n'
b' ---> 1763dc09f8a1\n'
b'Step 3/18 : RUN apt-get -y update && apt-get install -y --no-install-recommends          wget          python          nginx          ca-certificates     && rm -rf /var/lib/apt/lists/*\n'
b' ---> Using cache\n'
b' ---> 6519c33ad5e7\n'
b'Step 4/18 : RUN pip install --upgrade pip\n'
b' ---> Using cache\n'
b' ---> 3667f16d1f7a\n'
b'Step 5/18 : RUN pip install numpy\n'
b' ---> Using cache\n'
b' ---> 00fcb8dae7f8\n'
b'Step 6/18 : RUN pip install scipy\n'
b' ---> Using cache\n'
b' ---> d5d3aa627bc2\n'
b'Step 7/18 : RUN pip install scikit-learn\n'
b' ---> Using cache\n'
b' ---> 4cdab33f1dae\n'
b'Step 8/18 : RUN pip install pandas\n'
b' ---> Using cache\n'
b' ---> 293467c93fd3\n'
b'Step 9/18 : RUN pip install flask\n'
b' ---> Using cache\n'
b' --->[...]<jupyter_text>Train the modelTrain model 'container/local_test/train_local.sh rul-estimator'  Sart locallyFrom command line:Start 'container/local_test/serve_local.sh rul-estimator'  Test local model<jupyter_code>run_interactive('./predict_local.sh ../../test.csv', 'container/local_test')<jupyter_output>b'  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n'
b'                                 Dload  Upload   Total   Spent    Left  Speed\n'
b'\r  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0*   Trying ::1...\n'
b'* TCP_NODELAY set\n'
b'* Connected to localhost (::1) port 8080 (#0)\n'
b'> POST /invocations HTTP/1.1\r\n'
b'> Host: localhost:8080\r\n'
b'> User-Agent: curl/7.55.1\r\n'
b'> Accept: */*\r\n'
b'> Content-Type: text/csv\r\n'
b'> Content-Length: 20173\r\n'
b'> Expect: 100-continue\r\n'
b'> \r\n'
b'< HTTP/1.1 100 Continue\r\n'
b'} [16384 bytes data]\n'
b'* We are completely uploaded and fine\n'
b'< HTTP/1.1 200 OK\r\n'
b'< Server: nginx/1.10.3 (Ubuntu)\r\n'
b'< Date: Wed, 31 Oct 2018 14:50:40 GMT\r\n'
b'< Content-Type: text/csv; charset=utf-8\r\n'
b'< Content-Length: 10\r\n'
b'< Connection: keep-alive\r\n'
b'< \r\n'
b'{ [10 bytes data]\n'
b'\r100 20183  100    10  100 20173     10  20173  0:00:01 --:--:--  0:00:01 36696\[...]<jupyter_text>--------- Publish image on AWS ECS<jupyter_code># publish image on AWS ECS file    
run_interactive('./build_and_push.sh rul-estimator','container/')
print('now you can eccass to https://console.aws.amazon.com/ecs/ ')<jupyter_output>now you can eccass to https://console.aws.amazon.com/ecs/
{'fraction_non_alphanumeric': 0.15244334277620397, 'fraction_numerical': 0.11685552407932011, 'mean_word_length': 2.806603773584906, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 53, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '42432', 'license': 'MIT', 'n_tokens_mistral': 2655, 'n_tokens_neox': 2261, 'n_words': 533, 'path': 'Chapter15/aws_sagemaker/sagemaker_local_test.ipynb', 'repo_name': 'NAzT/Hands-On-Industrial-Internet-of-Things', 'size': '5665'}