<jupyter_start><jupyter_text>Excercise: use autoencoder (LSTM) for anomaly detection in accelerometer based vibration dataset.  The autoencoder tries to reconstruct the input at the output. Hence, for healthy data, it tries to reconstruct healthy data. But it will have a hard time trying to reconstruct faulty data in its neural network bottleneck (LSTM). That's how the anomaly detector works.<jupyter_code>import numpy as np
import pandas as pd
import scipy.io as sio
import os<jupyter_output><empty_output><jupyter_text>Download healthy data<jupyter_code># original site seems to be permantly offline, using cached data

#!wget http://csegroups.case.edu/sites/default/files/bearingdatacenter/files/Datafiles/97.mat
#!wget http://csegroups.case.edu/sites/default/files/bearingdatacenter/files/Datafiles/98.mat
#!wget http://csegroups.case.edu/sites/default/files/bearingdatacenter/files/Datafiles/99.mat
#!wget http://csegroups.case.edu/sites/default/files/bearingdatacenter/files/Datafiles/100.mat
!wget https://github.com/IBM/skillsnetwork/raw/master/coursera_ai/week3/data/cwr_healthy/97.mat
!wget https://github.com/IBM/skillsnetwork/raw/master/coursera_ai/week3/data/cwr_healthy/98.mat
!wget https://github.com/IBM/skillsnetwork/raw/master/coursera_ai/week3/data/cwr_healthy/99.mat
!wget https://github.com/IBM/skillsnetwork/raw/master/coursera_ai/week3/data/cwr_healthy/100.mat

!mkdir cwr_healthy
!mv *.mat cwr_healthy/
!ls -lahr cwr_healthy/
mlf = sio.loadmat('./cwr_healthy/100.mat')
mlf<jupyter_output><empty_output><jupyter_text>Store data in dataframes<jupyter_code>def read_folder(folder):
    data = 'dummy'
    skip = False
    for file in os.listdir(folder):
        file_id = file[:-4]
        mat_file_dict = sio.loadmat(folder+file)
        del data
        for key, value in mat_file_dict.items():
            if 'DE_time' in key or 'FE_time' in key:
                a = np.array(mat_file_dict[key])
                try:
                    data
                except NameError:
                    data = a
                else:
                    if (data.shape[0] != a.shape[0]):
                        print('skipping ' + file_id)
                        skip = True
                        continue
                    data = np.hstack((data,a))
        if skip:
            skip = False
            continue
        id = np.repeat(file_id, data.shape[0])
        id.shape = (id.shape[0], 1)
        data = np.hstack((id, data))
        if data.shape[1] == 2:
            zeros = np.repeat(float(0), data.shape[0])
            zeros.shape = (data.shape[0], 1)
            data = np.hstack((data, zeros))
        try:
            result
        except NameError:
            result = data
        else:
            result = np.vstack((result, data))
    return result
result_healthy = read_folder('./cwr_healthy/')
print(result_healthy.shape)
result_healthy<jupyter_output>(1213484, 3)<jupyter_text>Store healthy data in dataframe<jupyter_code>pdf = pd.DataFrame(result_healthy)
pdf.to_csv('result_healthy_pandas.csv', header=False, index=True)
!head result_healthy_pandas.csv<jupyter_output>0,100,0.014603076923076923,0.19292181818181817
1,100,0.05444861538461539,0.16436363636363635
2,100,0.10764553846153846,0.09081090909090908
3,100,0.13372246153846154,0.08649636363636364
4,100,0.11265230769230769,0.09923454545454545
5,100,0.08240307692307691,0.09307090909090908
6,100,0.08699261538461538,0.08197636363636364
7,100,0.11056615384615384,0.03307818181818182
8,100,0.12767261538461538,0.01171090909090909
9,100,0.11348676923076922,0.02218909090909091<jupyter_text>Download faulty data<jupyter_code># uncommented because way too much data
#!for url in `curl -s csegroups.case.edu/bearingdatacenter/pages/12k-drive-end-bearing-fault-data |grep mat |grep http |awk -F'href="' '{print $2}' |awk -F'">' '{print $1}'`; do wget $url; done
#!for url in `curl -s csegroups.case.edu/bearingdatacenter/pages/48k-drive-end-bearing-fault-data |grep mat |grep http |awk -F'href="' '{print $2}' |awk -F'">' '{print $1}'`; do wget $url; done
#!for url in `curl -s csegroups.case.edu/bearingdatacenter/pages/12k-fan-end-bearing-fault-data |grep mat |grep http |awk -F'href="' '{print $2}' |awk -F'">' '{print $1}'`; do wget $url; done
#!mkdir cwr_faulty
#!mv *.mat cwr_faulty/
# original site seems to be permantly offline, using cached data
# !wget http://csegroups.case.edu/sites/default/files/bearingdatacenter/files/Datafiles/105.mat
# !wget http://csegroups.case.edu/sites/default/files/bearingdatacenter/files/Datafiles/106.mat
# !wget http://csegroups.case.edu/sites/default/files/bearingdatacenter/files/Datafiles/107.mat
# !wget http://csegroups.case.edu/sites/default/files/bearingdatacenter/files/Datafiles/108.mat
!wget https://github.com/IBM/skillsnetwork/raw/master/coursera_ai/week3/data/cwr_faulty/105.mat
!wget https://github.com/IBM/skillsnetwork/raw/master/coursera_ai/week3/data/cwr_faulty/106.mat
!wget https://github.com/IBM/skillsnetwork/raw/master/coursera_ai/week3/data/cwr_faulty/107.mat
!wget https://github.com/IBM/skillsnetwork/raw/master/coursera_ai/week3/data/cwr_faulty/108.mat
!mkdir cwr_faulty
!mv *.mat cwr_faulty/
!ls cwr_faulty/<jupyter_output>105.mat  106.mat  107.mat  108.mat<jupyter_text>Store faulty data in dataframe<jupyter_code>result_faulty = read_folder('./cwr_faulty/')
pdf = pd.DataFrame(result_faulty)
pdf.to_csv('result_faulty_pandas.csv', header=False, index=True)
!head result_faulty_pandas.csv<jupyter_output>0,106,-0.2776016367265469,0.040885454545454544
1,106,-0.04434479041916167,0.06985454545454545
2,106,0.11760303393213573,0.3377672727272727
3,106,-0.14505457085828344,0.2516818181818182
4,106,-0.111430499001996,0.1088909090909091
5,106,0.1309227145708583,0.07889454545454545
6,106,0.03281189620758483,0.02116181818181818
7,106,-0.1970338123752495,0.12861454545454545
8,106,-0.07488259481037925,0.20196181818181816
9,106,0.009583672654690619,-0.04684363636363636<jupyter_text>Store healthy and faulty dataframes in IBM cloud<jupyter_code># In order to obtain the correct values for "credentias", "bucket_name" and "endpoint" 
# please follow the tutorial at https://github.com/IBM/skillsnetwork/wiki/Cloud-Object-Storage-Setup

credentials = {
  "apikey": "<KEY>",
  "cos_hmac_keys": {
    "access_key_id": "<KEY>",
    "secret_access_key": "<KEY>"
  },
  "endpoints": "https://control.cloud-object-storage.cloud.ibm.com/v2/endpoints",
  "iam_apikey_description": "Auto-generated for key <KEY>",
  "iam_apikey_name": "Service credentials-1",
  "iam_role_crn": "crn:v1:bluemix:public:iam::::serviceRole:Writer",
  "iam_serviceid_crn": "crn:v1:bluemix:public:iam-identity::a/3dba62a148ab4574867f8eb140c3a44e::serviceid:ServiceId-109769b1-d4d5-4997-93a1-faefc036bfa9",
  "resource_instance_id": "crn:v1:bluemix:public:cloud-object-storage:global:a/3dba62a148ab4574867f8eb140c3a44e:643e3143-6265-453a-877a-15ae3947ef9a::"
}

bucket_name = "cloud-object-storage-appliedaideeplearning"
endpoint = "https://s3.eu-de.cloud-object-storage.appdomain.cloud"
import base64
from ibm_botocore.client import Config
import ibm_boto3
import time

# Create client 
client = ibm_boto3.client(
    's3',
    aws_access_key_id = credentials["cos_hmac_keys"]['access_key_id'],
    aws_secret_access_key = credentials["cos_hmac_keys"]["secret_access_key"],
    endpoint_url = endpoint
)

client.upload_file('result_healthy_pandas.csv', bucket_name, 'result_healthy_pandas.csv')
client.upload_file('result_faulty_pandas.csv', bucket_name, 'result_faulty_pandas.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12926249008723237, 'fraction_numerical': 0.13045202220459953, 'mean_word_length': 4.443884892086331, 'pattern_counts': {'":': 10, '<': 29, '<?xml version=': 0, '>': 32, 'https://': 11, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '368960', 'license': 'MIT', 'n_tokens_mistral': 3505, 'n_tokens_neox': 2885, 'n_words': 436, 'path': '03_Applied_AI_DeepLearning/notebooks/week_3_1_anomaly_detection_keras_1.ipynb', 'repo_name': 'hsotoparada/Advanced-Data-Science-Specialization', 'size': '7706'}