<jupyter_start><jupyter_text>Preparation stepsInstall iotfunctions with`pip install git+https://github.com/ibm-watson-iot/functions@development`This projects contains the code for the Analytics Service pipeline as well as the anomaly functions and should pull in most of this notebook's dependencies.The plotting library matplotlib is the exception, so you need to run`pip install matplotlib`<jupyter_code># Real life data

import logging
import threading
import itertools
import pandas as pd  
import numpy as np  
import json
import matplotlib.pyplot as plt
from matplotlib import cm
from mpl_toolkits.mplot3d import axes3d
import seaborn as seabornInstance
from sqlalchemy import Column, Integer, String, Float, DateTime, Boolean, func
from iotfunctions import base
from iotfunctions import bif
from iotfunctions import entity
from iotfunctions import metadata
from iotfunctions.metadata import EntityType
from iotfunctions.db import Database
from iotfunctions.dbtables import FileModelStore
from iotfunctions.enginelog import EngineLogging
from iotfunctions import estimator
from iotfunctions.bif import PythonExpression
from iotfunctions.ui import (UISingle, UIMultiItem, UIFunctionOutSingle,
                 UISingleItem, UIFunctionOutMulti, UIMulti, UIExpression,
                 UIText, UIStatusFlag, UIParameters)
from iotfunctions.anomaly import (SaliencybasedGeneralizedAnomalyScore, SpectralAnomalyScore,
                 FFTbasedGeneralizedAnomalyScore, KMeansAnomalyScore,
                 SaliencybasedGeneralizedAnomalyScoreV2, FFTbasedGeneralizedAnomalyScoreV2, 
                 KMeansAnomalyScoreV2, BayesRidgeRegressor)
from mmfunctions.anomaly import (FeatureBuilder, GBMForecaster)
from poc.functions import State_Timer
import datetime as dt
from sklearn.model_selection import train_test_split 
from sklearn.linear_model import LinearRegression
from sklearn import metrics
from pandas.plotting import register_matplotlib_converters
register_matplotlib_converters()

EngineLogging.configure_console_logging(logging.INFO)
# set up a db object with a FileModelStore to support scaling
with open('credentials_as_monitor_demo.json', encoding='utf-8') as F:
    credentials = json.loads(F.read())
db_schema=None
fm = FileModelStore()
db = Database(credentials=credentials, model_store=fm)
print (db)
# Run on the good pump first 
# Get stuff in
df_i = pd.read_csv('./Weather.csv', 
                   dtype={"Snowfall":object, "PoorWeather":object, "SNF": object, "TSHDSBRSGF":object},
                   index_col=False, parse_dates=['Date'])

df_i.rename(columns={'Date': 'timestamp'}, inplace=True)

# drastic filtering
df_i = df_i[df_i['STA'] == 10001]

df_i['entity'] = df_i['STA'].astype(str)



# and sort it by timestamp
df_i = df_i[['entity','timestamp','Precip','MaxTemp','MinTemp','MeanTemp']].sort_values(by=['entity','timestamp'])


df_i = df_i.set_index(['entity','timestamp']).dropna()


df_i.head(8)

# Now run the anomaly functions as if they were executed in a pipeline
EngineLogging.configure_console_logging(logging.DEBUG)
jobsettings = { 'db': db, 
               '_db_schema': 'public', 'save_trace_to_file' : True}

spsi = GBMForecaster(['MaxTemp'], ['MaxTemp'], ['MaxTemp_pred'],[24,25,31])
et = spsi._build_entity_type(columns = [Column('MinTemp',Float())], **jobsettings)
spsi._entity_type = et
df_i = spsi.execute(df=df_i)

EngineLogging.configure_console_logging(logging.INFO)
df_i.describe()

#
df_disp = df_i.tail(100)
plots = 1
fig, ax = plt.subplots(plots, 1, figsize=(20,5), squeeze=False)
cnt = 0
ax[cnt,0].plot(df_disp.unstack(level=0).index, df_disp['MaxTemp'],linewidth=0.7,color='green',label='MaxTemp')
ax[cnt,0].plot(df_disp.unstack(level=0).index + pd.DateOffset(hours=24),
               df_disp['MaxTemp_pred'],linewidth=0.7,color='red',label='MaxTemp')

ax[cnt,0].legend(bbox_to_anchor=(1.1, 1.05))
ax[cnt,0].set_ylabel('WWII Weather Data\n(predict max temp from min temp)',fontsize=12,weight="bold")

#df_i[['MaxTemp','__forecast__']].tail(100).plot(figsize=(10,3))
#df_i[['MaxTemp_pred']].tail(48).plot(figsize=(10,3))
# load data
df_input = pd.read_csv('BP_DATA/IoTData.csv', parse_dates=['DATETIME'], comment='#')
df_input.rename(columns={'DATETIME': 'timestamp'}, inplace=True)
df_input['entity'] = 'demand'
#df_input = df_input.asfreq('H')
df_input.set_index('timestamp', inplace=True)

# fix blue Monday - Monday 9 and 10 am
df_input = df_input[(df_input.index.weekday != 0) | (df_input.index.hour > 10) & (df_input.index.hour < 9)].resample('1H').\
            interpolate().reset_index().set_index(['entity','timestamp'])

df_input[['KW']].head(900).plot(figsize=(10,5))
# Now run the anomaly functions as if they were executed in a pipeline
EngineLogging.configure_console_logging(logging.DEBUG)
jobsettings = { 'db': db, 
               '_db_schema': 'public', 'save_trace_to_file' : True}

spsi = GBMForecaster(['KW', 'TEMP_AIR'], ['KW'], ['KW_forecast'],[24,48, 168])
et = spsi._build_entity_type(columns = [Column('MinTemp',Float())], **jobsettings)
spsi._entity_type = et
df_input = spsi.execute(df=df_input)

EngineLogging.configure_console_logging(logging.INFO)
df_input.describe()
#df_input[['KW','KW_forecast']].tail(26)
#
df_disp = df_input.tail(200)
df_fore = df_input.tail(200)
plots = 1
fig, ax = plt.subplots(plots, 1, figsize=(20,7), squeeze=False)
cnt = 0
ax[cnt,0].plot(df_disp.unstack(level=0).index, df_disp['KW'],linewidth=0.7,color='green',label='KW')
#ax[cnt,0].plot(df_disp.unstack(level=0).index, df_disp['KW_forecast'],linewidth=0.7,color='blue',label='forecast')
ax[cnt,0].plot(df_disp.unstack(level=0).index + pd.DateOffset(hours=24), df_disp['KW_forecast'],linewidth=0.7,color='red',label='forecast')

ax[cnt,0].legend(bbox_to_anchor=(1.1, 1.05))
#ax[cnt,0].set_ylabel('WWII Weather Data\n(predict max temp from min temp)',fontsize=12,weight="bold")

# load data
df_inp = pd.read_csv('AllOfArmstark.csv', parse_dates=['timestamp'], comment='#')
#df_inp.rename(columns={'DATETIME': 'timestamp'}, inplace=True)
#df_inp.set_index(['entity','timestamp'], inplace=True)

df_inp.head(3)
#
df_i1 = df_inp[df_inp['entity']=='04714B6046D5']
df_i2 = df_inp[df_inp['entity']=='04714B604101']
df_vx1 = df_i1[['Vx']]
df_sp1 = df_i1[['As']]
df_vx2 = df_i2[['Vx']]
df_sp2 = df_i2[['Ap']]

df_sp2

plots = 2
fig, ax = plt.subplots(plots, 2, figsize=(20,7), squeeze=False)
cnt = 0
ax[0,0].plot(df_vx1,linewidth=0.7,color='green',label='Vibration 1')
ax[0,1].plot(df_sp1,linewidth=0.7,color='red',label='Speed 1')
#ax[cnt,0].plot(df_disp.unstack(level=0).index, df_disp['KW_forecast'],linewidth=0.7,color='blue',label='forecast')
cnt = 1
ax[1,0].plot(df_vx2, linewidth=0.7,color='green',label='Vibration 2')
ax[1,1].plot(df_sp2,linewidth=0.7,color='red',label='Speed 2')

#ax[cnt,0].legend(bbox_to_anchor=(1.1, 1.05))
#ax[cnt,0].set_ylabel('WWII Weather Data\n(predict max temp from min temp)',fontsize=12,weight="bold")

df_i1.set_index(['entity','timestamp'], inplace=True)
# Now run the anomaly functions as if they were executed in a pipeline
EngineLogging.configure_console_logging(logging.DEBUG)
jobsettings = { 'db': db, 
               '_db_schema': 'public', 'save_trace_to_file' : True}

spsi = GBMForecaster(['Vx', 'As'], ['Vx'], ['Vx_forecast'],[100,200,300])
et = spsi._build_entity_type(columns = [Column('MinTemp',Float())], **jobsettings)
spsi._entity_type = et
df_i1  = spsi.execute(df=df_i1)

EngineLogging.configure_console_logging(logging.INFO)
df_i1.head(3)
df_in2 = df_i2.copy()
df_in2['entity'] = '04714B6046D5'
df_in2.set_index(['entity','timestamp'], inplace=True)
# Now run the anomaly functions as if they were executed in a pipeline
EngineLogging.configure_console_logging(logging.DEBUG)
jobsettings = {'db': db, 
               '_db_schema': 'public', 'save_trace_to_file' : True}

spsi = GBMForecaster(['Vx', 'As'], ['Vx'], ['Vx_forecast'],[100,200,300])
et = spsi._build_entity_type(columns = [Column('MinTemp',Float())], **jobsettings)
spsi._entity_type = et
df_in2  = spsi.execute(df=df_in2)

EngineLogging.configure_console_logging(logging.INFO)
df_in2.head(3)
df_i1.reset_index().set_index('timestamp')[['Vx']]
#
#df_inp.reset_index(inplace=True)

#df_ii1 = df_inp[df_inp['entity']=='04714B6046D5']
#df_ii2 = df_inp[df_inp['entity']=='04714B604101']

df_i1['error'] = np.abs(df_i1['Vx'] - df_i1['Vx_forecast'])
df_vxi1 = df_i1.reset_index().set_index('timestamp')[['Vx']]
df_spi1 = df_i1.reset_index().set_index('timestamp')[['Vx_forecast']]
df_er1 = df_i1.reset_index().set_index('timestamp')[['error']]

df_in2['error'] = np.abs(df_in2['Vx'] - df_in2['Vx_forecast'])
df_vxi2 = df_in2.reset_index().set_index('timestamp')[['Vx']]
df_spi2 = df_in2.reset_index().set_index('timestamp')[['Vx_forecast']]
df_er2 = df_in2.reset_index().set_index('timestamp')[['error']]




plots = 2
fig, ax = plt.subplots(plots, 2, figsize=(20,7), squeeze=False)
cnt = 0
ax[0,0].plot(df_vxi1,linewidth=0.7,color='green',label='Vibration 1')
ax[0,0].plot(df_spi1,linewidth=0.7,color='red',zorder=10, label='Vibration forecast 1')
ax[0,1].plot(df_er1, linewidth=0.7,color='blue', label='Error 1')
#ax[cnt,0].plot(df_disp.unstack(level=0).index, df_disp['KW_forecast'],linewidth=0.7,color='blue',label='forecast')
cnt = 1
ax[1,0].plot(df_vxi2, linewidth=0.7,color='green',label='Vibration 2')
ax[1,0].plot(df_spi2,linewidth=0.7,color='red',zorder=10, label='Vibration forecast 2')
ax[1,1].plot(df_er2, linewidth=0.7,color='blue', label='Error 2')

df_spi1<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1631684421474528, 'fraction_numerical': 0.03449003269697289, 'mean_word_length': 7.042408821034775, 'pattern_counts': {'":': 4, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '659821', 'license': 'MIT', 'n_tokens_mistral': 3693, 'n_tokens_neox': 3496, 'n_words': 610, 'path': 'MonitorGBMForecaster.ipynb', 'repo_name': 'ankit-jha/mmfunctions', 'size': '9481'}