<jupyter_start><jupyter_text>Project ECO AI Beta Testing Model Copyright 2021 <NAME>icensed under the Apache License, Version 2.0 (the "License");you may not use this file except in compliance with the License.You may obtain a copy of the License at    http://www.apache.org/licenses/LICENSE-2.0Unless required by applicable law or agreed to in writing, softwaredistributed under the License is distributed on an "AS IS" BASIS,WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.See the License for the specific language governing permissions andlimitations under the License.<jupyter_code>!pip install tabulate
!curl -o label_classes.py https://raw.githubusercontent.com/Harrow-Enigma/TeamEngima-ProjectEco-AI/main/label_classes.py
import numpy as np
import pandas as pd
import tensorflow as tf
import matplotlib as mpl
import matplotlib.pyplot as plt
from label_classes import Label, FloatLabel, IntClass, IntClassMap

from datetime import timedelta, timezone
from datetime import datetime as dt
from tqdm import tqdm
import pickle as pkl
import requests
import json
import os
mpl.rcParams['figure.figsize'] = (8, 6)
mpl.rcParams['axes.grid'] = False
mpl.rcParams['axes.titlecolor'] = 'green'
mpl.rcParams['axes.labelcolor'] = 'black'
mpl.rcParams['xtick.color'] = 'black'
mpl.rcParams['ytick.color'] = 'black'
np.random.seed(219)
tf.random.set_seed(219)<jupyter_output><empty_output><jupyter_text>Loading the data<jupyter_code>beta_api_url = 'https://dev-test.projecteco.ml/api/v1/rest/output/forms/'
response = requests.get(beta_api_url)
json_data = response.json()
json_data = json_data
for k in range(len(json_data)):
  try:
    _ = json_data[k]['localpollutiondata']
    _has_key = True
  except:
    print('Error: No `localpollutiondata` key')
    _has_key = False
  
  if _has_key:
    for pol_key in list(json_data[k]['localpollutiondata']\
                        ['data'].keys()):
      json_data[k][pol_key] = json_data[k]['localpollutiondata']\
                                       ['data'][pol_key]['v']
    json_data[k].pop('localpollutiondata')
print(json.dumps(json_data, indent=4, sort_keys=True))<jupyter_output><empty_output><jupyter_text>Defining the labels<jupyter_code>FEATURE_CLASSES = [
  FloatLabel('h'), FloatLabel('no2'), FloatLabel('o3'),
  FloatLabel('p'), FloatLabel('pm10'), FloatLabel('pm25'),
  FloatLabel('t'), FloatLabel('w'), FloatLabel('wg'),
]
LABEL_CLASSES = [
  IntClass('Q1 (General feeling)', _key='q1'),
  IntClass('Q2 (Concentration)', _key='q2'),
  IntClass('Q3 (Work Stress)', _key='q3'),
  IntClassMap('Q4 (Dizziness, headaches, shortness of breath)',
              {'no': 0, 'yes': 1},
              _key='q4'),
  IntClassMap('Q5 (Allergic responses)',
              {'no': 0, 'yes': 1},
              _key='q5')
]<jupyter_output><empty_output><jupyter_text>Data Pre-Processing<jupyter_code>def handle_exception(feat_class, timestep_data):
  try:
    _val = timestep_data[feat_class.key]
    if _val is not None:
      return feat_class.fwd_call(_val)
    return feat_class.fallback
  except:
    return feat_class.fallback
features = []
labels = []

for t in json_data:
  features.append([handle_exception(c, t)
                    for c in FEATURE_CLASSES])
  labels.append([handle_exception(c, t)
                    for c in LABEL_CLASSES])

features = np.array(features, np.float32)
labels = np.array(labels, np.float32)
def standardize(arr):
  m = arr.mean(0)
  s = arr.std(0)
    
  for i in range(len(s)):
    if s[i] == 0:
      s[i] = 1e-8
  
  arr = (arr - m)/s
  return arr, m, s

def destandardize(arr, m, s):
  return arr * s + m

def rescale(arr, delta=0.01):
  arr_max = arr.max(axis=0) + delta
  arr_min = arr.min(axis=0) - delta
  arr_range = arr_max - arr_min
  arr_ofst = (arr_max + arr_min) / 2
  return (arr - arr_ofst) / arr_range, arr_ofst, arr_range

def descale(arr, arr_ofst, arr_range):
  return arr * arr_range + arr_ofst

def normalize(arr):
  arr_std, m, s = standardize(arr)
  arr_norm, o, r = rescale(arr_std)
  return arr_norm, m, s, o, r

def denormalize(arr, m, s, o, r):
  return destandardize(descale(arr, o, r), m, s)
(features_norm,
 features_mean,
 features_stddv,
 features_ofst,
 features_range) = normalize(features)

(labels_norm,
 labels_mean,
 labels_stddv,
 labels_ofst,
 labels_range) = normalize(labels)
'''
assert features.all() == denormalize(features_norm, features_mean,
                                     features_stddv, features_ofst, features_range).all()
assert labels.all() == denormalize(labels_norm, labels_mean,
                                   labels_stddv, labels_ofst, labels_range).all()
'''
dataset = tf.data.Dataset.from_tensor_slices((features, labels))
dataset_size = len(features_norm)
ratio = 9/10
train_no = int(ratio * dataset_size)
test_no = dataset_size - train_no
print('Number of training samples: {}'.format(train_no))
print('Number of testing samples: {}'.format(test_no))
train_features_norm = features_norm[:train_no]
test_features_norm = features_norm[train_no:]
train_labels_norm = labels_norm[:train_no]
test_labels_norm = labels_norm[train_no:]
train_dataset = tf.data.Dataset.from_tensor_slices(
    (train_features_norm, train_labels_norm)
).batch(2, drop_remainder=False)
test_dataset = tf.data.Dataset.from_tensor_slices(
    (test_features_norm, test_labels_norm)
).batch(2, drop_remainder=False)
pkl.dump({
    'features': {
        'classes': FEATURE_CLASSES,
        'mean':    features_mean,
        'std_dev': features_stddv,
        'offset':  features_ofst,
        'range':    features_range
    },
    'labels': {
        'classes': LABEL_CLASSES,
        'mean':    labels_mean,
        'std_dev': labels_stddv,
        'offset':  labels_ofst,
        'range':    labels_range
    },
}, open('beta_data_aux.data', 'wb'))<jupyter_output><empty_output><jupyter_text>Building the model<jupyter_code>class DNNModel(tf.keras.Model):
  def __init__(self,
               inp_shape: int,
               out_shape: int,
               units=[32, 64, 64],
               name='DNNModel'):
    super(DNNModel, self).__init__(name = name)

    self.inp_shape = inp_shape
    self.out_shape = out_shape
    
    self.stack = [tf.keras.layers.Dense(i, activation='relu') for i in units]
    self.out = tf.keras.layers.Dense(self.out_shape, activation='tanh')
  
  def call(self, inp):
    x = inp
    for _layer in self.stack:
      x = _layer(x)
    return self.out(x)
  
  def functional(self):
    inputs = tf.keras.Input(self.inp_shape)
    outputs = self.call(inputs)
    return tf.keras.Model(inputs, outputs, name=self.name)
sample_model = DNNModel(
    len(FEATURE_CLASSES), len(LABEL_CLASSES)
)
for i in dataset.take(1):
  sample_pred = sample_model(tf.expand_dims(i[0], 0))
  print('Sample prediction of shape {}:\n{}'.format(
      sample_pred.shape, sample_pred
  ))<jupyter_output><empty_output><jupyter_text>Model Visualization<jupyter_code>sample_model.summary()
tf.keras.utils.plot_model(sample_model.functional(), to_file="model.png")<jupyter_output><empty_output><jupyter_text>Defining losses and optimizers<jupyter_code>mae = tf.keras.losses.MeanAbsoluteError()
optim = tf.keras.optimizers.Adam(1e-3)
model = DNNModel(
    len(FEATURE_CLASSES), len(LABEL_CLASSES)
)
model.compile(optimizer = optim, loss = mae)<jupyter_output><empty_output><jupyter_text>Defining training checkpoints<jupyter_code>checkpoint_dir = './ProjectECO_Beta_Checkpoints/'

if not os.path.exists(checkpoint_dir):
  os.mkdir(checkpoint_dir)

checkpoint_prefix = os.path.join(checkpoint_dir, "ckpt_{epoch}")

checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
    filepath = checkpoint_prefix,
    save_weights_only = True)<jupyter_output><empty_output><jupyter_text>TRAINING!!<jupyter_code>model.fit(train_dataset.repeat(), callbacks=[checkpoint_callback],
          steps_per_epoch = 1000, epochs = 10)
model.evaluate(test_dataset)
model.save_weights('beta_weights.h5')
model.functional().save('beta_model_func.h5')
from google.colab import files

for name in os.listdir('./'):
  if not os.path.isdir(name):
    files.download(name)<jupyter_output><empty_output><jupyter_text>Standalone Prediction<jupyter_code>!pip install tabulate
!curl -o label_classes.py https://raw.githubusercontent.com/Harrow-Enigma/TeamEngima-ProjectEco-AI/main/label_classes.py
import numpy as np
import pandas as pd
import tensorflow as tf
import matplotlib as mpl
import matplotlib.pyplot as plt
from label_classes import Label, FloatLabel, IntClass, IntClassMap

from datetime import timedelta, timezone
from datetime import datetime as dt
from tabulate import tabulate
from tqdm import tqdm
import pickle as pkl
import requests
import json
import os<jupyter_output><empty_output><jupyter_text>Pre-Requisite Objects<jupyter_code>def standardize_on_params(arr, m, s):
  return (arr - m)/s

def destandardize(arr, m, s):
  return arr * s + m

def rescale_on_params(arr, arr_ofst, arr_range):
  return (arr - arr_ofst) / arr_range

def descale(arr, arr_ofst, arr_range):
  return arr * arr_range + arr_ofst

def normalize_on_params(arr, m, s, o, r):
  arr_std = standardize_on_params(arr, m, s)
  arr_norm = rescale_on_params(arr_std, o, r)
  return arr_norm

def denormalize(arr, m, s, o, r):
  return destandardize(descale(arr, o, r), m, s)
obj = pkl.load(open('beta_data_aux.data', 'rb'))
FEATURE_CLASSES = obj['features']['classes']
features_mean = obj['features']['mean']
features_stddv = obj['features']['std_dev']
features_ofst = obj['features']['offset']
features_range = obj['features']['range']

LABEL_CLASSES = obj['labels']['classes']
labels_mean = obj['labels']['mean']
labels_stddv = obj['labels']['std_dev']
labels_ofst = obj['labels']['offset']
labels_range = obj['labels']['range']
model = tf.keras.models.load_model('beta_model_func.h5')<jupyter_output><empty_output><jupyter_text>Pre-Requisite Data<jupyter_code>def handle_exception(feat_class, timestep_data):
  try:
    _val = timestep_data[feat_class.key]
    if _val is not None:
      return feat_class.fwd_call(_val)
    return feat_class.fallback
  except:
    return feat_class.fallback

beta_api_url = 'https://dev-test.projecteco.ml/api/v1/rest/output/forms/'
response = requests.get(beta_api_url)
json_data = response.json()

for k in range(len(json_data)):
  try:
    _ = json_data[k]['localpollutiondata']
    _has_key = True
  except:
    print('Error: No `localpollutiondata` key')
    _has_key = False
  
  if _has_key:
    for pol_key in list(json_data[k]['localpollutiondata']\
                        ['data'].keys()):
      json_data[k][pol_key] = json_data[k]['localpollutiondata']\
                                       ['data'][pol_key]['v']
    json_data[k].pop('localpollutiondata')
  
server_features = []
server_labels = []

for t in json_data:
  server_features.append([handle_exception(c, t)
                          for c in FEATURE_CLASSES])
  server_labels.append([handle_exception(c, t)
                        for c in LABEL_CLASSES])

server_features = np.array(server_features, np.float32)
server_labels = np.array(server_labels, np.float32)<jupyter_output><empty_output><jupyter_text>Visualization Code<jupyter_code>def visualize_inputs(inps, headers=('Input name', 'Input value')):
  data_arr = [[c.name, c.fwd_call(val)] for c, val in zip(FEATURE_CLASSES, inps)]
  print(tabulate(data_arr, headers=headers))

def visualize_features(feat, headers=('Feature name', 'Feature value')):
  data_arr = [[c.name, val] for c, val in zip(FEATURE_CLASSES, feat)]
  print(tabulate(data_arr, headers=headers))

def visualize_preds(preds, headers=('Prediction name', 'Prediction value')):
  data_arr = [[c.name, c.rev_call(val)] for c, val in zip(LABEL_CLASSES, preds)]
  print(tabulate(data_arr, headers))
def preproc(inps):
  feats = np.array([[c.fwd_call(val) for c, val in zip(FEATURE_CLASSES, entry)]
                   for entry in inps])
  feats_norm = normalize_on_params(feats, features_mean, features_stddv, 
                                   features_ofst, features_range)
  return feats_norm

def denorm_preds(preds_norm):
  return denormalize(preds_norm, labels_mean, labels_stddv,
                     labels_ofst, labels_range)

def visualize(inps, see_feats=False):
  feats_norm = preproc(inps)
  preds_norm = model.predict(feats_norm)
  preds = denormalize(preds_norm, labels_mean, labels_stddv,
                      labels_ofst, labels_range)
  
  for e, (i, f, p) in enumerate(zip(inps, feats_norm, preds)):
    print('Visualizing input {}'.format(e))
    print()
    visualize_inputs(i)
    print()
    if see_feats:
      visualize_features(f)
      print()
    visualize_preds(p)
    print('\n=====================================================================\n\n')

def compare_vis(inps, labs, see_feats=False):
  feats_norm = preproc(inps)
  preds_norm = model.predict(feats_norm)
  preds = denorm_preds(preds_norm)
  
  for e, (i, f, p, g) in enumerate(zip(inps, feats_norm, preds, labs)):
    print('Visualizing input {}'.format(e))
    print()
    visualize_inputs(i)
    print()
    if see_feats:
      visualize_features(f)
      print()
    visualize_preds(p, headers=("Prediction Name", "Model Output"))
    print()
    visualize_preds(g, headers=("Prediction Name", "Ground Truth"))
    print('\n==================================================================\n\n')
def locate_label_by_name(_name, classes):
  for e, i in enumerate(classes):
    if i.name == _name:
      return e

def replace_in_arr(arr, idx, newval):
  ret = arr.copy()
  ret[idx] = newval
  return ret

def makegraph(x, y, xlab='', ylab='', title=''):
  plt.figure(figsize=(8, 5), dpi=150)
  plt.plot(x, y)
  plt.xlabel(xlab)
  plt.ylabel(ylab)
  plt.title(title)
  plt.show()

def autorange(featurename, sub_d=100):
  feats = server_features[:, locate_label_by_name(featurename,
                                                  FEATURE_CLASSES)]
  step = (feats.max() - feats.min()) / (sub_d - 1)
  return np.arange(feats.min(), feats.max() + step, step)

def plot_relation_by_name(featurename,
                          labelname,
                          _range,
                          _vars=server_features[0]):
  _vars = server_features[0]
  _feat_loc = locate_label_by_name(featurename, FEATURE_CLASSES)
  _lab_loc = locate_label_by_name(labelname, LABEL_CLASSES)

  _x = [replace_in_arr(_vars, _feat_loc, i) for i in _range]
  _y = denorm_preds(model.predict(preproc(_x)))[:, _lab_loc]

  makegraph(_range, _y, featurename, labelname,
            'How {} responses vary with {} levels'.format(labelname,
                                                          featurename))

def plot_relation_from_data(featurename,
                            labelname):
  _x = server_features[:, locate_label_by_name(featurename, FEATURE_CLASSES)]
  _y = server_labels[:, locate_label_by_name(labelname, LABEL_CLASSES)]

  _idx = np.argsort(_x)
  _x = _x[_idx]
  _y = _y[_idx]

  makegraph(_x, _y, featurename, labelname,
            'How {} responses vary with {} levels'.format(labelname,
                                                          featurename))<jupyter_output><empty_output><jupyter_text>Visualization<jupyter_code>visualize(server_features)
compare_vis(server_features, server_labels)
server_features[:, locate_label_by_name('pm10', FEATURE_CLASSES)]
# How stress levels vary with PM2.5 concentration,
# assuming that all other values follow server_features[0]
plot_relation_by_name('pm10', 'Q3 (Work Stress)',
                      autorange('pm25', 100))
plot_relation_by_name('no2', 'Q2 (Concentration)',
                      autorange('pm25', 100))
plot_relation_from_data('no2', 'Q2 (Concentration)')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13235668789808916, 'fraction_numerical': 0.007388535031847133, 'mean_word_length': 4.092766785598443, 'pattern_counts': {'":': 0, '<': 58, '<?xml version=': 0, '>': 58, 'https://': 4, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '31695', 'license': 'Apache-2.0', 'n_tokens_mistral': 5548, 'n_tokens_neox': 5395, 'n_words': 1110, 'path': 'Project_Eco_AI_Beta_Testing.ipynb', 'repo_name': 'Harrow-Enigma/TeamEngima-ProjectEco-AI', 'size': '15706'}