<jupyter_start><jupyter_text>Read data from csv<jupyter_code>df = pd.read_csv(data_file_path)
df.shape
df.head()<jupyter_output><empty_output><jupyter_text>Basic sanity check of data and normalization<jupyter_code># check for null data
df.isnull().values.any()

# Standard scaling : mean 0, stddev 1
scaler = preprocessing.StandardScaler()
df['CPU'] = scaler.fit_transform(df['CPU'].values.reshape(-1, 1))

scaler_mean = scaler.mean_
scaler_var = scaler.var_ ** 0.5

print(scaler_mean)  ## mean
print(scaler_var)   ## std dev

df.head()
# let's explore the distribution of input data
count_classes = pd.value_counts(df['Class'], sort = False)
print(count_classes.count)
count_classes.plot(kind = 'bar', rot=0)
plt.title("Transaction class distribution")
plt.xticks(range(2), LABELS)
plt.xlabel("Class")
plt.ylabel("Frequency");
# Just checking the relative counts
anomaly = df[df.Class == 1]
normal = df[df.Class == 0]

anomaly.shape
normal.shape
anomaly.CPU.describe()
normal.CPU.describe()<jupyter_output><empty_output><jupyter_text>Any correlation between time and CPU metrics ?<jupyter_code>f, (ax1, ax2) = plt.subplots(2, 1, sharex=True)
f.suptitle('Time vs CPU by class')

ax1.scatter(anomaly.Time, anomaly.CPU)
ax1.set_title('Anomaly')

ax2.scatter(normal.Time, normal.CPU)
ax2.set_title('Normal')

plt.xlabel('Time (in Seconds)')
plt.ylabel('CPU')
plt.show()<jupyter_output><empty_output><jupyter_text>Feature Re-engineering<jupyter_code>def widenX(width, x) :
    slicestart = 0
    sliceend = width

    X_data = []
    newlength = x.size - width + 1
    for i in range(newlength):
        X_data.append(x[slicestart:sliceend])
        slicestart = slicestart + 1
        sliceend = sliceend + 1

    return np.reshape(X_data, newshape=(newlength, width))

def widenY(width, y):
    return y[width-1:]
df.head()
anomaly = df[df.Class == 1]
anomaly.shape
X_train = widenX(lookback, df['CPU'])
X_train.shape
Y_train = widenY(lookback, df['Class'])
Y_train.shape
class_0 = list(filter(lambda x: x == 0.0, Y_train)) 
class_1 = list(filter(lambda x: x == 1.0, Y_train)) 
print(len(class_0), len(class_1))<jupyter_output>83439 16560<jupyter_text>Split into training and test set<jupyter_code>original_X_train_size = X_train.shape[0]
train_set_size = int(0.8 * original_X_train_size)
test_set_size = original_X_train_size - train_set_size

X_test = X_train[original_X_train_size - test_set_size : -1]
X_train = X_train[0 : train_set_size]

Y_test = Y_train[original_X_train_size - test_set_size : -1]
Y_train = Y_train[0 : train_set_size]

print(int(train_set_size))
print(int(test_set_size))
print(X_train.shape)
print(X_test.shape)
X_train.shape
X_test.shape<jupyter_output><empty_output><jupyter_text>In order to get data into BigDL engine we need to convert the numpy arrays into Spark RDD. Here's a helper function that does this. Note that we will use negative log likelihood for loss computation - hence based on the documentation of BigDL, the labels need to start from 1. Hence we add a 1 in the function below to all the labels.<jupyter_code>def get_rdd_from_ndarray(sc):
    rdd_X_train = sc.parallelize(X_train)
    rdd_Y_train = sc.parallelize(Y_train)
    rdd_X_test = sc.parallelize(X_test)
    rdd_Y_test = sc.parallelize(Y_test)

    rdd_train_sample = rdd_X_train.zip(rdd_Y_train).map(lambda labeledFeatures:
                                       common.Sample.from_ndarray(labeledFeatures[0], labeledFeatures[1]+1))
    rdd_test_sample = rdd_X_test.zip(rdd_Y_test).map(lambda labeledFeatures:
                                     common.Sample.from_ndarray(labeledFeatures[0], labeledFeatures[1]+1))
    return (rdd_train_sample, rdd_test_sample)

(train_data, test_data) = get_rdd_from_ndarray(sc)<jupyter_output><empty_output><jupyter_text>Build the Model<jupyter_code># create a graph model
def make_new_model(X_train):

    ## input layer with relu and dropout
    initial = Linear(X_train.shape[1], 16).set_name("input")()
    relu1 = ReLU()(initial)
    dropout1 = Dropout(0.3)(relu1)

    ## first hidden layer with relu and dropout
    hidden1 = Linear(16, 32)(dropout1)
    relu2 = ReLU()(hidden1)
    dropout2 = Dropout(0.4)(relu2)

    ## second hidden layer with relu and dropout
    hidden2 = Linear(32, 32)(dropout2)
    relu3 = ReLU()(hidden2)
    dropout3 = Dropout(0.4)(relu3)

    ## output layer with softmax(2) and dropout
    output = Linear(32, 2)(dropout3)
    softmax = SoftMax().set_name("output")(output)

    return Model([initial], [softmax])<jupyter_output><empty_output><jupyter_text>Fetch Hyperparameters We want to fetch hyperparameters from a file. Besides externalizing the hyperparameters, this also allows us to train multiple models by passing in different hyperparameters.<jupyter_code># read hyperparameters from file, if exists
if (os.path.exists(hyperparams_file_path)):
    print("Got hyperparameter file ..")    
    config = configparser.RawConfigParser()
    config.read(hyperparams_file_path)
    learning_rate = float(config.get('HyperparameterSection', 'learning_rate'))
    training_epochs = int(config.get('HyperparameterSection', 'training_epochs'))
    batch_size = int(config.get('HyperparameterSection', 'batch_size'))
else:
    learning_rate = 0.001
    training_epochs = 8
    batch_size = 256
    
print('learning rate', learning_rate)
print('training epochs', training_epochs)
print('batch size', batch_size)

# since we save the model every time, we need to hash the file name based on hyperparameter
# settings, so that the correct file name is picked up when we attempt to find and load an exiating model
local_model_file_path = '/tmp/model-' + str(learning_rate) + '-' + str(training_epochs) + '-' + str(batch_size) + '.bigdl'
local_model_weights_file_path = '/tmp/model-' + str(learning_rate) + '-' + str(training_epochs) + '-' + str(batch_size) + '.bin'
print(local_model_file_path)
print(local_model_weights_file_path)<jupyter_output>/tmp/model-0.001-8-256.bigdl
/tmp/model-0.001-8-256.bin<jupyter_text>Load Model for Incremental Learning We would like to do incremental learning. Hence if we find an existing BigDL model and the weights, then we load from that model and then continue training. Else we build the new model. The model and the weights file names are hashed based on hyperparameter settings.<jupyter_code>if (os.path.exists(local_model_file_path) and os.path.exists(local_model_weights_file_path)):
    print("Got existing model .. loading ..")
    model = Model.loadModel(local_model_file_path, local_model_weights_file_path) # load from local fs
else:
    model = make_new_model(X_train)<jupyter_output>Got existing model .. loading ..<jupyter_text>Optimize and Train<jupyter_code>optimizer = Optimizer(
    model = model,
    training_rdd = train_data,
    criterion = ClassNLLCriterion(),
    optim_method = Adam(learningrate=learning_rate),
    end_trigger = MaxEpoch(training_epochs),
    batch_size = batch_size)

# Set the validation logic
optimizer.set_validation(
    batch_size = batch_size,
    val_rdd = test_data,
    trigger = EveryEpoch(),
    val_method = [Top1Accuracy()]
)

log_dir = 'mylogdir'

app_name='anomaly-cpu-' + dt.datetime.now().strftime("%Y%m%d-%H%M%S")
train_summary = TrainSummary(log_dir=log_dir, app_name=app_name)
train_summary.set_summary_trigger("Parameters", SeveralIteration(50))
val_summary = ValidationSummary(log_dir=log_dir, app_name=app_name)
optimizer.set_train_summary(train_summary)
optimizer.set_val_summary(val_summary)
print("saving logs to ", app_name)
%%time
# Boot training process
trained_model = optimizer.optimize()
print("Optimization Done.")
# save BigDL model locally
model.saveModel(local_model_file_path, local_model_weights_file_path, True) # save to local fs

# model.save_tensorflow([("input", [1, 3])], "/tmp/model.pb")
model.save_tensorflow([("input", [1, 3])], model_pb_file_path)

loss = np.array(train_summary.read_scalar("Loss"))
top1 = np.array(val_summary.read_scalar("Top1Accuracy"))

plt.figure(figsize = (12,12))
plt.subplot(2,1,1)
plt.plot(loss[:,0],loss[:,1],label='loss')
plt.xlim(0,loss.shape[0]+10)
plt.grid(True)
plt.title("loss")
plt.subplot(2,1,2)
plt.plot(top1[:,0],top1[:,1],label='top1')
plt.xlim(0,loss.shape[0]+10)
plt.title("top1 accuracy")
plt.grid(True)<jupyter_output><empty_output><jupyter_text>Predict on test set<jupyter_code>def map_predict_label(l):
    return np.array(l).argmax()
def map_groundtruth_label(l):
    return int(l[0] - 1)
predictions = trained_model.predict(test_data)
%%time
predictions = trained_model.predict(test_data)
print('Ground Truth labels:')
print(', '.join(str(map_groundtruth_label(s.label.to_ndarray())) for s in test_data.take(50)))
print('Predicted labels:')
print(', '.join(str(map_predict_label(s)) for s in predictions.take(50)))
labels = [map_groundtruth_label(s.label.to_ndarray()) for s in test_data.take(20000)]
df_prediction = pd.DataFrame({'Real Class' :np.array(labels)})
predicted_labels = [map_predict_label(s) for s in predictions.take(20000)]
df_prediction['Prediction'] = predicted_labels
total_size = X_test.shape[0]
mismatch_size = df_prediction[ df_prediction['Real Class'] != df_prediction['Prediction'] ].size
accuracy = ((total_size - mismatch_size) / total_size) * 100
print(total_size)
print(mismatch_size)
print(accuracy)
import datetime
now = datetime.datetime.now()

# with open("/tmp/model-attrib.properties", "w") as fp:
with open(model_attrib_file_path, "w") as fp:
    fp.write("width=" + str(lookback) + "\n")
    fp.write("mean=" + str(scaler_mean[0]) + "\n")
    fp.write("std=" + str(scaler_var[0]) + "\n")
    fp.write("input=input\n")
    fp.write("output=output\n")
    fp.write("generatedAt=" + str(datetime.datetime.now()) + "\n")
with open(generation_complete_file_path, "w") as fp:
    fp.write(str(datetime.datetime.now()))<jupyter_output><empty_output><jupyter_text>Predict on a new dataset<jupyter_code>df = pd.read_csv("data/CPU_examples_test.csv")

## standardize and widen
df['CPU'] = df['CPU'].apply(lambda x: (x - scaler_mean) / scaler_var)
X_test = widenX(lookback, df['CPU'])
Y_test = widenY(lookback, df['Class'])

print(X_test.shape)
anomaly = df[df.Class == 1]
anomaly.shape
def get_rdd_from_ndarray_test(sc):
    rdd_X_test = sc.parallelize(X_test)
    rdd_Y_test = sc.parallelize(Y_test)

    rdd_test_sample = rdd_X_test.zip(rdd_Y_test).map(lambda labeledFeatures:
                                     common.Sample.from_ndarray(labeledFeatures[0], labeledFeatures[1]+1))
    return rdd_test_sample

test_data = get_rdd_from_ndarray_test(sc)
%%time
predictions = trained_model.predict(test_data)
print('Ground Truth labels:')
print(', '.join(str(map_groundtruth_label(s.label.to_ndarray())) for s in test_data.take(50)))
print('Predicted labels:')
print(', '.join(str(map_predict_label(s)) for s in predictions.take(50)))
labels = [map_groundtruth_label(s.label.to_ndarray()) for s in test_data.take(X_test.shape[0])]
df_prediction = pd.DataFrame({'Real Class' :np.array(labels)})
predicted_labels = [map_predict_label(s) for s in predictions.take(X_test.shape[0])]
df_prediction['Prediction'] = predicted_labels
total_size = X_test.shape[0]
mismatch_size = df_prediction[ df_prediction['Real Class'] != df_prediction['Prediction'] ].size
accuracy = ((total_size - mismatch_size) / total_size) * 100
print(total_size)
print(mismatch_size)
print(accuracy)<jupyter_output>49999
5290
89.41978839576792
{'fraction_non_alphanumeric': 0.130411873188724, 'fraction_numerical': 0.01879336085009221, 'mean_word_length': 5.647985989492119, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '264743', 'license': 'Apache-2.0', 'n_tokens_mistral': 4203, 'n_tokens_neox': 4029, 'n_words': 823, 'path': 'apps/python/lightbend/dnn_anomaly_bigdl.ipynb', 'repo_name': 'lightbend/analytics-zoo', 'size': '11387'}