<jupyter_start><jupyter_text>Auto-reload any function that are configured on the back-end.<jupyter_code>%load_ext autoreload
%autoreload 2
import os
from os import walk
# main app directory
main_app_path = os.path.dirname(os.path.dirname(os.getcwd()))
# changes the current working directory to the given path
os.chdir(main_app_path)
import sys
print("Python version")
print (sys.version)
print("Version info.")
print (sys.version_info)
''' 
imports:
functions calculating mean, standard deviation, 
and stacking the 2d matrices along a 3rd dimension.
'''
from numpy import mean
from numpy import std
from numpy import dstack
from pathlib import Path
# imports
import pandas 
from pandas import read_csv
from pandas import DataFrame
# reads the csv file as a DataFrame 
from matplotlib import pyplot
# library for plotting
from sklearn.preprocessing import StandardScaler
# standard scaling 
import os
from datetime import datetime
import tensorflow
from tensorflow import keras
# different layers we are going to need for the NN design
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import Flatten
from tensorflow.keras.layers import Dropout
from tensorflow.keras.layers import Conv1D
from tensorflow.keras.layers import MaxPooling1D 
# one-hot encoding
from tensorflow.keras.utils import to_categorical

# Load the TensorBoard notebook extension, for visualization purposes and more
# %load_ext tensorboard
tensorflow.__version__
tensorflow.keras.backend.set_floatx('float64')
pandas.set_option('display.max_columns', 30)
class LoadDatasetNeural:
    def __init__(self, filepath):
        self.filepath = filepath

    # load a single file as a numpy array
    def load_dataset_nn_file(self):
        datasets_dir = 'datasets'
        curr_dir = Path().absolute()  # current dir
#         filepath = os.path.join(curr_dir.parent, datasets_dir, self.filepath)
        filepath = os.path.join(curr_dir, datasets_dir, self.filepath)
        dataframe = read_csv(filepath, header=None, delim_whitespace=True)
        return dataframe.values
# load a list of files and return as a 3d numpy array
def load_group(filenames, prefix=''):
    loaded = list()
    for name in filenames:
        data_obj = LoadDatasetNeural(prefix + name)
        data = data_obj.load_dataset_nn_file()
        loaded.append(data)
    # stack group so that features are the 3rd dimension
    loaded = dstack(loaded)
    return loaded
# load a dataset group, such as train or test
def load_dataset_group(group, prefix=''):
    filepath = prefix + group + '/Inertial Signals/'
    # load all 9 files as a single array
    filenames = list()
    # total acceleration
    filenames += ['total_acc_x_' + group + '.txt', 'total_acc_y_' + group + '.txt', 'total_acc_z_' + group + '.txt']
    # body acceleration
    filenames += ['body_acc_x_' + group + '.txt', 'body_acc_y_' + group + '.txt', 'body_acc_z_' + group + '.txt']
    # body gyroscope
    filenames += ['body_gyro_x_' + group + '.txt', 'body_gyro_y_' + group + '.txt', 'body_gyro_z_' + group + '.txt']
    # load input data
    X = load_group(filenames, filepath)
    # load class output
    load_file = LoadDatasetNeural(prefix + group + '/y_' + group + '.txt')
    y = load_file.load_dataset_nn_file()
    return X, y
# load the dataset, returns train and test X and y elements
def load_dataset(prefix=''):
    # load all train
    trainX, trainy = load_dataset_group('train', prefix + 'HARDataset/')
    print(trainX.shape, trainy.shape)
    # load all test
    testX, testy = load_dataset_group('test', prefix + 'HARDataset/')
    print(testX.shape, testy.shape)
    # zero-offset class values
    trainy = trainy - 1
    testy = testy - 1
    # one hot encode y
    trainy = keras.utils.to_categorical(trainy)
    print('One hot trainy shape:', trainy.shape)
    testy = keras.utils.to_categorical(testy)
    print('One hot testy shape:', testy.shape)
    return trainX, trainy, testX, testy
class SoftmaxResults:

    def __init__(self, model, dataset, names):
        self.model = model
        self.dataset = dataset
        self.names = names
        self.df = pandas.DataFrame()

        '''
        The test data is passed to the model and we get the results.
        training = False is set so that we evaluate the model's predictions.
        '''
    def get_softmax_results(self):
        predictions = self.model(self.dataset, training=False)
        # we get the logits and iterate over the predictions
        for i, logits in enumerate(predictions):
            # get the probabilities from the logits, from the softmax layer.
            p = tensorflow.nn.softmax(logits)
            # turn the tf Tensor into a NumPy array for easier manipulation
            p = p.numpy()
            # then, multiply by 100 to get the probabilities as a percentage.
            p = 100 * p
            # turn the NumPy array to a list for easier manipulation
            smaxlist = p.tolist()
            # index is the class index, so we have to get the index of the max value of softmax.
            index = smaxlist.index(max(smaxlist))
            # create a df based on the probabilities matrix and column names.
            # [p] is used because p is a numpy array.
            df = pandas.DataFrame([p], columns=self.names)
            # add a column to the df with the class name (classification result)
            df['CLASS'] = self.names[index]
            # append the row to the final DataFrame
            self.df = self.df.append(df)
        # create a new column with the max value of the DataFrame (efficient)
        self.df['ARGMAX'] = self.df.max(axis=1)
        return self.df
# fit and evaluate a model
def evaluate_model(trainX, trainy, testX, testy, i):
    verbose, epochs, batch_size = 0, 10, 32
    n_timesteps, n_features, n_outputs = trainX.shape[1], trainX.shape[2], trainy.shape[1]
    model = keras.models.Sequential()
    model.add(keras.layers.Conv1D(filters=64, kernel_size=3, activation='relu', input_shape=(n_timesteps, n_features)))
    model.add(keras.layers.Conv1D(filters=64, kernel_size=3, activation='relu'))
    model.add(keras.layers.Dropout(0.5))
    model.add(keras.layers.MaxPooling1D(pool_size=2))
    model.add(keras.layers.Flatten())
    model.add(keras.layers.Dense(100, activation='relu'))
    model.add(keras.layers.Dense(n_outputs, activation='softmax'))
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
    # fit network
    model.fit(trainX, trainy, epochs=epochs, batch_size=batch_size, verbose=verbose)
    # evaluate model
    _, accuracy = model.evaluate(testX, testy, batch_size=batch_size, verbose=0)
    return accuracy, model
# summarize scores
def summarize_results(scores):
    print(scores)
    m, s = mean(scores), std(scores)
    print('Accuracy: %.3f%% (+/-%.3f)' % (m, s))
''' 
Repeats a run of the NN as many times as specified by the "repeats" argument
scores is a list containing the results of each run, 
so as to be able to get the mean and std of all runs.
'''
def run_experiment(repeats):
	# load data
	names =["WALKING", "WALKING_UPSTAIRS", 
            "WALKING_DOWNSTAIRS", "SITTING" , "STANDING", "LAYING"]
	trainX, trainy, testX, testy = load_dataset()
	scores = list()
	for r in range(repeats):
		score, model = evaluate_model(trainX, trainy, testX, testy, r)
		score = score * 100.0
		print('#%d: %.3f' % (r+1, score))
		scores.append(score)
	# summarize results
	summarize_results(scores)
	model.summary()
	softmax = SoftmaxResults(model, testX, names)
	x = softmax.get_softmax_results()
	print(x.info())
#     print(x.info())
run_experiment(repeats=1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10160705028512182, 'fraction_numerical': 0.006998444790046656, 'mean_word_length': 3.491850989522701, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '393298', 'license': 'MIT', 'n_tokens_mistral': 2485, 'n_tokens_neox': 2398, 'n_words': 753, 'path': 'tasks/human_activity_recognition/model_jupyter_2.ipynb', 'repo_name': 'tzamalisp/tasks_opensource', 'size': '7716'}