<jupyter_start><jupyter_text>SPLITTING DATASET IN TRAIN AND TEST SET<jupyter_code># Keep 20% of the data out for validation
### START CODE HERE ### (1 line)
train_reference_df, val_reference_df = train_test_split(dataset_one_hot_df, test_size=0.2, shuffle=True, random_state=123)
### END CODE HERE ###


def print_dataset_statistics(train_reference_df, val_reference_df):

    # Count the elements in the sets
    num_train_data_sit = sum(train_reference_df['gt_sit'] == 1)
    num_train_data_stand = sum(train_reference_df['gt_stand'] == 1)
    num_train_data_walk = sum(train_reference_df['gt_walk'] == 1)
    num_train_data_bike = sum(train_reference_df['gt_bike'] == 1)
    num_train_data_stairs_up = sum(train_reference_df['gt_stairsup'] == 1)
    num_train_data_stairs_down = sum(train_reference_df['gt_stairsdown'] == 1)

    num_val_data_sit = sum(val_reference_df['gt_sit'] == 1)
    num_val_data_stand = sum(val_reference_df['gt_stand'] == 1)
    num_val_data_walk = sum(val_reference_df['gt_walk'] == 1)
    num_val_data_bike = sum(val_reference_df['gt_bike'] == 1)
    num_val_data_stairs_up = sum(val_reference_df['gt_stairsup'] == 1)
    num_val_data_stairs_down = sum(val_reference_df['gt_stairsdown'] == 1)



    print('TRAIN SET')
    print('\tSit:\t\t{} ({:.2f}%)'.format(num_train_data_sit, 100 * num_train_data_sit / len(train_reference_df)))
    print('\tStand:\t\t{} ({:.2f}%)'.format(num_train_data_stand, 100 * num_train_data_stand / len(train_reference_df)))
    print('\tWalk:\t\t{} ({:.2f}%)'.format(num_train_data_walk, 100 * num_train_data_walk / len(train_reference_df)))
    print('\tBike:\t\t{} ({:.2f}%)'.format(num_train_data_bike, 100 * num_train_data_bike / len(train_reference_df)))
    print('\tStairs up:\t{} ({:.2f}%)'.format(num_train_data_stairs_up, 100 * num_train_data_stairs_up / len(train_reference_df)))
    print('\tStairs down:\t{} ({:.2f}%)'.format(num_train_data_stairs_down, 100 * num_train_data_stairs_down / len(train_reference_df)))


    print('VALIDATION SET')
    print('\tSit:\t\t{} ({:.2f}%)'.format(num_val_data_sit, 100 * num_val_data_sit / len(val_reference_df)))
    print('\tStand:\t\t{} ({:.2f}%)'.format(num_val_data_stand, 100 * num_val_data_stand / len(val_reference_df)))
    print('\tWalk:\t\t{} ({:.2f}%)'.format(num_val_data_walk, 100 * num_val_data_walk / len(val_reference_df)))
    print('\tBike:\t\t{} ({:.2f}%)'.format(num_val_data_bike, 100 * num_val_data_bike / len(val_reference_df)))
    print('\tStairs up:\t{} ({:.2f}%)'.format(num_val_data_stairs_up, 100 * num_val_data_stairs_up / len(val_reference_df)))
    print('\tStairs down:\t{} ({:.2f}%)'.format(num_val_data_stairs_down, 100 * num_val_data_stairs_down / len(val_reference_df)))


print_dataset_statistics(train_reference_df, val_reference_df)

print(train_reference_df)

def extract_basic_features(acc_x, acc_y, acc_z):
    # prova = np.array(np.apply_along_axis(np.histogram, 1, acc_x)[0]).reshape(2,1)

    np_acc_x = np.array(acc_x)
    np_acc_y = np.array(acc_y)
    np_acc_z = np.array(acc_z)

    mean_x = np.expand_dims(np.mean(np_acc_x, axis=1), axis=0).T 
    mean_y = np.expand_dims(np.mean(np_acc_y, axis=1), axis=0).T 
    mean_z = np.expand_dims(np.mean(np_acc_z, axis=1), axis=0).T 

    basic_features = np.concatenate( (
        # insert MEANS 
        mean_x,
        mean_y,
        mean_z,

        # insert STD
        np.expand_dims(np.std(np_acc_x, axis=1), axis=0).T, 
        np.expand_dims(np.std(np_acc_y, axis=1), axis=0).T, 
        np.expand_dims(np.std(np_acc_z, axis=1), axis=0).T, 

        # insert sum of thew absolute values
        np.expand_dims(np.mean(abs(np_acc_x - mean_x), axis=1), axis=1),
        np.expand_dims(np.mean(abs(np_acc_y - mean_y), axis=1), axis=1),
        np.expand_dims(np.mean(abs(np_acc_z - mean_z), axis=1), axis=1),

        np.expand_dims(np.mean( np.sqrt( np.power(np_acc_x, 2) + np.power(np_acc_y,2) + np.power(np_acc_z, 2) ), axis=1), axis=0).T
        
    ), axis=1).tolist()

    for i in range(0, len(acc_x)):
        bins_x, centers_x = np.histogram(acc_x[i], bins=10)
        bins_y, centers_y = np.histogram(acc_y[i], bins=10)
        bins_z, centers_z = np.histogram(acc_z[i], bins=10)

        basic_features[i].extend(bins_x / len(acc_x))
        basic_features[i].extend(bins_y / len(acc_y))
        basic_features[i].extend(bins_z / len(acc_z))

    return basic_features

prova_feat = np.array(extract_basic_features( [[1,10,3,4,5,6,7,8,9,10], [1,2,3,4,5,6,7,8,9,10]] , [[1,3,3,4,5,6,7,8,9,10], [1,2,3,4,5,6,7,8,9,10]], [[1,3,3,4,5,6,7,8,9,10], [1,2,3,4,5,6,7,8,9,10]]))

print(prova_feat)
def create_dataset(reference_df, batch_size, shuffle, cache_file, center_data=False):
    target = reference_df[['gt_sit','gt_stand','gt_walk','gt_bike','gt_stairsup','gt_stairsdown']].values.astype(int).tolist()

    # RESHAPING DATAS
    np_data = np.array(reference_df.iloc[:,0:750])

    np_reshaped_data = np.reshape(np_data.copy(), (np_data.shape[0], 6, 125))

    # Data centering
    if center_data:
        for i in range(len(np_reshaped_data)):
            window = np_reshaped_data[i]
            means = np.mean(window, axis=1)
            centered_acc = np.array(([window[j] - means[j] for j in range(3)]))
            np_reshaped_data[i] = np.concatenate((centered_acc, window[3:]), axis=0)
             
    
    # Extract manual features
    np_basic_features = np.array(extract_basic_features(np_data[:, 0:125], np_data[:, 125:250], np_data[:, 250: 375]))

    # Create dataset obj
    dataset = tf.data.Dataset.from_tensor_slices( ({"input_1": np_reshaped_data, "input_2": np_basic_features}, target) )

    # Cache dataset
    if cache_file:
        dataset = dataset.cache(cache_file)

    # Shuffle
    if shuffle:
        dataset = dataset.shuffle(len(target))

    # Repeat the dataset indefinitely
    dataset = dataset.repeat()

    # Batch
    dataset = dataset.batch(batch_size=batch_size)

    # Prefetch
    dataset = dataset.prefetch(buffer_size=1)

    return dataset

batch_size = 128

training_dataset = create_dataset(train_reference_df, batch_size=batch_size, shuffle=False, cache_file=None)
val_dataset = create_dataset(val_reference_df, batch_size=batch_size, shuffle=False, cache_file=None)

for train, targ in training_dataset.take(1):
  print ('Features: {}, Target: {}'.format(train, targ))

train_steps = int(np.ceil(len(train_reference_df)/batch_size))
val_steps = int(np.ceil(len(val_reference_df)/batch_size))
def build_model(input_shape):

    l2_reg = 5e-4

    encoder = tf.keras.models.load_model('encoder.h5')

    # NOT TRAIN THE MODEL
    encoder.trainable = False

    # Define the input placeholder as a tensor with shape input_shape. Think of this as your input image!
    training_input = tf.keras.Input(shape=input_shape, dtype=tf.float32, name='input_1')
    basic_feat_input = tf.keras.Input(shape=40, dtype=tf.float32, name='input_2')

    CNN = tf.keras.layers.Conv1D(196, 16, activation='relu', padding='same')(training_input)
    CNN = tf.keras.layers.MaxPool1D(4, padding='same')(CNN)
    
    feautures_CCN = tf.keras.layers.Flatten()(CNN)
    
    featuers_encoder = encoder(training_input)

    features = tf.concat((feautures_CCN, basic_feat_input), 1) 

    #features = tf.concat((feautures_CCN), 1) 

    FFNN = tf.keras.layers.Dense(1024, activation='relu', kernel_regularizer=tf.keras.regularizers.L2(l2_reg), activity_regularizer=tf.keras.regularizers.L2(l2_reg))(features)
    FFNN = tf.keras.layers.Dropout(0.05)(FFNN)
    model_output = tf.keras.layers.Dense(6, activation='softmax')(FFNN)

    model = tf.keras.Model(inputs = [training_input, basic_feat_input], outputs = model_output, name='OurModel')

    return model

model = build_model((6,125))

adam_optimizer = tf.keras.optimizers.Adam(learning_rate=5e-4)
loss_funct = tf.keras.losses.CategoricalCrossentropy()

model.compile(optimizer = adam_optimizer, loss = loss_funct, metrics = ["accuracy"])
print(model.summary())

early_stopping_callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=3)

model.fit(training_dataset, epochs = 100, steps_per_epoch=train_steps, validation_data=val_dataset, validation_steps=val_steps,  callbacks = [early_stopping_callback])
model.save('./models/model.h5')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15311293436293436, 'fraction_numerical': 0.030284749034749033, 'mean_word_length': 4.720496894409938, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '299996', 'license': 'MIT', 'n_tokens_mistral': 3370, 'n_tokens_neox': 3178, 'n_words': 464, 'path': 'training_model_no_user_splitting.ipynb', 'repo_name': 'lparolari/har', 'size': '8288'}