<jupyter_start><jupyter_text>Hyperparameters<jupyter_code>BEST_PATH = './models/ResNet.h5'
TRAINING_EPOCHS = 200
LEARNING_RATE = 0.002
EPSILON = 1e-06
BATCH_SIZE = 16<jupyter_output><empty_output><jupyter_text>Data loading<jupyter_code>l = np.load('./results/2020_S/fw_dataset.npz', allow_pickle=True)
data_indices = l['data_indices']
input_data = l['input_data']
output_label = l['output_label']
INPUT_MAXS = l['INPUT_MAXS']
INPUT_MINS = l['INPUT_MINS']
OUTPUT_MAX = l['OUTPUT_MAX']
OUTPUT_MIN = l['OUTPUT_MIN']
input_data = input_data.astype('float32')
output_label = output_label.astype('float32')
print(input_data.shape)
print(output_label.shape)
print(INPUT_MAXS)
print(INPUT_MINS)
print(OUTPUT_MAX)
print(OUTPUT_MIN)
data_indices, input_data, output_label = shuffle(data_indices, input_data, output_label, random_state=3101)
N_TRAIN = int(input_data.shape[0]*.8)
train_input = input_data[:N_TRAIN, ...]
train_label = output_label[:N_TRAIN, ...]
val_input = input_data[N_TRAIN:, ...]
val_label = output_label[N_TRAIN:, ...]
print(f'number of training set: {train_input.shape[0]}')
print(f'number of validation set: {val_input.shape[0]}')
with strategy.scope():
    train_dataset = tf.data.Dataset.from_tensor_slices((train_input, train_label))
    train_dataset = train_dataset.cache().shuffle(BATCH_SIZE*10).batch(BATCH_SIZE, drop_remainder=False)
    val_dataset = tf.data.Dataset.from_tensor_slices((val_input, val_label))
    val_dataset = val_dataset.cache().shuffle(BATCH_SIZE*10).batch(BATCH_SIZE, drop_remainder=False)<jupyter_output><empty_output><jupyter_text>Model construction<jupyter_code>class ResNet1D(Model):
    def __init__(self):
        super(ResNet1D, self).__init__()
        self.n = [128, 128, 256, 256, 512] # number of nodes
        self.k = [1, 5, 10, 20, 50] # kernal size
        self.s = 2 # stride (= pooling size)
                
        self.conv1_1 = layers.Conv1D(self.n[0]/4, self.k[0], kernel_initializer='glorot_normal', padding='same')
        self.conv1_2 = layers.Conv1D(self.n[0]/4, self.k[1], kernel_initializer='glorot_normal', padding='same')
        self.conv1_3 = layers.Conv1D(self.n[0]/4, self.k[2], kernel_initializer='glorot_normal', padding='same')
        self.conv1_4 = layers.Conv1D(self.n[0]/4, self.k[3], kernel_initializer='glorot_normal', padding='same')
        self.conv1_5 = layers.Conv1D(self.n[0]/4, self.k[4], kernel_initializer='glorot_normal', padding='same')
        self.batch1 = layers.BatchNormalization()
        self.activation1 = layers.Activation(tf.nn.leaky_relu)
        self.pool1 = layers.MaxPooling1D(2)
        
        self.conv2 = layers.Conv1D(self.n[1], 1, kernel_initializer='glorot_normal', padding='valid')
        self.batch2 = layers.BatchNormalization()
        self.activation2 = layers.Activation(tf.nn.leaky_relu)
        self.pool2 = layers.MaxPooling1D(2)
        
        self.conv3 = layers.Conv1D(self.n[2], 1, kernel_initializer='glorot_normal', padding='valid')
        self.batch3 = layers.BatchNormalization()
        self.activation3 = layers.Activation(tf.nn.leaky_relu)
        self.pool3 = layers.MaxPooling1D(2)
                                          
        self.conv4 = layers.Conv1D(self.n[3], 1, kernel_initializer='glorot_normal', padding='valid')
        self.batch4 = layers.BatchNormalization()
        self.activation4 = layers.Activation(tf.nn.leaky_relu)
        self.pool4 = layers.MaxPooling1D(2)
        
        self.output_conv = layers.Conv1D(self.n[4], 1, kernel_initializer='glorot_normal', padding='valid')
        self.gate = layers.Dense(1, activation=tf.nn.sigmoid)
        
    def call(self, inp):
        
        inp = tf.concat([self.conv1_1(inp), self.conv1_2(inp), self.conv1_3(inp), self.conv1_4(inp), self.conv1_5(inp)], -1)
        inp = self.pool1(self.activation1(self.batch1(inp)))
        
        inp = self.pool2(self.activation2(self.batch2(self.conv2(inp))))
        inp = self.pool3(self.activation3(self.batch3(self.conv3(inp))))
        inp = self.pool4(self.activation4(self.batch4(self.conv4(inp))))
        
        inp = self.gate(self.output_conv(inp))
        
        return inp
callbacks = tf.keras.callbacks.ReduceLROnPlateau(
    monitor='val_loss', factor=.5, patience=2, verbose=0, mode='min',
    min_delta=0.0001, cooldown=0, min_lr=0)

save = tf.keras.callbacks.ModelCheckpoint(
    BEST_PATH, monitor='val_loss', verbose=0,
    save_best_only=True, save_weights_only=True, mode='min', save_freq='epoch')

early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=20) 
with strategy.scope():
    opt = tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE, epsilon=EPSILON)
    model = ResNet1D()
    model.compile(optimizer=opt, loss='mae')
    model.fit(train_dataset, epochs=TRAINING_EPOCHS, validation_data=val_dataset,
                  verbose=1, callbacks=[callbacks, save, early_stop]) 
model.load_weights(BEST_PATH)
model.evaluate(val_dataset)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.142799919887843, 'fraction_numerical': 0.03424794712597637, 'mean_word_length': 4.256842105263158, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '159835', 'license': 'Apache-2.0', 'n_tokens_mistral': 1905, 'n_tokens_neox': 1826, 'n_words': 241, 'path': 'legacy/4_1D_GatedCNN-Copy2.ipynb', 'repo_name': 'ataraxno/weighing_dev', 'size': '4993'}