<jupyter_start><jupyter_text>Hyperparameters<jupyter_code>PRE_TRAINED = './models/pg_convnet.h5'
BEST_PATH = './models/pg_cv_transfer.h5'
TRAINING_EPOCHS = 200
LEARNING_RATE = 0.0015
EPSILON = 1e-08
BATCH_SIZE = 32<jupyter_output><empty_output><jupyter_text>Data preparation<jupyter_code>l = np.load('./results/pg_dataset.npz', allow_pickle=True)
data_indices_2020S = l['data_indices_2020S']
output_label_2020S = l['output_label_2020S']
data_indices_2020W = l['data_indices_2020W']
output_label_2020W = l['output_label_2020W']
OUTPUT_MAXS = l['OUTPUT_MAXS']
OUTPUT_MINS = l['OUTPUT_MINS']
COLUMNS = ['node_len', 'node_dia', 'plant_h', 'leaf_area']
output_2020S_df = pd.DataFrame(output_label_2020S, index=data_indices_2020S, columns=COLUMNS)
output_2020W_df = pd.DataFrame(output_label_2020W, index=data_indices_2020W, columns=COLUMNS)
DIRECTORY = './images/2020_W/'
dir_list = os.listdir(DIRECTORY)
dataset_list = [direc for direc in dir_list if direc.startswith('LAI_OVER') or direc.startswith('LAI2_OVER')]
dataset_list.sort()
data_indices = []
input_images = []
output_labels = []
for DATE in output_2020W_df.index:
    for DIRECTORY in dataset_list:
        if DATE in DIRECTORY:
            file_list = os.listdir(f'./images/2020_W/{DIRECTORY}')
            file_list = [file for file in file_list if file.endswith('.jpg')]
            for FILE in file_list:
                image = Image.open(f'./images/2020_W/{DIRECTORY}/{FILE}')
                data_indices.append(pd.Timestamp(DATE))
                input_images.append(img_to_array(image))
                output_labels.append(output_2020W_df.loc[DATE].values)
input_images = np.stack(input_images, axis=0)
output_labels = np.stack(output_labels, axis=0)
data_indices = np.array(data_indices)
print(len(data_indices))
print(input_images.shape)
print(output_labels.shape)
# data_indices, input_images, output_labels = resample(data_indices, input_images, output_labels, n_samples=5000, replace=False, random_state=4574)
print(len(data_indices))
print(input_images.shape)
print(output_labels.shape)
gc.collect()
data_indices, input_images, output_labels = shuffle(data_indices, input_images, output_labels, random_state=4574)
N_TRAIN = int(output_labels.shape[0]*.3)
train_input = input_images[:N_TRAIN, ...]
train_label = output_labels[:N_TRAIN, ...]
train_index = data_indices[:N_TRAIN]
test_input = input_images[N_TRAIN:, ...]
test_label = output_labels[N_TRAIN:, ...]
test_index = data_indices[N_TRAIN:]
datagen = ImageDataGenerator(
    rescale=1/255,
    width_shift_range=0.2,
    height_shift_range=0.2,
    shear_range=0.2,
    zoom_range=0.2,
    horizontal_flip=True,
    fill_mode='nearest')
train_iterator = datagen.flow(train_input, train_label, batch_size=BATCH_SIZE)
test_iterator = datagen.flow(test_input, test_label, batch_size=BATCH_SIZE)
gc.collect()<jupyter_output><empty_output><jupyter_text>Model construction<jupyter_code>with strategy.scope():
    model = Sequential([
        layers.Conv2D(32, 2, padding='same', activation='relu', input_shape=(128, 128, 3)),
        layers.MaxPooling2D(),
        layers.Conv2D(64, 2, padding='same', activation='relu'),
        layers.MaxPooling2D(),
        layers.Conv2D(128, 2, padding='same', activation='relu'),
        layers.MaxPooling2D(),
        layers.Conv2D(128, 3, padding='same', activation='relu'),
        layers.MaxPooling2D(),
        layers.Conv2D(256, 3, padding='same', activation='relu'),
        layers.MaxPooling2D(),
        layers.Conv2D(512, 3, padding='same', activation='relu'),
        layers.MaxPooling2D(),
        layers.Conv2D(1024, 3, padding='same', activation='relu'),
        layers.MaxPooling2D(),
        layers.Flatten(),
        layers.Dense(128, activation='relu'),
        layers.Dense(4, activation='sigmoid')
        ])
cbs = callbacks.ReduceLROnPlateau(
    monitor='val_loss', factor=.5, patience=2, verbose=0, mode='min',
    min_delta=0.0001, cooldown=0, min_lr=0)

save = callbacks.ModelCheckpoint(
    BEST_PATH, monitor='val_loss', verbose=0,
    save_best_only=True, save_weights_only=True, mode='min', save_freq='epoch')

early_stop = callbacks.EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=20) 
with strategy.scope():
    opt = optimizers.Adam(learning_rate=LEARNING_RATE, epsilon=EPSILON)
    model.compile(optimizer=opt, loss='mae')
    model.predict(test_iterator)
    model.load_weights(PRE_TRAINED)
model.trainable = False
with strategy.scope():
    base_model = Sequential(model.layers[:-2])
    prediction_layer1 = layers.Dense(128, activation='relu')
    prediction_layer2 = layers.Dense(4, activation='sigmoid')
with strategy.scope():
    new_model = Sequential([
        base_model,
        prediction_layer1,
        prediction_layer2
    ])
gc.collect()
with strategy.scope():
    opt = optimizers.Adam(learning_rate=LEARNING_RATE, epsilon=EPSILON)
    new_model.compile(optimizer=opt, loss='mae')
    new_model.fit(train_iterator, epochs=TRAINING_EPOCHS, validation_data=test_iterator,
                  verbose=1, callbacks=[cbs, save, early_stop]) 
new_model.load_weights(BEST_PATH)
pred_result = model.predict(test_input/255)
test_label = (OUTPUT_MAXS - OUTPUT_MINS)*test_label + OUTPUT_MINS
pred_result = (OUTPUT_MAXS - OUTPUT_MINS)*pred_result + OUTPUT_MINS
pred_df = pd.DataFrame(np.concatenate([test_label, pred_result], axis=1), index=test_index)
pred_df.columns = ['label_node_len', 'label_node_dia', 'label_plant_h', 'label_leaf_area', 'pred_node_len', 'pred_node_dia', 'pred_plant_h', 'pred_leaf_area']
pred_df.to_csv('./results/model_output/pg_transfer_pred_result.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13329773981135434, 'fraction_numerical': 0.036305392418579815, 'mean_word_length': 5.321709786276715, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '432950', 'license': 'Apache-2.0', 'n_tokens_mistral': 2130, 'n_tokens_neox': 1990, 'n_words': 291, 'path': 'legacy/5_pg_CNN_transfer-Copy1.ipynb', 'repo_name': 'ataraxno/weighing_dev', 'size': '5619'}