<jupyter_start><jupyter_text>Molecules<jupyter_code>from SCFInitialGuess.utilities.dataset import ScreenedData

target = "P"
basis = "6-311++g**"

data = ScreenedData(r_max=10)

data.include(data_path = "../../thesis/dataset/MethanT/", postfix = "MethanT", target=target)
from SCFInitialGuess.utilities.dataset import ScreenedData

target = "P"

data = ScreenedData(r_max=10)

data.include(data_path = "../../thesis/dataset/MethanT/", postfix = "MethanT", target=target)
data.include(data_path = "../../thesis/dataset/MethanT2/", postfix = "MethanT2", target=target)
data.include(data_path = "../../thesis/dataset/MethanT3/", postfix = "MethanT3", target=target)
data.include(data_path = "../../thesis/dataset/MethanT4/", postfix = "MethanT4", target=target)

data.include(data_path = "../../thesis/dataset/EthanT/",  postfix = "EthanT", target=target)
data.include(data_path = "../../thesis/dataset/EthanT2/", postfix = "EthanT2", target=target)
data.include(data_path = "../../thesis/dataset/EthanT3/", postfix = "EthanT3", target=target)
data.include(data_path = "../../thesis/dataset/EthanT4/",  postfix = "EthanT4", target=target)
data.include(data_path = "../../thesis/dataset/EthanT5/",  postfix = "EthanT5", target=target)
data.include(data_path = "../../thesis/dataset/EthanT6/",  postfix = "EthanT6", target=target)

data.include(data_path = "../../thesis/dataset/EthenT/",  postfix = "EthenT", target=target)
data.include(data_path = "../../thesis/dataset/EthenT2/", postfix = "EthenT2", target=target)
data.include(data_path = "../../thesis/dataset/EthenT3/", postfix = "EthenT3", target=target)
data.include(data_path = "../../thesis/dataset/EthenT4/",  postfix = "EthenT4", target=target)
data.include(data_path = "../../thesis/dataset/EthenT5/",  postfix = "EthenT5", target=target)
data.include(data_path = "../../thesis/dataset/EthenT6/",  postfix = "EthenT6", target=target)


data.include(data_path = "../../thesis/dataset/EthinT/",  postfix = "EthinT", target=target)
data.include(data_path = "../../thesis/dataset/EthinT2/", postfix = "EthinT2", target=target)
data.include(data_path = "../../thesis/dataset/EthinT3/", postfix = "EthinT3", target=target)

#data.include(data_path = "../../dataset/QM9/", postfix = "QM9-300")<jupyter_output><empty_output><jupyter_text>Descriptor<jupyter_code>from SCFInitialGuess.descriptors.high_level import AtomicNumberWeighted
from SCFInitialGuess.descriptors.cutoffs import BehlerCutoff1
from SCFInitialGuess.descriptors.models import RADIAL_GAUSSIAN_MODELS, make_uniform
from SCFInitialGuess.descriptors.coordinate_descriptors import \
    Gaussians, SPHAngularDescriptor
import pickle

model = make_uniform(25, 5, eta_max=60, eta_min=20)

descriptor_C = AtomicNumberWeighted(
    Gaussians(*model),
    SPHAngularDescriptor(4),
    BehlerCutoff1(5)
)


pickle.dump(descriptor_C, open(model_path + "descriptor_C.dump", "wb"))
    
descriptor_C.radial_descriptor.number_of_descriptors, descriptor_C.angular_descriptor.number_of_descriptors, descriptor_C.number_of_descriptors
from SCFInitialGuess.descriptors.high_level import AtomicNumberWeighted
from SCFInitialGuess.descriptors.cutoffs import BehlerCutoff1
from SCFInitialGuess.descriptors.models import RADIAL_GAUSSIAN_MODELS, make_uniform
from SCFInitialGuess.descriptors.coordinate_descriptors import \
    Gaussians, SPHAngularDescriptor
import pickle

model = make_uniform(25, 5, eta_max=60, eta_min=20)

descriptor_H = AtomicNumberWeighted(
    Gaussians(*model),
    SPHAngularDescriptor(3),
    BehlerCutoff1(5)
)


pickle.dump(descriptor_H, open(model_path + "descriptor_H.dump", "wb"))
    
descriptor_H.radial_descriptor.number_of_descriptors, descriptor_H.angular_descriptor.number_of_descriptors, descriptor_H.number_of_descriptors<jupyter_output><empty_output><jupyter_text>Package Dataset<jupyter_code>from SCFInitialGuess.utilities.dataset import make_block_dataset, extract_HETERO_block_dataset_pairs

dataset = make_block_dataset(
    [descriptor_C, descriptor_H],
    data.molecules,
    data.T,
    ["C", "H"],
    extract_HETERO_block_dataset_pairs
)

np.save(model_path + "normalisation.npy", (dataset.x_mean, dataset.x_std))

len(dataset.training[0]), len(dataset.validation[0]), len(dataset.testing[0]), 
from SCFInitialGuess.utilities.constants import number_of_basis_functions as N_BASIS


dim_C = N_BASIS[basis]["C"]
dim_H = N_BASIS[basis]["H"]<jupyter_output><empty_output><jupyter_text>NN Utils<jupyter_code>#keras.backend.clear_session()

#activation = "elu"
#learning_rate = 1e-5

intializer = keras.initializers.TruncatedNormal(mean=0.0, stddev=0.01)

def make_model(
        structure, 
        input_dim, 
        output_dim,
        activation="elu", 
        learning_rate=1e-3
    ):

    model = keras.Sequential()

    # input layer
    model.add(keras.layers.Dense(
        structure[0], 
        activation=activation, 
        input_dim=input_dim, 
        kernel_initializer=intializer
    ))

    for layer in structure[1:]:
        model.add(keras.layers.Dense(
            layer, 
            activation=activation, 
            kernel_initializer=intializer, 
            #bias_initializer='zeros',
            kernel_regularizer=keras.regularizers.l2(5e-3)
        ))

    #output
    model.add(keras.layers.Dense(output_dim))

    model.compile(
        optimizer=keras.optimizers.Adam(learning_rate), 
        loss='MSE', 
        metrics=['mse']
    )
    
    return model



early_stopping = keras.callbacks.EarlyStopping(
    monitor="val_mean_squared_error", 
    min_delta=1e-8, 
    patience=20, 
    verbose=1
)

reduce_lr = keras.callbacks.ReduceLROnPlateau(
    monitor='val_mean_squared_error', 
    factor=0.1, 
    patience=3, 
    verbose=1, 
    mode='auto', 
    min_delta=1e-6, 
    cooldown=2, 
    min_lr=1e-10
)


epochs = 1000


def train_model(model, dataset, filepath=None, learning_rate=1e-4, log_dir=None):

    if not log_dir is None:
        tensorboard = keras.callbacks.TensorBoard(
            log_dir=log_dir, 
            histogram_freq=0, 
            batch_size=32, 
            #update_freq='epoch'
        )
    
    if not filepath is None:
        checkpoint = keras.callbacks.ModelCheckpoint(
            filepath, 
            monitor='val_mean_squared_error', 
            verbose=1, 
            save_best_only=True, 
            save_weights_only=False, 
            mode='auto', 
            period=1
        )
    
    fig = plt.figure()
    ax = fig.add_subplot(1,1,1)
    
    train, validation = [], []
    while True:
        keras.backend.set_value(model.optimizer.lr, learning_rate)
            
        history = model.fit(
            x = dataset.training[0],
            y = dataset.training[1],
            epochs=epochs,
            shuffle=True,
            validation_data=dataset.validation, 
            verbose=1, 
            callbacks=[
                early_stopping, 
                reduce_lr,
                checkpoint,
                #tensorboard
            ]
        )
            
        
        #error.append(model.evaluate(
        #    dataset.testing[0], 
        #    dataset.testing[1], 
        #    verbose=1
        #)[1])
    
    return error<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>dataset.training[0].shape, dataset.training[1].shape
descriptor_C.number_of_descriptors, descriptor_H.number_of_descriptors, dim_C * dim_H
structure = [200, 200]
keras.backend.clear_session()
model = make_model(
    structure=structure,
    input_dim=descriptor_C.number_of_descriptors + descriptor_H.number_of_descriptors,
    output_dim=dim_C * dim_H,
    
)

model.summary()
train_model(
    model, 
    dataset, 
    model_path + "model.h5",
    learning_rate=1e-3,
    #log_dir=None#"./logs/H/" + name + "_" + "x".join(list(map(str, structure))) + "_" + str(i)
)<jupyter_output>Train on 49416 samples, validate on 12408 samples
Epoch 1/1000
49416/49416 [==============================] - 4s 85us/step - loss: 4.8210e-04 - mean_squared_error: 3.5001e-04 - val_loss: 4.4461e-04 - val_mean_squared_error: 3.1651e-04

Epoch 00001: val_mean_squared_error improved from inf to 0.00032, saving model to ../../thesis/models/DescriptorHetero/CH/model.h5
Epoch 2/1000
49416/49416 [==============================] - 3s 69us/step - loss: 3.0434e-04 - mean_squared_error: 2.5791e-04 - val_loss: 3.1714e-04 - val_mean_squared_error: 2.7634e-04

Epoch 00002: val_mean_squared_error improved from 0.00032 to 0.00028, saving model to ../../thesis/models/DescriptorHetero/CH/model.h5
Epoch 3/1000
49416/49416 [==============================] - 3s 70us/step - loss: 2.7576e-04 - mean_squared_error: 2.2424e-04 - val_loss: 3.1802e-04 - val_mean_squared_error: 2.6742e-04

Epoch 00003: val_mean_squared_error improved from 0.00028 to 0.00027, saving model to ../../thesis/models/DescriptorHetero/CH/[...]
{'fraction_non_alphanumeric': 0.13998190863862506, 'fraction_numerical': 0.036635006784260515, 'mean_word_length': 3.554582904222451, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '266923', 'license': 'MIT', 'n_tokens_mistral': 3342, 'n_tokens_neox': 3099, 'n_words': 460, 'path': 'notebooks/ApplicationCarbohydrates/Traning_Hetero.ipynb', 'repo_name': 'jcartus/SCFInitialGuess', 'size': '8844'}