<jupyter_start><jupyter_text>1. Cross-Validation Tuning and Train  Import data and preprocessThis dataset contains all the literature data + 87 experimental datapoints<jupyter_code>with open('./full_data.json', 'r') as file:
    data = pd.read_json(json.load(file), orient='table')

data<jupyter_output><empty_output><jupyter_text>CV Tuning with XGBoost<jupyter_code>inputs = np.asarray(data.loc[:,'Units of A':'M'])
labels = np.asarray(data['Cloud Point'])

scaler_regressor = MinMaxScaler()
inputs = scaler_regressor.fit_transform(inputs)

inputs_train, inputs_valid, labels_train, labels_valid = train_test_split(
    inputs, labels, test_size=0.1, random_state=seed)
XGB_Options = { 
                'cv':               10,
                'scoring':          'neg_mean_squared_error',
                'seed':             seed, 
                'max_depth':        np.arange(2,14,2),
                'min_child_weight': np.arange(1,8,1),
                'n_estimators':     np.arange(10,80,5),

                'gamma':            np.arange(0.05,0.45,0.05),
                'colsample_bytree': np.arange(0.60, 0.95, 0.05),
                'subsample':        np.arange(0.60, 0.95, 0.05),
                'reg_alpha':        [1e-5, 1e-2, 0.1, 0.5, 1, 5, 10], #alpha 
                'reg_lambda':       [1e-5, 1e-2, 0.1, 0.5, 1, 5, 10],#lambda
                'learning_rate':    np.arange(0.025,0.150,0.025),
                'scaler':           scaler_regressor,
                'n_jobs':           4,
                'verbose':          1             
              }
trained_regressor =  predictor.XGB_Regressor(options=XGB_Options)
trained_regressor.fit(inputs_train, labels_train)
tuned_regressor = trained_regressor.regressor
n_splits = 3
np.random.seed(seed)

targets = []
results = []

for n in range(n_splits):
    inputs_train, inputs_test, labels_train, labels_test = train_test_split(inputs, labels, test_size=0.1)
    tuned_regressor.fit(inputs_train, labels_train)
    pred_train = tuned_regressor.predict(inputs_train)
    pred_test = tuned_regressor.predict(inputs_test)
    results.append({'train': pred_train, 'test': pred_test, 'name': 'Split {}'.format(n+1)})
    targets.append({'train': labels_train, 'test': labels_test})
plot_comparisons(targets, results)<jupyter_output><empty_output><jupyter_text>2. Train NN as Secondary Validation<jupyter_code>import tensorflow as tf

tf.reset_default_graph()
np.random.seed(seed)
tf.set_random_seed(seed) 
sess = tf.Session()
NUM_NN = 10
regressors = []

for n in range(NUM_NN):
    options = {'input_shape': [6, ], 'target_shape': [1, ],
               'n_layers': 2, 'layer_widths': [64, 128], 
               'name': 'nn_{}'.format(n), 'session': sess}
    regressors.append(predictor.NeuralNetRegressor(options=options))

ensb_regressor = predictor.EnsembleRegressor(regressors, scaler_regressor)
options={'monitor': False,
         'reinitialize': True,
         'n_iter': 10000}
list_of_options = [options,]*NUM_NN
ensb_regressor.fit(inputs_train, labels_train.reshape(-1, 1), list_of_options)
predicted_train = ensb_regressor.predict_mean(inputs_train)
predicted_validation = ensb_regressor.predict_mean(inputs_valid)

train_rmse = rmse(predicted_train, labels_train)
validation_rmse = rmse(predicted_validation, labels_valid)

print('Train RMSE: ', train_rmse, 'validation RMSE: ', validation_rmse)<jupyter_output>Train RMSE:  2.1540056388662414 validation RMSE:  1.576115105091678<jupyter_text>Train all data<jupyter_code>ensb_regressor.fit(inputs, labels.reshape(-1,1), list_of_options)

predicted_nn = ensb_regressor.predict_mean(inputs)
std_nn = np.sqrt(ensb_regressor.predict_covariance(inputs))
rmse_nn = rmse(predicted_nn, labels)<jupyter_output>Fitting Regressor: nn_0
Fitting Regressor: nn_1
Fitting Regressor: nn_2
Fitting Regressor: nn_3
Fitting Regressor: nn_4
Fitting Regressor: nn_5
Fitting Regressor: nn_6
Fitting Regressor: nn_7
Fitting Regressor: nn_8
Fitting Regressor: nn_9<jupyter_text>3. Inverse Design  Train model with tuned parameters on the full dataset<jupyter_code>trained_regressor.regressor.fit(inputs, labels)
predicted = trained_regressor.predict(inputs)
print('Train RMSE: ', rmse(predicted, labels))<jupyter_output>Train RMSE:  2.573435823938268<jupyter_text>Constraint and Target SpecificationsWe solve the problem$$    \min_{x} \mathrm{objective}(x, CP)$$subject to$$    \mathrm{constraints}(x) \geq 0$$Some soft constraints are written into the objective function as a regularizer, and some hard constraints are moved to the selection criterion function, which must evaluates to true before we consider the solution admissiable. This is to maximize efficiency of the particle swarm method.<jupyter_code>lb = np.array([0]*6)
ub = np.array([203, 187, 43, 96, 0.1, 23196])

def objective(x, target_CP, regressor):
    '''
    loss function:
        The loss function contains 2 parts
            1. 0.5*(predicted_CP(x) - target_CP)**2
            2. coef * regularizer
    '''
    
    def regularizer(x):
        '''
        Regularizer (or soft constraints):
            We want to drive the system towards
            binary systems involving A,B,C,D
            Here, we use the simple penalizer
                \sum_{(i,j,k) \in (A,B,C,D)} i*j*k

            For task 1:
                We also want to minimize A
        '''
        reg = 0.0
        for c in combinations(x[:4], 3):
            reg += np.prod(c)**(1.0/3.0)

        reg += x[0]  # minimize A
        return reg

    
    return 0.5 * (regressor.predict_transform(x) - target_CP)**2 \
        + 1e-1 * regularizer(x)


def constraints(x, *args):
    ''' hard constraints '''
    Final_M    = np.dot(x[:4], [99.13, 113.16, 111.14, 113.16])
    cons_Mp_ub = [Final_M * 1.2 - x[5]]
    cons_Mp_lb = [x[5] - Final_M * 0.8]
    return np.asarray(cons_Mp_ub + cons_Mp_lb) 

def selection_criteria(x):
    """
    Selection criteria:
        Desired solution x should evaluates to all
        non-negative
            1. Minimum non-zero component must be at least 10%
               of the max component. (note: non-zero is defined
               to be <= ZERO_CUTOFF, purely for numerical stability)
            2. Allow for at most ternary systems
    """
    ZERO_CUTOFF    = 0.1    

    # Criterion 1
    xcomps         = x[:4]
    x_non_zero     = xcomps[xcomps > ZERO_CUTOFF]
    x_max          = xcomps.max()
    x_min          = x_non_zero.min()
    comp_condition = [x_min - 0.1*x_max]

    # Criterion 2
    num_non_zero      = np.sum(x[:4] > ZERO_CUTOFF)
    nonzero_condition = [3.1 - num_non_zero]
    # return np.asarray([1.0])
        
    return all([c >= 0 for c in comp_condition + nonzero_condition])<jupyter_output><empty_output><jupyter_text>Optimze via Particle Swarm OptimizerThe optimisation options are found using a systematic parameter study that we omit here for brevity<jupyter_code>opt = optimizer.PSO_Optimizer(
    regressor=trained_regressor,
    scaler=scaler_regressor,
    objective=objective,
    constraints=constraints,
    selection_criteria=selection_criteria
)

optimisation_options = { 'criteria': 0.1,
                        'max_rounds': 200,
                        'n_solutions': 50,
                        'lb':lb, 'ub':ub, 
                        'opt_vars': [ 'Units of A', 'Units of B', 'Units of C', 'Units of D', 'Units of E', 'M'],
                        'swarmsize': 400, 'omega':0.1, 'phip':0.9, 'phig': 0.8, 'maxiter': 100, 
                        'debug_flag': False,
                        'nprocessors': 12
                       }<jupyter_output><empty_output><jupyter_text>1. Target = 37$^{\circ}$C<jupyter_code>optimisation_options['target_CP'] = 37
results37, runtime  =  opt.optimisation_parallel(optimisation_options)
results37<jupyter_output>Optimisation sequence started at time:
2018-11-16 02:03:55
Working on round: 1
Optimization No.: 1
Optimization No.: 2
Optimization No.: 3
Optimization No.: 4
Optimization No.: 5
Optimization No.: 6
Optimization No.: 7
Optimization No.: 8
Optimization No.: 9
Optimization No.: 10
Optimization No.: 12
Optimization No.: 11
Stopping search: Swarm best objective change less than 1e-08
Stopping search: Swarm best objective change less than 1e-08
Stopping search: Swarm best objective change less than 1e-08
Stopping search: Swarm best objective change less than 1e-08
Stopping search: Swarm best objective change less than 1e-08
Stopping search: Swarm best objective change less than 1e-08
Stopping search: Swarm best objective change less than 1e-08
Stopping search: maximum iterations reached --> 100
Stopping search: maximum iterations reached --> 100
Stopping search: maximum iterations reached --> 100
Stopping search: maximum iterations reached --> 100
Stopping search: maximum iterations reached[...]<jupyter_text>2. Target = 45$^{\circ}$C<jupyter_code>optimisation_options['target_CP'] = 45
results45, runtime  =  opt.optimisation_parallel(optimisation_options)
results45<jupyter_output>Optimisation sequence started at time:
2018-11-16 02:04:59
Working on round: 1
Optimization No.: 1
Optimization No.: 2
Optimization No.: 3
Optimization No.: 4
Optimization No.: 5
Optimization No.: 6
Optimization No.: 7
Optimization No.: 8
Optimization No.: 9
Optimization No.: 10
Optimization No.: 12
Optimization No.: 11
Stopping search: Swarm best objective change less than 1e-08
Stopping search: Swarm best objective change less than 1e-08
Stopping search: Swarm best objective change less than 1e-08
Stopping search: Swarm best objective change less than 1e-08
Stopping search: Swarm best objective change less than 1e-08
Stopping search: Swarm best objective change less than 1e-08
Stopping search: maximum iterations reached --> 100
Stopping search: maximum iterations reached --> 100
Stopping search: maximum iterations reached --> 100
Stopping search: maximum iterations reached --> 100
Stopping search: maximum iterations reached --> 100
Stopping search: maximum iterations reached --> 100
[...]<jupyter_text>3. Target = 60$^{\circ}$C<jupyter_code>optimisation_options['target_CP'] = 60

results60, runtime  =  opt.optimisation_parallel(optimisation_options)
results60<jupyter_output>Optimisation sequence started at time:
2018-11-16 02:06:29
Working on round: 1
Optimization No.: 1
Optimization No.: 2
Optimization No.: 3
Optimization No.: 4
Optimization No.: 5
Optimization No.: 6
Optimization No.: 7
Optimization No.: 8
Optimization No.: 9
Optimization No.: 10
Optimization No.: 11
Optimization No.: 12
Stopping search: Swarm best objective change less than 1e-08
Stopping search: Swarm best objective change less than 1e-08
Stopping search: Swarm best objective change less than 1e-08
Stopping search: Swarm best objective change less than 1e-08
Stopping search: Swarm best objective change less than 1e-08
Stopping search: Swarm best objective change less than 1e-08
Stopping search: Swarm best objective change less than 1e-08
Stopping search: Swarm best objective change less than 1e-08
Stopping search: Swarm best objective change less than 1e-08
Stopping search: maximum iterations reached --> 100
Stopping search: maximum iterations reached --> 100
Stopping search: maximum [...]<jupyter_text>4. Target = 80$^{\circ}$C<jupyter_code>optimisation_options['target_CP'] = 80
results80, runtime  =  opt.optimisation_parallel(optimisation_options)
results80<jupyter_output>Optimisation sequence started at time:
2018-11-16 02:08:18
Working on round: 1
Optimization No.: 1
Optimization No.: 2
Optimization No.: 3
Optimization No.: 4
Optimization No.: 5
Optimization No.: 6
Optimization No.: 7
Optimization No.: 8
Optimization No.: 10
Optimization No.: 9
Optimization No.: 11
Optimization No.: 12
Stopping search: Swarm best objective change less than 1e-08
Stopping search: Swarm best objective change less than 1e-08
Stopping search: Swarm best objective change less than 1e-08
Stopping search: Swarm best objective change less than 1e-08
Stopping search: Swarm best objective change less than 1e-08
Stopping search: Swarm best objective change less than 1e-08
Stopping search: Swarm best objective change less than 1e-08
Stopping search: maximum iterations reached --> 100
Stopping search: Swarm best objective change less than 1e-08
Stopping search: Swarm best objective change less than 1e-08
Stopping search: Swarm best objective change less than 1e-08
Stopping search:[...]<jupyter_text>NN validation of the XGB results<jupyter_code>def NN_Mean_Std(input_df, scaler, ensb_reg):
    
    data = deepcopy(input_df)
    inputs   = scaler_regressor.transform(data.iloc[:, :6 ].values)
    
    mean = ensb_regressor.predict_mean(inputs)
    cov  = ensb_regressor.predict_covariance(inputs)
    std  = np.sqrt(cov.squeeze())
    
    data['Mean_Pred_NN'] = mean
    data['Std_Error_NN']  = std
    
    data['%Error_Mean_Pred_NN'] = 100*(data.Mean_Pred_NN - data.target_CP)/data.target_CP
    
    
    return data
NN_validation37 = NN_Mean_Std(results37, scaler_regressor, ensb_regressor)
NN_validation37
NN_validation45 = NN_Mean_Std(results45, scaler_regressor, ensb_regressor)
NN_validation45
NN_validation60 = NN_Mean_Std(results60, scaler_regressor, ensb_regressor)
NN_validation60
NN_validation80 = NN_Mean_Std(results80, scaler_regressor, ensb_regressor)
NN_validation80<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10010521569216894, 'fraction_numerical': 0.04449120697429731, 'mean_word_length': 3.678973277074543, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 58, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '406317', 'license': 'MIT', 'n_tokens_mistral': 4798, 'n_tokens_neox': 4205, 'n_words': 1209, 'path': 'inverse_design.ipynb', 'repo_name': 'InHouse-Banana/CloudPoint-MachineLearning', 'size': '13306'}