<jupyter_start><jupyter_text>Uses the reduction from the rl util to infer human preferences. **Run the export_activations_reductions.ipynb notebook before this one** to get the reduction saved in your drive.<jupyter_code>import joblib
import os
import itertools
import random
import time
from sklearn.utils import shuffle
import numpy as np
import sys

%tensorflow_version 2.x
import tensorflow as tf
from google.colab import drive
drive.mount('/content/drive')
file_path = '/content/drive/My Drive/pref_extract_files/'
ys = joblib.load(os.path.join(file_path, 'ys.pkl'))
observations = joblib.load(os.path.join(file_path, 'observations.pkl'))
dense_activations = joblib.load(os.path.join(file_path, 'dense_activations.pkl'))
conv_activations = joblib.load(os.path.join(file_path, 'conv_activations.pkl'))
conv_reduced = joblib.load(os.path.join(file_path, 'reduced_train_attri_opt_act.pkl'))
def normalise(x):
  x -= np.mean(x)
  x /= np.std(x)
  return x

observations = normalise(observations)
dense_activations = normalise(dense_activations)
conv_activations = normalise(conv_activations)
conv_reduced = normalise(conv_reduced)
print(dense_activations.shape)

# Change for different training data
img_like_for_x = conv_reduced
# img_like_for_x = conv_activations
print(img_like_for_x.shape)

xs = []
for img_like, dense_activation in zip(img_like_for_x, dense_activations):
  xs.append([img_like, dense_activation])
def get_val_auc(logs):
      for key in logs:
        if key.startswith('val_auc'):
          return logs[key]

class BestStats(tf.keras.callbacks.Callback):
  """A callback to keep track of the best val accuracy and auc seen so far."""
  def on_train_begin(self, logs):
      self.bestMetric = -float('inf')
      self.bestLogs = None
      self.bestTrain = -float('inf')
      self.num_epochs = 0

  def on_epoch_end(self, epoch, logs):
    self.num_epochs += 1
    self.bestTrain = max(self.bestTrain, logs.get('accuracy'))

    val_accuracy = logs.get('val_accuracy')
    if val_accuracy == None:
      return 

    val_auc = get_val_auc(logs)
    
    metric = (val_accuracy + val_auc) / 2.0

    if metric > self.bestMetric:
      self.bestMetric = metric
      self.bestLogs = logs
def get_model(reg_amount, drop_rate, do_ave_pool, conv_layer_params, fc_layer_sizes, concat_top_layer, reduction_alg, n_components):
  del reduction_alg, n_components
  """
  Args:
    conv_layer_params: Optional list of convolution layers parameters, where
      each item is either a length-three tuple indicating
      `(filters, kernel_size, stride)`.
      """

  conv_model = tf.keras.models.Sequential()

  for config in conv_layer_params:
    (filters, kernel_size, strides) = config
    conv_model.add(
        tf.keras.layers.Conv2D(
            filters=filters,
            kernel_size=kernel_size,
            strides=strides,
            activation='relu',
            ))
  if do_ave_pool:
    conv_model.add(tf.keras.layers.GlobalAveragePooling2D())
  conv_model.add(tf.keras.layers.Flatten())

  dense_model = tf.keras.models.Sequential(name='output')
  for layer_size in fc_layer_sizes:
    dense_model.add(tf.keras.layers.Dense(layer_size, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(reg_amount)))
    dense_model.add(tf.keras.layers.Dropout(drop_rate))
  dense_model.add(tf.keras.layers.Dense(1, activation='sigmoid'))

  img_input = tf.keras.layers.Input(shape=img_like_for_x.shape[1:], name='img_input')
  dense_input = tf.keras.layers.Input(shape=dense_activations.shape[1:], name='dense_input')

  if concat_top_layer:
    final_img_input = img_input
  else:
    # Tile the dense input to be the same size as img_input, and concat them.
    dense_input_expand = tf.keras.layers.Reshape([1, 1, dense_input.shape[-1]])(dense_input)
    dense_input_tile = tf.keras.backend.tile(dense_input_expand, [1, img_input.shape[1], img_input.shape[2], 1])
    final_img_input = tf.keras.layers.concatenate([img_input, dense_input_tile])

  conv_output = conv_model(final_img_input)
  
  if concat_top_layer:
    # Concatenate the dense_input to the output of the conv model.
    final_dense_input = tf.keras.layers.concatenate([conv_output, dense_input])
  else:
    final_dense_input = conv_output
    
  output = dense_model(final_dense_input)

  model = tf.keras.models.Model(inputs=[img_input, dense_input], outputs=[output])

  model.compile(optimizer=tf.keras.optimizers.Adam(.01),
                loss='binary_crossentropy',
                metrics=['accuracy', tf.keras.metrics.AUC()],
                )
  return model

all_hparam_possibilities = [
    {"concat_top_layer": [0, 1], "do_ave_pool": [0, 1], "conv_layer_params": [((16, 3, 1),),  ((32, 3, 1), (16, 3, 2))], "reg_amount": [.1, .5, 1], "drop_rate": [0.0, .2, .4], "fc_layer_sizes": [(16,), (32,16),], 'reduction_alg': [None], 'n_components': [None]},
]

use_activations = False
num_train = 50
num_val = 1000
epochs = 400
print("use_activations:", use_activations, "num_train:", num_train, "epochs", epochs)
if num_train > 50:
  print("More than 50 train data!!!!!!!!")

# each item in all_hparam_possibilities specifies valid hyper params to try. Put parameters that don't make sense together in separate lists.

hparam_combinations = []
for hparam_possibilities in all_hparam_possibilities:
  hparam_keys, hparam_values = zip(*hparam_possibilities.items())
  hparam_combinations.extend([dict(zip(hparam_keys, v)) for v in itertools.product(*hparam_values)])
random.shuffle(hparam_combinations)
print("len(hparam_combinations)", len(hparam_combinations), "hparam_combinations", hparam_combinations)

def modify_x_for_reduce(xs):
  reshaped_x = np.reshape(xs, [xs.shape[0], -1])
  # Make everything positive because some reductions don't work with negatives.
  reshaped_x -= np.min(reshaped_x)
  return reshaped_x

def unsup_exstract(xs, reg_amount, drop_rate, layer_sizes, reduction_alg, n_components):
  del reg_amount, drop_rate, layer_sizes

  print("Using unsupervised feature extraction.")

  dim_reduct_model = ChannelReducer(reduction_alg=reduction_alg, n_components=n_components)
  xs = dim_reduct_model.fit_transform(modify_x_for_reduce(xs))
  return xs

def convert_np(lst, i):
  result = []
  for x in lst:
    result.append(x[i])
  return np.asarray(result)

def convert_xs(xs):
  return {'img_input': convert_np(xs, 0), 'dense_input': convert_np(xs, 1)}

def train_best_logs(xs, ys, num_val, do_summary, hparams, get_model):
  """Trains the model and retruns the logs of the best epoch. randomly splits the train and val data before training."""
  tf.keras.backend.clear_session()
  model = get_model(**hparams)
  xs, ys = shuffle(xs, ys)

  xs_val = xs[num_train:num_train+num_val]
  ys_val = ys[num_train:num_train+num_val]
  early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=30, verbose=0)
  best_stats = BestStats()
  model.fit(convert_xs(xs[:num_train]), ys[:num_train], epochs=epochs, batch_size=256, validation_freq=1, callbacks=[best_stats, early_stopping], validation_data=(convert_xs(xs_val), ys_val), verbose=0)
  if do_summary:
    model.summary()
    print("best train accuracy:", best_stats.bestTrain)
    print("Number of epochs:", best_stats.num_epochs)
  return best_stats.bestLogs

def multiple_train_ave(hparams):
  """Trains the model multiple times with the same parameters and returns the average metrics"""
  start = time.time()
  all_val_auc = []
  all_val_accuracy = []

  if hparams['reduction_alg'] != None:
    xs_for_train = unsup_exstract(xs, **hparams)
  else:
    xs_for_train = xs

  do_summary = True
  for i in range(5):
    logs = train_best_logs(xs_for_train, ys, num_val, do_summary, hparams, get_model)
    all_val_auc.append(get_val_auc(logs))
    all_val_accuracy.append(logs.get('val_accuracy'))
    do_summary = False 

  mean_val_auc = np.mean(all_val_auc)
  mean_val_accuracy = np.mean(all_val_accuracy)
  metric = (mean_val_auc + mean_val_accuracy) / 2.0
  print_data = ("mean_val_auc", mean_val_auc, "mean_val_accuracy", mean_val_accuracy, "metric", metric, "val_auc_std", np.std(all_val_auc), "val_accuracy_std", np.std(all_val_accuracy))

  end = time.time()
  print("Seconds per hyperparam config", end - start)
  # GPU: ('Seconds per hyperparam config', 16.970870971679688)

  return metric, print_data

best_metric = -float('inf')

run_num = 0
for hparams in hparam_combinations:
  print("hparams", hparams)

  metric, print_data = multiple_train_ave(hparams)

  print(print_data)
  if metric > best_metric:
    best_metric = metric
    best_print_data = print_data
    best_hparams = hparams

  run_num += 1
  print("fract done", run_num/float(len(hparam_combinations)))
  print
  print("==============================================================================================")
  print
  sys.stdout.flush()
print("best_hparams", best_hparams)
print("best results", best_print_data)
print("Retraining on the best_hparams to make sure we didn't just get good results by random chance.")

_, print_data = multiple_train_ave(best_hparams)
print("Result of retrain on the best hyperparameters", print_data)<jupyter_output>best_hparams {'concat_top_layer': 0, 'do_ave_pool': 1, 'conv_layer_params': ((16, 3, 1),), 'reg_amount': 0.5, 'drop_rate': 0.2, 'fc_layer_sizes': (32, 16), 'reduction_alg': None, 'n_components': None}
best results ('mean_val_auc', 0.9142109990119934, 'mean_val_accuracy', 0.8658000111579895, 'metric', 0.8900055050849915, 'val_auc_std', 0.011914377924630034, 'val_accuracy_std', 0.014743140828623479)
Retraining on the best_hparams to make sure we didn't just get good results by random chance.
Model: "model"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
dense_input (InputLayer)        [(None, 64)]         0                                            
______________________________________________________________________________________________[...]
{'fraction_non_alphanumeric': 0.1315893385982231, 'fraction_numerical': 0.01994076999012833, 'mean_word_length': 4.900407687827606, 'pattern_counts': {'":': 6, '<': 4, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '431795', 'license': 'Apache-2.0', 'n_tokens_mistral': 3596, 'n_tokens_neox': 3364, 'n_words': 738, 'path': 'pref_extract_more_inputs.ipynb', 'repo_name': 'arunraja-hub/Preference_Extraction', 'size': '10130'}