# Path: disentanglement_lib/evaluation/metrics/dci.py
# def compute_dci(ground_truth_data, representation_function, random_state,
#                 artifact_dir=None,
#                 num_train=gin.REQUIRED,
#                 num_test=gin.REQUIRED,
#                 batch_size=16):
# def _compute_dci(mus_train, ys_train, mus_test, ys_test):
# def compute_dci_on_fixed_data(observations, labels, representation_function,
#                               train_percentage=gin.REQUIRED, batch_size=100):
# def compute_importance_gbt(x_train, y_train, x_test, y_test):
# def disentanglement_per_code(importance_matrix):
# def disentanglement(importance_matrix):
# def completeness_per_factor(importance_matrix):
# def completeness(importance_matrix):
# 
# Path: disentanglement_lib/evaluation/metrics/utils.py
# def generate_batch_factor_code(ground_truth_data, representation_function,
#                                num_points, random_state, batch_size):
# def split_train_test(observations, train_percentage):
# def obtain_representation(observations, representation_function, batch_size):
# def discrete_mutual_info(mus, ys):
# def discrete_entropy(ys):
# def make_discretizer(target, num_bins=gin.REQUIRED,
#                      discretizer_fn=gin.REQUIRED):
# def _histogram_discretize(target, num_bins=gin.REQUIRED):
# def normalize_data(data, mean=None, stddev=None):
# def make_predictor_fn(predictor_fn=gin.REQUIRED):
# def logistic_regression_cv():
# def gradient_boosting_classifier():


from disentanglement_lib.evaluation.metrics import dci
from disentanglement_lib.evaluation.metrics import utils
from six.moves import range
import numpy as np
import gin.tf
  Returns:
    Dictionary with scores.
  """
  del artifact_dir
  scores = {}
  # Loop on different sizes of the training 'batch', as specified with gin.
  for train_size in num_train:
    size_string = str(train_size)
    mus_train, ys_train = utils.generate_batch_factor_code(
        ground_truth_data, representation_function, train_size, random_state,
        batch_size)
    mus_test, ys_test = utils.generate_batch_factor_code(
        ground_truth_data, representation_function, num_test, random_state,
        batch_size)
    # Create variables for aggregated scores.
    reduced_factor_train_scores = []
    other_factors_train_scores = []
    reduced_factor_test_scores = []
    other_factors_test_scores = []
    # Compute the reduced representation and test it for each factor of
    # variation.
    for factor_of_interest in range(ground_truth_data.num_factors):
      # Copy the training data and eliminate the k most informative factors.
      reduced_mus_train = mus_train.copy()
      reduced_mus_test = mus_test.copy()
      for _ in range(num_factors_to_remove):
        reduced_mus_train, reduced_mus_test =\
          compute_reduced_representation(reduced_mus_train, ys_train,
                                         reduced_mus_test, ys_test,
                                         factor_of_interest)

# Path: disentanglement_lib/evaluation/metrics/dci.py
# def compute_dci(ground_truth_data, representation_function, random_state,
#                 artifact_dir=None,
#                 num_train=gin.REQUIRED,
#                 num_test=gin.REQUIRED,
#                 batch_size=16):
# def _compute_dci(mus_train, ys_train, mus_test, ys_test):
# def compute_dci_on_fixed_data(observations, labels, representation_function,
#                               train_percentage=gin.REQUIRED, batch_size=100):
# def compute_importance_gbt(x_train, y_train, x_test, y_test):
# def disentanglement_per_code(importance_matrix):
# def disentanglement(importance_matrix):
# def completeness_per_factor(importance_matrix):
# def completeness(importance_matrix):
# 
# Path: disentanglement_lib/evaluation/metrics/utils.py
# def generate_batch_factor_code(ground_truth_data, representation_function,
#                                num_points, random_state, batch_size):
# def split_train_test(observations, train_percentage):
# def obtain_representation(observations, representation_function, batch_size):
# def discrete_mutual_info(mus, ys):
# def discrete_entropy(ys):
# def make_discretizer(target, num_bins=gin.REQUIRED,
#                      discretizer_fn=gin.REQUIRED):
# def _histogram_discretize(target, num_bins=gin.REQUIRED):
# def normalize_data(data, mean=None, stddev=None):
# def make_predictor_fn(predictor_fn=gin.REQUIRED):
# def logistic_regression_cv():
# def gradient_boosting_classifier():


# Path: disentanglement_lib/evaluation/metrics/reduced_downstream_task.py
from disentanglement_lib.evaluation.metrics import dci
from disentanglement_lib.evaluation.metrics import utils
from six.moves import range
import numpy as np
import gin.tf

  Returns:
    Dictionary with scores.
  """
  del artifact_dir
  scores = {}
  # Loop on different sizes of the training 'batch', as specified with gin.
  for train_size in num_train:
    size_string = str(train_size)
    mus_train, ys_train = utils.generate_batch_factor_code(
        ground_truth_data, representation_function, train_size, random_state,
        batch_size)
    mus_test, ys_test = utils.generate_batch_factor_code(
        ground_truth_data, representation_function, num_test, random_state,
        batch_size)
    # Create variables for aggregated scores.
    reduced_factor_train_scores = []
    other_factors_train_scores = []
    reduced_factor_test_scores = []
    other_factors_test_scores = []
    # Compute the reduced representation and test it for each factor of
    # variation.
    for factor_of_interest in range(ground_truth_data.num_factors):
      # Copy the training data and eliminate the k most informative factors.
      reduced_mus_train = mus_train.copy()
      reduced_mus_test = mus_test.copy()
      for _ in range(num_factors_to_remove):
        reduced_mus_train, reduced_mus_test =\
          compute_reduced_representation(reduced_mus_train, ys_train,
                                         reduced_mus_test, ys_test,
                                         factor_of_interest)

      predictor_model = utils.make_predictor_fn()