# Path: crowdastro/crowd/util.py
# def balanced_accuracy(y_true, y_pred):
#     """Computes the balanced accuracy of a predictor.
# 
#     y_true: (n_examples,) array of true labels.
#     y_pred: (n_examples,) (masked) array of predicted labels.
#     -> float or None (if the balanced accuracy isn't defined).
#     """
#     if hasattr(y_pred, 'mask') and not isinstance(y_pred.mask, bool):
#         cm = sklearn.metrics.confusion_matrix(
#                 y_true[~y_pred.mask], y_pred[~y_pred.mask]).astype(float)
#     else:
#         cm = sklearn.metrics.confusion_matrix(y_true, y_pred).astype(float)
# 
#     tp = cm[1, 1]
#     n, p = cm.sum(axis=1)
#     tn = cm[0, 0]
#     if not n or not p:
#         return None
# 
#     ba = (tp / p + tn / n) / 2
#     return ba
# 
# Path: crowdastro/active_learning/random_sampler.py
# class RandomSampler(Sampler):
# class BalancedSampler(RandomSampler):
#     def sample_index(self):
#     def sample_indices(self, n):
#     def sample_index(self):
#     def sample_indices(self, n):
# 
# Path: crowdastro/active_learning/qbc_sampler.py
# class QBCSampler(Sampler):
#     def __init__(self, pool, labels, Classifier, n_classifiers=20,
#                  subsample_rate=0.75, classifier_params=None):
#     def train(self):
#     def compute_disagreement(self):
#     def sample_index(self):
#     def sample_indices(self, n):
#     def retrain(self):
#     def score(self, test_xs, test_ts):
#     def ba(self, test_xs, test_ts):
# 
# Path: crowdastro/plot.py
# def fillbetween(xs, ys, facecolour='lightgreen', edgecolour='green',
#                 marker='x', facealpha=1.0, facekwargs=None, **kwargs):
#     """Plots a line plot with error represented by filled-between lines.
# 
#     xs: List of x values.
#     ys: List of lists of y values.
#     facecolour: Colour of the filled-between lines. Default light green.
#     edgecolour: Colour of the central line. Default green.
#     marker: Point marker. Default 'x'.
#     facealpha: Alpha value of filled section. Default 1.0.
#     facekwargs: Keyword arguments to pass to fill_between. Default {}.
#     """
#     facekwargs = facekwargs or {}
#     means = numpy.mean(ys, axis=1)
#     stds = numpy.std(ys, axis=1)
#     plt.plot(xs, means, color=edgecolour, marker=marker, **kwargs)
#     plt.fill_between(xs, means - stds, means + stds, color=facecolour,
#                      alpha=facealpha, linewidth=0, **facekwargs)
#     plt.fill_between(xs, means - stds, means + stds, facecolor='None',
#                      edgecolor=edgecolour, alpha=0.5, **facekwargs)


import argparse
import logging
import os.path
import h5py
import matplotlib
import matplotlib.pyplot as plt
import numpy
import sklearn.cross_validation
import sklearn.linear_model
from crowdastro.crowd.util import balanced_accuracy
from crowdastro.active_learning import random_sampler, qbc_sampler
from crowdastro.plot import fillbetween
2016
"""





def main(crowdastro_h5_path, training_h5_path, results_npy_path,
         overwrite=False, plot=False, n_trials=25):
    with h5py.File(crowdastro_h5_path, 'r') as crowdastro_h5:
        with h5py.File(training_h5_path, 'r') as training_h5:
            n_instances, = training_h5['labels'].shape
            n_splits, n_test_instances = crowdastro_h5[
                '/wise/cdfs/test_sets'].shape
            n_train_indices = n_instances - n_test_instances
            n_methods = 3  # QBC, Passive, Stratified Passive
            instance_counts = [int(i) for i in numpy.logspace(
                numpy.log10(100), numpy.log10(n_train_indices), n_trials)]
            results = numpy.zeros((n_methods, n_splits, n_trials))
            features = training_h5['features'].value
            labels = training_h5['labels'].value
            norris = crowdastro_h5['/wise/cdfs/norris_labels'].value

            if os.path.exists(results_npy_path):
                with open(results_npy_path, 'rb') as f:
                    logging.info('Loading from NPY file.')
                    results = numpy.load(f, allow_pickle=False)
            else:
                for method_index, Sampler in enumerate([
                        qbc_sampler.QBCSampler, random_sampler.RandomSampler,

# Path: crowdastro/crowd/util.py
# def balanced_accuracy(y_true, y_pred):
#     """Computes the balanced accuracy of a predictor.
# 
#     y_true: (n_examples,) array of true labels.
#     y_pred: (n_examples,) (masked) array of predicted labels.
#     -> float or None (if the balanced accuracy isn't defined).
#     """
#     if hasattr(y_pred, 'mask') and not isinstance(y_pred.mask, bool):
#         cm = sklearn.metrics.confusion_matrix(
#                 y_true[~y_pred.mask], y_pred[~y_pred.mask]).astype(float)
#     else:
#         cm = sklearn.metrics.confusion_matrix(y_true, y_pred).astype(float)
# 
#     tp = cm[1, 1]
#     n, p = cm.sum(axis=1)
#     tn = cm[0, 0]
#     if not n or not p:
#         return None
# 
#     ba = (tp / p + tn / n) / 2
#     return ba
# 
# Path: crowdastro/active_learning/random_sampler.py
# class RandomSampler(Sampler):
# class BalancedSampler(RandomSampler):
#     def sample_index(self):
#     def sample_indices(self, n):
#     def sample_index(self):
#     def sample_indices(self, n):
# 
# Path: crowdastro/active_learning/qbc_sampler.py
# class QBCSampler(Sampler):
#     def __init__(self, pool, labels, Classifier, n_classifiers=20,
#                  subsample_rate=0.75, classifier_params=None):
#     def train(self):
#     def compute_disagreement(self):
#     def sample_index(self):
#     def sample_indices(self, n):
#     def retrain(self):
#     def score(self, test_xs, test_ts):
#     def ba(self, test_xs, test_ts):
# 
# Path: crowdastro/plot.py
# def fillbetween(xs, ys, facecolour='lightgreen', edgecolour='green',
#                 marker='x', facealpha=1.0, facekwargs=None, **kwargs):
#     """Plots a line plot with error represented by filled-between lines.
# 
#     xs: List of x values.
#     ys: List of lists of y values.
#     facecolour: Colour of the filled-between lines. Default light green.
#     edgecolour: Colour of the central line. Default green.
#     marker: Point marker. Default 'x'.
#     facealpha: Alpha value of filled section. Default 1.0.
#     facekwargs: Keyword arguments to pass to fill_between. Default {}.
#     """
#     facekwargs = facekwargs or {}
#     means = numpy.mean(ys, axis=1)
#     stds = numpy.std(ys, axis=1)
#     plt.plot(xs, means, color=edgecolour, marker=marker, **kwargs)
#     plt.fill_between(xs, means - stds, means + stds, color=facecolour,
#                      alpha=facealpha, linewidth=0, **facekwargs)
#     plt.fill_between(xs, means - stds, means + stds, facecolor='None',
#                      edgecolor=edgecolour, alpha=0.5, **facekwargs)


# Path: crowdastro/experiment/experiment_rgz_qbc.py
import argparse
import logging
import os.path
import h5py
import matplotlib
import matplotlib.pyplot as plt
import numpy
import sklearn.cross_validation
import sklearn.linear_model
from crowdastro.crowd.util import balanced_accuracy
from crowdastro.active_learning import random_sampler, qbc_sampler
from crowdastro.plot import fillbetween

2016
"""





def main(crowdastro_h5_path, training_h5_path, results_npy_path,
         overwrite=False, plot=False, n_trials=25):
    with h5py.File(crowdastro_h5_path, 'r') as crowdastro_h5:
        with h5py.File(training_h5_path, 'r') as training_h5:
            n_instances, = training_h5['labels'].shape
            n_splits, n_test_instances = crowdastro_h5[
                '/wise/cdfs/test_sets'].shape
            n_train_indices = n_instances - n_test_instances
            n_methods = 3  # QBC, Passive, Stratified Passive
            instance_counts = [int(i) for i in numpy.logspace(
                numpy.log10(100), numpy.log10(n_train_indices), n_trials)]
            results = numpy.zeros((n_methods, n_splits, n_trials))
            features = training_h5['features'].value
            labels = training_h5['labels'].value
            norris = crowdastro_h5['/wise/cdfs/norris_labels'].value

            if os.path.exists(results_npy_path):
                with open(results_npy_path, 'rb') as f:
                    logging.info('Loading from NPY file.')
                    results = numpy.load(f, allow_pickle=False)
            else:
                for method_index, Sampler in enumerate([
                        qbc_sampler.QBCSampler, random_sampler.RandomSampler,

                        random_sampler.BalancedSampler]):