# Path: crowdastro/crowd/util.py
# def balanced_accuracy(y_true, y_pred):
#     """Computes the balanced accuracy of a predictor.
# 
#     y_true: (n_examples,) array of true labels.
#     y_pred: (n_examples,) (masked) array of predicted labels.
#     -> float or None (if the balanced accuracy isn't defined).
#     """
#     if hasattr(y_pred, 'mask') and not isinstance(y_pred.mask, bool):
#         cm = sklearn.metrics.confusion_matrix(
#                 y_true[~y_pred.mask], y_pred[~y_pred.mask]).astype(float)
#     else:
#         cm = sklearn.metrics.confusion_matrix(y_true, y_pred).astype(float)
# 
#     tp = cm[1, 1]
#     n, p = cm.sum(axis=1)
#     tn = cm[0, 0]
#     if not n or not p:
#         return None
# 
#     ba = (tp / p + tn / n) / 2
#     return ba
# 
# Path: crowdastro/active_learning/random_sampler.py
# class RandomSampler(Sampler):
# class BalancedSampler(RandomSampler):
#     def sample_index(self):
#     def sample_indices(self, n):
#     def sample_index(self):
#     def sample_indices(self, n):
# 
# Path: crowdastro/active_learning/qbc_sampler.py
# class QBCSampler(Sampler):
#     def __init__(self, pool, labels, Classifier, n_classifiers=20,
#                  subsample_rate=0.75, classifier_params=None):
#     def train(self):
#     def compute_disagreement(self):
#     def sample_index(self):
#     def sample_indices(self, n):
#     def retrain(self):
#     def score(self, test_xs, test_ts):
#     def ba(self, test_xs, test_ts):
# 
# Path: crowdastro/plot.py
# def fillbetween(xs, ys, facecolour='lightgreen', edgecolour='green',
#                 marker='x', facealpha=1.0, facekwargs=None, **kwargs):
#     """Plots a line plot with error represented by filled-between lines.
# 
#     xs: List of x values.
#     ys: List of lists of y values.
#     facecolour: Colour of the filled-between lines. Default light green.
#     edgecolour: Colour of the central line. Default green.
#     marker: Point marker. Default 'x'.
#     facealpha: Alpha value of filled section. Default 1.0.
#     facekwargs: Keyword arguments to pass to fill_between. Default {}.
#     """
#     facekwargs = facekwargs or {}
#     means = numpy.mean(ys, axis=1)
#     stds = numpy.std(ys, axis=1)
#     plt.plot(xs, means, color=edgecolour, marker=marker, **kwargs)
#     plt.fill_between(xs, means - stds, means + stds, color=facecolour,
#                      alpha=facealpha, linewidth=0, **facekwargs)
#     plt.fill_between(xs, means - stds, means + stds, facecolor='None',
#                      edgecolor=edgecolour, alpha=0.5, **facekwargs)


import argparse
import logging
import os.path
import h5py
import matplotlib
import matplotlib.pyplot as plt
import numpy
import sklearn.cross_validation
import sklearn.linear_model
from crowdastro.crowd.util import balanced_accuracy
from crowdastro.active_learning import random_sampler, qbc_sampler
from crowdastro.plot import fillbetween
                                n_required_queries))
                            queries = sampler.sample_indices(n_required_queries)
                            queried_labels = queryable_labels.data[queries]
                            sampler.add_labels(queries, queried_labels)
                            logging.debug('Total labels known: {}'.format(
                                (~sampler.labels.mask).sum()))
                            results[
                                method_index, split_id, count_index + 1
                            ] = sampler.ba(features[split], norris[split])

                        with open(results_npy_path, 'wb') as f:
                            numpy.save(f, results, allow_pickle=False)

                        # TODO(MatthewJA): Implement overwrite parameter.
                        # TODO(MatthewJA): Implement loading .npy file.

            matplotlib.rcParams['font.family'] = 'serif'
            matplotlib.rcParams['font.serif'] = ['Palatino Linotype']
            plt.figure(figsize=(6, 6))
            results *= 100
            fillbetween(instance_counts, list(zip(*results[0, :])),
                        facecolour='green', edgecolour='green', facealpha=0.1,
                        linestyle='-', marker='.', facekwargs={
                            'linestyle': '-',
                        })
            fillbetween(instance_counts, list(zip(*results[1, :])),
                        facecolour='blue', edgecolour='blue', facealpha=0.1,
                        linestyle='-.', marker='+', facekwargs={
                            'linestyle': '-.',
                        })

# Path: crowdastro/crowd/util.py
# def balanced_accuracy(y_true, y_pred):
#     """Computes the balanced accuracy of a predictor.
# 
#     y_true: (n_examples,) array of true labels.
#     y_pred: (n_examples,) (masked) array of predicted labels.
#     -> float or None (if the balanced accuracy isn't defined).
#     """
#     if hasattr(y_pred, 'mask') and not isinstance(y_pred.mask, bool):
#         cm = sklearn.metrics.confusion_matrix(
#                 y_true[~y_pred.mask], y_pred[~y_pred.mask]).astype(float)
#     else:
#         cm = sklearn.metrics.confusion_matrix(y_true, y_pred).astype(float)
# 
#     tp = cm[1, 1]
#     n, p = cm.sum(axis=1)
#     tn = cm[0, 0]
#     if not n or not p:
#         return None
# 
#     ba = (tp / p + tn / n) / 2
#     return ba
# 
# Path: crowdastro/active_learning/random_sampler.py
# class RandomSampler(Sampler):
# class BalancedSampler(RandomSampler):
#     def sample_index(self):
#     def sample_indices(self, n):
#     def sample_index(self):
#     def sample_indices(self, n):
# 
# Path: crowdastro/active_learning/qbc_sampler.py
# class QBCSampler(Sampler):
#     def __init__(self, pool, labels, Classifier, n_classifiers=20,
#                  subsample_rate=0.75, classifier_params=None):
#     def train(self):
#     def compute_disagreement(self):
#     def sample_index(self):
#     def sample_indices(self, n):
#     def retrain(self):
#     def score(self, test_xs, test_ts):
#     def ba(self, test_xs, test_ts):
# 
# Path: crowdastro/plot.py
# def fillbetween(xs, ys, facecolour='lightgreen', edgecolour='green',
#                 marker='x', facealpha=1.0, facekwargs=None, **kwargs):
#     """Plots a line plot with error represented by filled-between lines.
# 
#     xs: List of x values.
#     ys: List of lists of y values.
#     facecolour: Colour of the filled-between lines. Default light green.
#     edgecolour: Colour of the central line. Default green.
#     marker: Point marker. Default 'x'.
#     facealpha: Alpha value of filled section. Default 1.0.
#     facekwargs: Keyword arguments to pass to fill_between. Default {}.
#     """
#     facekwargs = facekwargs or {}
#     means = numpy.mean(ys, axis=1)
#     stds = numpy.std(ys, axis=1)
#     plt.plot(xs, means, color=edgecolour, marker=marker, **kwargs)
#     plt.fill_between(xs, means - stds, means + stds, color=facecolour,
#                      alpha=facealpha, linewidth=0, **facekwargs)
#     plt.fill_between(xs, means - stds, means + stds, facecolor='None',
#                      edgecolor=edgecolour, alpha=0.5, **facekwargs)


# Path: crowdastro/experiment/experiment_rgz_qbc.py
import argparse
import logging
import os.path
import h5py
import matplotlib
import matplotlib.pyplot as plt
import numpy
import sklearn.cross_validation
import sklearn.linear_model
from crowdastro.crowd.util import balanced_accuracy
from crowdastro.active_learning import random_sampler, qbc_sampler
from crowdastro.plot import fillbetween

                                n_required_queries))
                            queries = sampler.sample_indices(n_required_queries)
                            queried_labels = queryable_labels.data[queries]
                            sampler.add_labels(queries, queried_labels)
                            logging.debug('Total labels known: {}'.format(
                                (~sampler.labels.mask).sum()))
                            results[
                                method_index, split_id, count_index + 1
                            ] = sampler.ba(features[split], norris[split])

                        with open(results_npy_path, 'wb') as f:
                            numpy.save(f, results, allow_pickle=False)

                        # TODO(MatthewJA): Implement overwrite parameter.
                        # TODO(MatthewJA): Implement loading .npy file.

            matplotlib.rcParams['font.family'] = 'serif'
            matplotlib.rcParams['font.serif'] = ['Palatino Linotype']
            plt.figure(figsize=(6, 6))
            results *= 100
            fillbetween(instance_counts, list(zip(*results[0, :])),
                        facecolour='green', edgecolour='green', facealpha=0.1,
                        linestyle='-', marker='.', facekwargs={
                            'linestyle': '-',
                        })
            fillbetween(instance_counts, list(zip(*results[1, :])),
                        facecolour='blue', edgecolour='blue', facealpha=0.1,
                        linestyle='-.', marker='+', facekwargs={
                            'linestyle': '-.',
                        })

            fillbetween(instance_counts, list(zip(*results[2, :])),