# Path: crowdastro/active_learning/sampler.py
# class Sampler(object):
#     """Pool-based active learning base class."""
# 
#     def __init__(self, pool, labels, Classifier, classifier_params=None):
#         """
#         pool: (n_samples, n_features) array of partially labelled data points.
#         labels: (n_samples,) masked array of binary labels.
#         classifier: Binary classifier class implementing a sklearn interface.
#         classifier_params: Parameters to pass to Classifier. Default None.
#         """
#         self.pool = pool
#         self.labels = labels
#         self.Classifier = Classifier
#         self.classifier_params = classifier_params or {}
# 
#         self.train()
# 
#     def sample_index(self):
#         """Finds index of the unlabelled point to sample."""
#         raise NotImplementedError()
# 
#     def sample_indices(self, n):
#         """Finds indices of the top n unlabelled points to sample."""
#         raise NotImplementedError()
# 
#     def add_label(self, index, label, retrain=True):
#         """Adds a label from an oracle.
# 
#         index: Index of data point to label.
#         label: Label from the oracle.
#         """
#         self.labels[index] = label
#         if retrain:
#             self.retrain()
# 
#     def add_labels(self, indices, labels, retrain=True):
#         """Adds labels from an oracle.
# 
#         indices: Indices of data points to label.
#         labels: Labels from the oracle.
#         """
#         for index, label in zip(indices, labels):
#             self.add_label(index, label, retrain=False)
#         if retrain:
#             self.retrain()
# 
#     def train(self):
#         """Trains the classifier."""
#         self.classifier = self.Classifier(**self.classifier_params)
#         self.classifier.fit(self.pool[~self.labels.mask],
#                             self.labels[~self.labels.mask])
# 
#     def retrain(self):
#         """Retrains the classifier."""
#         # TODO(MatthewJA): Not sure if we should use warm starts here, so for
#         # now I won't.
#         self.train()
# 
#     def score(self, test_xs, test_ts):
#         """Finds cross-entropy error on test data."""
#         return sklearn.metrics.log_loss(
#             test_ts,
#             self.classifier.predict(test_xs))
# 
#     def ba(self, test_xs, test_ts):
#         """Finds balanced accuracy on test data."""
#         return balanced_accuracy(test_ts, self.classifier.predict(test_xs))
# 
# Path: crowdastro/crowd/util.py
# def balanced_accuracy(y_true, y_pred):
#     """Computes the balanced accuracy of a predictor.
# 
#     y_true: (n_examples,) array of true labels.
#     y_pred: (n_examples,) (masked) array of predicted labels.
#     -> float or None (if the balanced accuracy isn't defined).
#     """
#     if hasattr(y_pred, 'mask') and not isinstance(y_pred.mask, bool):
#         cm = sklearn.metrics.confusion_matrix(
#                 y_true[~y_pred.mask], y_pred[~y_pred.mask]).astype(float)
#     else:
#         cm = sklearn.metrics.confusion_matrix(y_true, y_pred).astype(float)
# 
#     tp = cm[1, 1]
#     n, p = cm.sum(axis=1)
#     tn = cm[0, 0]
#     if not n or not p:
#         return None
# 
#     ba = (tp / p + tn / n) / 2
#     return ba
# 
# def majority_vote(y):
#     """Computes the majority vote of a set of crowd labels.
# 
#     y: (n_annotators, n_examples) NumPy masked array of labels.
#     -> (n_examples,) NumPy array of labels.
#     """
#     _, n_samples = y.shape
#     mv = numpy.zeros((n_samples,))
#     for i in range(n_samples):
#         labels = y[:, i]
# 
#         if labels.mask is False:
#             counter = collections.Counter(labels)
#         else:
#             counter = collections.Counter(labels[~labels.mask])
# 
#         if counter:
#             mv[i] = max(counter, key=counter.get)
#         else:
#             # No labels for this data point.
#             mv[i] = numpy.random.randint(2)  # ¯\_(ツ)_/¯
#     return mv


import collections
import logging
import numpy
import sklearn.cross_validation
import sklearn.metrics
from .sampler import Sampler
from ..crowd.util import balanced_accuracy, majority_vote
    def sample_index(self):
        """Finds index of the most disagreed upon unlabelled point."""
        index = self.disagreement.argmin()
        return index

    def sample_indices(self, n):
        """Finds indices of the top n most disagreed upon unlabelled points."""
        indices = self.disagreement.argsort()
        return indices[:n]

    def retrain(self):
        """Retrains the classifier."""
        super().retrain()
        self.compute_disagreement()

    def score(self, test_xs, test_ts):
        """Finds cross-entropy error on test data."""
        return numpy.mean([
                sklearn.metrics.log_loss(
                        test_ts, c.predict(test_xs))
                for c in self.classifiers])

    def ba(self, test_xs, test_ts):
        """Finds balanced accuracy on test data."""
        # labels = numpy.zeros((self.n_classifiers, len(test_xs)))
        # for c in range(self.n_classifiers):
        #     preds = self.classifiers[c].predict(test_xs)
        #     labels[c, :] = preds
        # labels = numpy.ma.MaskedArray(labels, mask=numpy.zeros(labels.shape))
        # return balanced_accuracy(test_ts, majority_vote(labels))

# Path: crowdastro/active_learning/sampler.py
# class Sampler(object):
#     """Pool-based active learning base class."""
# 
#     def __init__(self, pool, labels, Classifier, classifier_params=None):
#         """
#         pool: (n_samples, n_features) array of partially labelled data points.
#         labels: (n_samples,) masked array of binary labels.
#         classifier: Binary classifier class implementing a sklearn interface.
#         classifier_params: Parameters to pass to Classifier. Default None.
#         """
#         self.pool = pool
#         self.labels = labels
#         self.Classifier = Classifier
#         self.classifier_params = classifier_params or {}
# 
#         self.train()
# 
#     def sample_index(self):
#         """Finds index of the unlabelled point to sample."""
#         raise NotImplementedError()
# 
#     def sample_indices(self, n):
#         """Finds indices of the top n unlabelled points to sample."""
#         raise NotImplementedError()
# 
#     def add_label(self, index, label, retrain=True):
#         """Adds a label from an oracle.
# 
#         index: Index of data point to label.
#         label: Label from the oracle.
#         """
#         self.labels[index] = label
#         if retrain:
#             self.retrain()
# 
#     def add_labels(self, indices, labels, retrain=True):
#         """Adds labels from an oracle.
# 
#         indices: Indices of data points to label.
#         labels: Labels from the oracle.
#         """
#         for index, label in zip(indices, labels):
#             self.add_label(index, label, retrain=False)
#         if retrain:
#             self.retrain()
# 
#     def train(self):
#         """Trains the classifier."""
#         self.classifier = self.Classifier(**self.classifier_params)
#         self.classifier.fit(self.pool[~self.labels.mask],
#                             self.labels[~self.labels.mask])
# 
#     def retrain(self):
#         """Retrains the classifier."""
#         # TODO(MatthewJA): Not sure if we should use warm starts here, so for
#         # now I won't.
#         self.train()
# 
#     def score(self, test_xs, test_ts):
#         """Finds cross-entropy error on test data."""
#         return sklearn.metrics.log_loss(
#             test_ts,
#             self.classifier.predict(test_xs))
# 
#     def ba(self, test_xs, test_ts):
#         """Finds balanced accuracy on test data."""
#         return balanced_accuracy(test_ts, self.classifier.predict(test_xs))
# 
# Path: crowdastro/crowd/util.py
# def balanced_accuracy(y_true, y_pred):
#     """Computes the balanced accuracy of a predictor.
# 
#     y_true: (n_examples,) array of true labels.
#     y_pred: (n_examples,) (masked) array of predicted labels.
#     -> float or None (if the balanced accuracy isn't defined).
#     """
#     if hasattr(y_pred, 'mask') and not isinstance(y_pred.mask, bool):
#         cm = sklearn.metrics.confusion_matrix(
#                 y_true[~y_pred.mask], y_pred[~y_pred.mask]).astype(float)
#     else:
#         cm = sklearn.metrics.confusion_matrix(y_true, y_pred).astype(float)
# 
#     tp = cm[1, 1]
#     n, p = cm.sum(axis=1)
#     tn = cm[0, 0]
#     if not n or not p:
#         return None
# 
#     ba = (tp / p + tn / n) / 2
#     return ba
# 
# def majority_vote(y):
#     """Computes the majority vote of a set of crowd labels.
# 
#     y: (n_annotators, n_examples) NumPy masked array of labels.
#     -> (n_examples,) NumPy array of labels.
#     """
#     _, n_samples = y.shape
#     mv = numpy.zeros((n_samples,))
#     for i in range(n_samples):
#         labels = y[:, i]
# 
#         if labels.mask is False:
#             counter = collections.Counter(labels)
#         else:
#             counter = collections.Counter(labels[~labels.mask])
# 
#         if counter:
#             mv[i] = max(counter, key=counter.get)
#         else:
#             # No labels for this data point.
#             mv[i] = numpy.random.randint(2)  # ¯\_(ツ)_/¯
#     return mv


# Path: crowdastro/active_learning/qbc_sampler.py
import collections
import logging
import numpy
import sklearn.cross_validation
import sklearn.metrics
from .sampler import Sampler
from ..crowd.util import balanced_accuracy, majority_vote

    def sample_index(self):
        """Finds index of the most disagreed upon unlabelled point."""
        index = self.disagreement.argmin()
        return index

    def sample_indices(self, n):
        """Finds indices of the top n most disagreed upon unlabelled points."""
        indices = self.disagreement.argsort()
        return indices[:n]

    def retrain(self):
        """Retrains the classifier."""
        super().retrain()
        self.compute_disagreement()

    def score(self, test_xs, test_ts):
        """Finds cross-entropy error on test data."""
        return numpy.mean([
                sklearn.metrics.log_loss(
                        test_ts, c.predict(test_xs))
                for c in self.classifiers])

    def ba(self, test_xs, test_ts):
        """Finds balanced accuracy on test data."""
        # labels = numpy.zeros((self.n_classifiers, len(test_xs)))
        # for c in range(self.n_classifiers):
        #     preds = self.classifiers[c].predict(test_xs)
        #     labels[c, :] = preds
        # labels = numpy.ma.MaskedArray(labels, mask=numpy.zeros(labels.shape))
        # return balanced_accuracy(test_ts, majority_vote(labels))

        return balanced_accuracy(