from .baseline_util import parse_arguments, compute_f1
from os.path import join
# from aad.malware_aad import *
from aad.data_stream import *
from modAL.models import ActiveLearner
from sklearn.ensemble import RandomForestClassifier
from modAL.uncertainty import *
import time
from aad.query_model_euclidean_baseline import filter_by_euclidean_distance
from sklearn.metrics import accuracy_score, f1_score, confusion_matrix
from aad.classifier_trees import RandomForestAadWrapper
from pprint import pprint
# from aad.aad_support import configure_logger

import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from common.utils import *


class MalwareDataStream(object):
    def __init__(self, X, y=None, id_server=None):
        self.X = X
        self.X_orig = np.copy(X)
        self.y = y
        self.id_server = id_server
        self.cur_year = 1
        # self.yearwise_count = [2836, 1439, 1444, 1439, 1424, 1439]
        # self.yearwise_count = [5075, 3790, 2707, 4290, 2629, 3138]
        self.yearwise_count = [5938, 3790, 2707, 4290, 2629, 3138]
        # self.yearwise_count = [3221, 2717, 3790, 2707, 4290, 2629, 3138]
        self.max_year = len(self.yearwise_count)
        self.yearwise_data, self.yearwise_label = self.process_yearwise()
        # logger.debug("Processed dataset: %s" % str(self.yearwise_data))

    def process_yearwise(self):
        yearwise_data = {}
        yearwise_label = {}
        count = 0
        for i in range(len(self.yearwise_count)):
            yearwise_data[2010 + i] = self.X[count: count + self.yearwise_count[i]]
            yearwise_label[2010 + i] = self.y[count: count + self.yearwise_count[i]]
            count += self.yearwise_count[i]
        return yearwise_data, yearwise_label

    def read_next_from_stream(self, n=1):
        """Returns first n instances from X and removes these instances from X"""
        if self.cur_year > self.max_year:
            return None
        if n == -1:
            mask = None
            self.X = self.X_orig
            n = self.X.shape[0]
            logger.debug("mask is %s X shape %d" % (mask, self.X.shape[0]))
            mask = np.zeros(self.X.shape[0], dtype=bool)
            mask[np.arange(n)] = True
            instances = self.X[:]
            labels = self.y[:]
            self.X = self.X[~mask]
            logger.debug("DataStream.read_next_from_stream instances: %s" % str(instances.shape))
            if self.id_server is not None:
                ids = self.id_server.get_next(n)
            logger.debug("IDs %s" % str(ids))
        else:
            logger.debug("cur year: %s" % str(self.cur_year))
            n = min(self.yearwise_count[self.cur_year - 1], self.X.shape[0])
            mask = np.zeros(self.X.shape[0], dtype=bool)
            mask[np.arange(n)] = True
            self.X = self.X[~mask]
            instances = self.yearwise_data[2010 + self.cur_year - 1]
            labels = self.yearwise_label[2010 + self.cur_year - 1]
            logger.debug("DataStream.read_next_from_stream instances: %s" % str(instances.shape))
            self.cur_year += 1
            if self.id_server is not None:
                ids = self.id_server.get_next(n)
            logger.debug("IDs %s" % str(ids))
        logger.debug("instances %d" % (instances.shape[0]))
        return InstanceList(instances, labels, ids)

    def empty(self):
        # if self.cur_year > self.max_year:
        #     return True
        # return False
        return self.X is None or self.X.shape[0] == 0


class BaselineExperiments:
    def __init__(self, opts):
        self.opts = opts
        self.datafile = join(opts.filedir, "fullsamples", opts.datafile)
        self.opts.labelindex = 1
        self.X_full, self.y_full = read_data_as_matrix(self.opts)
        self.stream = MalwareDataStream(self.X_full, self.y_full, IdServer(initial=0))
        self.test_stream = MalwareDataStream(self.X_full, self.y_full, IdServer(initial=0))
        self.queried_X = None
        self.queried_y = None
        self.alpha = 0.05
        self.n_trees = 100
        self._prepare_training()

    def _prepare_training(self):
        training = self.stream.read_next_from_stream()
        self.cur_train_year = 1
        self.x_train, self.y_train, self.ids_train = training.x, training.y, training.ids
        logger.debug("Pre-trained with x: %s, y: %s " % (str(self.x_train.shape), str(self.y_train.shape)))

    def prepare_learner(self, query_strategy=uncertainty_sampling, base_learner=None):
        if self.opts.starting == 1:
            if self.queried_X is not None and self.queried_y is not None:
                logger.debug("carrying over the feedbacks")
                logger.debug("queried X : %s y : %s" % (self.queried_X.shape[0], self.queried_y.shape[0]))
                self.x_train = np.vstack((self.x_train, self.queried_X))
                self.y_train = np.append(self.y_train, self.queried_y.reshape(-1))
        elif self.opts.starting == 2:
            best_data_stream = MalwareDataStream(self.X_full, self.y_full, IdServer(initial=0))
            yr_idx = 1
            while yr_idx <= self.cur_train_year:
                _ = best_data_stream.read_next_from_stream()
                yr_idx += 1
            c_data = best_data_stream.read_next_from_stream()
            self.x_train = np.vstack((self.x_train, c_data.x))
            self.y_train = np.append(self.y_train, c_data.y)

        logger.debug("x: %s, y: %s " % (str(self.x_train.shape), str(self.y_train.shape)))
        seed = self.opts.seed + self.current_run_id * 100
        logger.debug("seed for epoch %d is %d" % (self.current_run_id, seed))
        learner = ActiveLearner(
            estimator=RandomForestClassifier(n_estimators=100, random_state=seed),
            query_strategy=query_strategy,
            # X_training=self.x_train[:, -164:], y_training=self.y_train)
            X_training = self.x_train, y_training = self.y_train)
        self.learner = learner
        if self.opts.weighted_update is True:
            self.compute_KL()
        return learner

    def initial_data(self, n_labeled_data=100):
        n_data = self.X_full.shape[0]
        seed = self.opts.seed + self.current_run_id * 100
        np.random.seed(seed)
        a = np.arange(n_data)
        np.random.shuffle(a)
        init_indexes = a[:n_labeled_data]
        logger.debug("initial selected candidates: %s" % (init_indexes))
        return init_indexes

    def compute_KL(self):
        tm = Timer()
        self.model = RandomForestAadWrapper(x=self.x_train, y=self.y_train, clf=self.learner.estimator)
        logger.debug("Wrapper model created with %d nodes" % len(self.model.w))

        # compute KL replacement threshold *without* p
        ref_kls, self.kl_q_alpha = self.model.get_KL_divergence_distribution(self.x_train, p=None, alpha=self.alpha)
        # now initialize reference p
        self.p = self.model.get_node_sample_distributions(self.x_train)
        print(tm.message("Time to prepare KL divergence:"))


    def compute_tree_exceeding(self, x, y):
        window = 0
        logger.debug("window %d loaded: %d" % (window, x.shape[0]))
        # compare KL-divergence of current data dist against reference dist p
        comp_kls, _ = self.model.get_KL_divergence_distribution(x, p=self.p)

        # find which trees exceed alpha-level threshold
        trees_exceeding_kl_q_alpha = self.model.get_trees_to_replace(comp_kls, self.kl_q_alpha)
        n_threshold = int(2 * self.alpha * self.n_trees)

        logger.debug("[%d] #trees_exceeding_kl_q_alpha: %d, threshold number of trees: %d\n%s" %
                     (window, len(trees_exceeding_kl_q_alpha), n_threshold, str(list(trees_exceeding_kl_q_alpha))))

        return trees_exceeding_kl_q_alpha

    def train_with_feedbacks(self, opts, X_pool, y_pool, id_pool=None, tree_exceeding_count=None):
        """get feedbacks and update the model"""
        n_queries = opts.n_query
        queried_index = list()
        ids_index = list()
        queried_X = None
        queried_y = None
        initial_x = np.copy(X_pool)
        initial_y = np.copy(y_pool)
        y_feedback = list()
        y_queried = list()
        n_pool = X_pool.shape[0]
        n_query_to_ask = min(int( n_pool / 4), n_queries)
        self.opts.n_asked_query = n_query_to_ask
        percentage_list = [i for i in range(0, int(n_pool/20), int(n_pool/100))]
        for i in range(int(n_pool/20), int(n_pool/4) + 1, int(n_pool/20)):
            percentage_list.append(i)
        # percentage_list = [i for i in range(0, 10, 3)]
        logger.debug("Evaluation points %s" % (str(percentage_list)))
        logger.debug("Init Pool status X %d" % (X_pool.shape[0]))
        n_pretrain = self.x_train.shape[0]
        for idx in range(n_query_to_ask):
            # handler for diverse query strategy
            logger.debug("##query count %d" % (idx))
            if opts.query_strategy == 2:
                idx += 2
            query_idx, query_instance = self.learner.query(X_pool)
            ids_index.extend(id_pool[query_idx])
            # print("picked query ", query_idx, " y_pool ", y_pool[query_idx])
            y_feedback.extend(y_pool[query_idx])
            y_queried.extend(self.learner.predict(X_pool[query_idx]))
            # print("Predicted: ", self.learner.predict(X_pool[query_idx]))
            # print("Pool :", self.learner.predict(X_pool[:30]))
            n_selected_query = len(query_idx)
            self.learner._add_training_data(X=X_pool[query_idx].reshape(n_selected_query, -1),
                                            y=y_pool[query_idx].reshape(n_selected_query,))

            if opts.weighted_update is True:
                n_examples = self.learner.X_training.shape[0]
                old_instance_weight = (1.0 - len(tree_exceeding_count) * 1.0/100)
                instance_weight = [old_instance_weight] * (n_pretrain)
                instance_weight.extend([1.0] * (n_examples - n_pretrain))
                # logger.debug("Old instance weights %s", old_instance_weight, " n_pretrain ", n_pretrain, " #new weights ", (n_examples - n_pretrain),
                #       " #instances ", len(instance_weight))
                logger.debug("#instance weights %d and new examples %d" % (len(instance_weight), n_examples))

                self.learner.fit(self.learner.X_training, self.learner.y_training, sample_weight=instance_weight)
            else:
                self.learner.fit(self.learner.X_training, self.learner.y_training)
            # self.learner.teach(
            #     X=X_pool[query_idx].reshape(n_selected_query, -1),
            #     y=y_pool[query_idx].reshape(n_selected_query, ),
            #     sample_weight=0.1
            # )
            # queried_index.extend(query_idx)
            if queried_X is not None and queried_y is not None:
                queried_X = np.vstack((queried_X, np.copy(X_pool[query_idx]).reshape(n_selected_query,-1)))
                queried_y = np.vstack((queried_y, np.copy(y_pool[query_idx]).reshape(n_selected_query, 1)))
            else:
                queried_X = np.vstack((np.copy(X_pool[query_idx])))
                queried_y = np.vstack((np.copy(y_pool[query_idx])))
            # remove queried instance from pool
            X_pool = np.delete(X_pool, query_idx, axis=0)
            y_pool = np.delete(y_pool, query_idx)
            id_pool = np.delete(id_pool, query_idx)
            # y_pred = learner.predict_proba(X_pool)
            # compute_f1(y_pool, y_pred[:, 1])
            # print("X_training.shape ", learner.X_training.shape)
            # print("Training score: ", learner.score(learner.X_training, learner.y_training))
            X_evaluate = np.vstack((X_pool, self.learner.X_training[queried_index]))
            # print("X_evaluate.shape ", X_evaluate.shape)
            # print("testing ", X_pool.shape, queried_X.shape, " y_pool ", y_pool.shape, " queried y ", queried_y.shape)
            eval_y = np.vstack((queried_y, y_pool.reshape(-1, 1)))
            eval_x = np.vstack((queried_X, X_pool))


            if ((idx + 1) in percentage_list and opts.qsname is not "diverse_query_strategy") or \
                    (opts.qsname == "diverse_query_strategy" and (3*idx + 1 in percentage_list or
                                                   3*idx + 2 in percentage_list or
                                                   3*idx + 3 in percentage_list)):
                f1_score = self.get_performance(X_pool, y_pool, y_queried, y_feedback)
                logger.debug("after %d feedback fair f1 score is %.4f" % (idx, f1_score))
                f1_score = self.get_performance(X_pool, y_pool, y_queried, y_feedback, optimistic=True)
                logger.debug("after %d feedback fair f1 score is %.4f" % (idx, f1_score))
                logger.debug("***Evaluation after %d feedback on year %s***" % ((idx + 1),
                                                                                (2011 + self.cur_train_year)))

                self.evaluate_learner(feedbacks=idx)
                logger.debug("##Evaluation done##")

        self.queried_X = np.copy(queried_X)
        self.queried_y = np.copy(queried_y)
        logger.debug("Ids queried %s" % (str(ids_index)))
        logger.debug("Pool status X %d" % (X_pool.shape[0]))
        self.store_queries(ids_index)

    def store_queries(self, ids_index):
        """We need to store the queries index"""
        logger.debug("")
        keys = self.opts.queried_indexes
        current_key = str(2011 + self.cur_train_year)
        logger.debug("###queried key %s" % current_key)
        if current_key not in keys:
            self.opts.queried_indexes[current_key] = list()
        self.opts.queried_indexes[current_key].extend(ids_index)


    def evaluate_learner(self, feedbacks=0):
        """We want to evaluate our learners performance on the test years"""
        logger.debug("#feedback till now %d" % feedbacks)
        if self.opts.train_years == -1:
            if self.pool_data is not None:
                logger.debug("evaluation done on rest of the pool data")
                self.get_performance(self.pool_data.x, self.pool_data.y)
        else:
            evaluate_stream = MalwareDataStream(self.X_full, self.y_full, IdServer(initial=0))
            start_yr = 2011
            logger.debug("self.cur_train_year  %d" % self.cur_train_year)
            for index in range(self.cur_train_year):
                logger.debug("skipping yr %d" % start_yr)
                start_yr += 1
                _ = evaluate_stream.read_next_from_stream()
            while not evaluate_stream.empty():
                logger.debug("evaluating on yr %d" % start_yr)
                test_data = evaluate_stream.read_next_from_stream()
                X_test = test_data.x
                y_test = test_data.y
                self.get_performance(X_test, y_test, feedbacks=feedbacks, yr=start_yr)
                start_yr += 1

    def get_performance(self, X, y, y_queried=None, y_feedback=None, feedbacks=0, yr=2011, optimistic=False):
        """given the predictions and ground truths compute the scores"""
        y = y.reshape(-1, 1)
        if y_queried is not None and y_feedback is not None:
            y_queried = np.array(y_queried).reshape(-1, 1)
            y_feedback = np.array(y_feedback).reshape(-1, 1)
            queried_score = accuracy_score(y_feedback, y_queried)
            queried_f1 = f1_score(y_feedback, y_queried)
            # print("For the same year accuracy %.2f and f1 is %.2f" %(queried_score, queried_f1))
            logger.debug("we have to evaluate the current year where we are getting feedback")
            logger.debug("y_queried shape %s" % (str(y_queried.shape[0])))
        # y_pred = self.learner.predict(X[:, -164:]).reshape(-1, 1)
        y_pred = self.learner.predict(X).reshape(-1, 1)
        if optimistic is False and y_queried is not None and y_feedback is not None:
            y_pred = np.vstack((y_pred, y_queried))
            y_true = np.vstack((y, y_feedback))
        elif optimistic is True and y_queried is not None and y_feedback is not None:
            y_pred = np.vstack((y_pred, y_feedback))
            y_true = np.vstack((y, y_feedback))
        else:
            y_true = y
        tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
        logger.debug("Detected malware %d total malware %d rate and ADR is %.4f" %(tp, tp+fn, tp* 1.0 / (tp+fn)))
        f1 = f1_score(y_true, y_pred, average='macro')
        logger.debug("F1 Score: %.4f, accuracy score is %.4f, false positive %.4f" % (f1, accuracy_score(y_true, y_pred), fp))
        result_key = str(2011 + self.cur_train_year) + "_" + str(yr)
        print("For " + result_key + " Detected (tp): ", tp, " fp: ", fp, " total malwares: ", tp + fn, " rate ", tp * 1.0 / (tp + fn))
        # n_y_true = int(y_true.shape[0] * 0.05)
        # t_tn, t_fp, t_fn, t_tp = confusion_matrix(y_true[:n_y_true], y_pred[:n_y_true]).ravel()
        # print ("For top 5%" + result_key + " Detected: ", t_tp, " total: ", t_tp + t_fn, " rate ", t_tp * 1.0 / (t_tp + t_fn))
        if y_queried is not None and y_feedback is not None and optimistic is False:
            result_key = str(2011 + self.cur_train_year) + "_" + str(2011 + self.cur_train_year)
            result_key += "_" + "fair_eval"
        if y_queried is not None and y_feedback is not None and optimistic is True:
            result_key = str(2011 + self.cur_train_year) + "_" + str(2011 + self.cur_train_year)
            result_key += "_" + "optmst_eval"
        if result_key not in self.opts.results_f1.keys():
            self.opts.results_f1[result_key] = list()
        if result_key not in self.opts.results_adr.keys():
            self.opts.results_adr[result_key] = list()
        self.opts.results_f1 [result_key].append(f1)
        self.opts.results_adr[result_key].append(tp * 1.0 / (tp+fn))
        # print("length ", len(self.opts.results_f1 [result_key]))
        logger.debug("result length %d" % len(self.opts.results_f1 [result_key]))
        return f1

    def plot_results(self):
        """We want to produce the plots for the results"""
        print(self.opts.results_f1.keys())
        if not os.path.exists(self.opts.save_fig_location):
            os.makedirs(self.opts.save_fig_location)
        save_location = os.path.join(self.opts.save_fig_location, self.opts.prefix)
        for i in range(1, 6):
            for j in range(0 + i, 6):
                key = str(2011 + i) + "_" + str(2011 + j)
                print(key + " " + str(self.opts.results_f1 [key]))
                plt.plot(self.opts.results_f1 [key], label=key)

            k = str(2011 + i) + "_" + str(2011 + i) + "_fair_eval"
            if k in self.opts.results_f1.keys():
                print("inside ", k)
                plt.plot(self.opts.results_f1[k], label=k)

            k = str(2011 + i) + "_" + str(2011 + i) + "_optmst_eval"
            if k in self.opts.results_f1.keys():
                print("inside ", k)
                print(k + " " + str(self.opts.results_f1[k]))
                plt.plot(self.opts.results_f1[k], label=k)
            plt.legend()
            plt.xlabel("Queried")
            plt.ylabel("F1 Score")
            # plt.xlim([0, 5, 10, 15, 20, 25, 100])
            plt.xticks(np.arange(0, 6, 1), ('No', '5%', '10%', '15%', '20%', '25%'))
            plt.title("Feedback effect " + str(2011 + i))
            plt.savefig(save_location + "-feedback-" +str(2011 + i) + "-f1.pdf")
            plt.close()

        print(self.opts.results_adr.keys())
        for i in range(1, 6):
            for j in range(0 + i, 6):
                key = str(2011 + i) + "_" + str(2011 + j)
                print(key + " " + str(self.opts.results_adr[key]))
                plt.plot(self.opts.results_adr[key], label=key)
            k = str(2011 + i) + "_" + str(2011 + i) + "_fair_eval"
            if k in self.opts.results_adr.keys():
                print("inside ", k)
                plt.plot(self.opts.results_adr[k], label=k)

            k = str(2011 + i) + "_" + str(2011 + i) + "_optmst_eval"
            if k in self.opts.results_adr.keys():
                print("inside ", k)
                print(k + " " + str(self.opts.results_adr[k]))
                plt.plot(self.opts.results_adr[k], label=k)
            plt.legend()
            plt.xlabel("Fraction of data queried")
            plt.ylabel("Anomaly Detection Rate")
            plt.xticks(np.arange(0, 6, 1), ('No', '5%', '10%', '15%', '20%', '25%'))
            plt.title("Feedback effect " + str(2011 + i))
            plt.savefig(save_location + "-feedback-" + str(2011 + i) + "-adr.pdf")
            plt.close()

    def save_results(self):
        """Saving the results with proper file name prefix"""
        # todo: for each key save in a seperate file as we have multiple runs (10)
        # format will be

        if not os.path.exists(self.opts.save_results_location):
            os.makedirs(self.opts.save_results_location)

        self._save_results(self.opts.results_adr, suffix="adr")
        self._save_results(self.opts.results_f1, suffix="f1")
        self._save_results(self.opts.queried_indexes, suffix="queried")
        self.opts.results_adr = {}
        self.opts.results_f1 = {}
        self.opts.queried_indexes = {}

    def _save_results(self, res_map, suffix=None):
        """save for any result map"""
        if suffix is None:
            suffix = "nameless"
        result_folder = os.path.join(self.opts.save_results_location, self.opts.prefix)
        if not os.path.exists(result_folder):
            os.makedirs(result_folder)
        result_location = os.path.join(result_folder, self.opts.prefix)
        keys = res_map.keys()
        for key in keys:
            filename = result_location + "-" + key + "-" + suffix +".txt"
            if not os.path.exists(filename):
                with open(filename, 'a'):
                    pass
            if self.current_run_id == 0:
                with open(filename, 'w'):
                    pass
            with open(filename, "a") as f:
                first = True
                for item in res_map[key]:
                    if first:
                        if suffix is "queried":
                            f.write("%d" % item)
                        else:
                            f.write("%.4f" % item)
                        first = False
                    else:
                        if suffix is "queried":
                            f.write(", %d" % item)
                        else:
                            f.write(", %.4f" % item)
                f.write("\n")
        logger.debug("saving done and now clearing the map")


def greedy_query_strategy(classifier, X):
    """We are now choosing the greedy query strategy as another baseline"""
    # measure the utility of each instance in the pool
    y_scores = classifier.predict_proba(X)
    y_scores = y_scores[:, 1]
    # select the indices of the instances to be queried
    query_idx = np.array([np.argmax(y_scores)])

    # return the indices and the instances
    return query_idx, X[query_idx]


def diverse_query_strategy(classifier, X, a_keyword_argument=42):
    """We are now choosing the diversified query strategy as baseline"""
    # measure the utility of each instance in the pool
    u_query_idx, x_query_idx = uncertainty_sampling(classifier, X, n_instances=10)
    # print ("selected 10 idx ", u_query_idx)
    # select the indices of the instances to be queried
    query_idx = filter_by_euclidean_distance(X, u_query_idx)
    # print ("selected diverse 3 idx ", query_idx)
    # return the indices and the instances
    return query_idx, X[query_idx]


def prefix_preparation(opts):
    """prepare the prefix for the experiment"""
    # query_strategy-ptrain-fbcount-base-
    # this pt is for experiment with full data
    if opts.train_years == -1:
        pt = 0
        prefix = str(opts.qsname) + "-ptrain-" + str(pt) + "-fbcount-" + str(opts.n_query) + "-base-RF" + "-starting-" \
                 + str(opts.starting) + "-weighted_update-" + str(opts.weighted_update) + "-nruns-" + str(opts.n_runs)

    else:
        prefix = str(opts.qsname) + "-ptrain-1" + "-fbcount-" + str(opts.n_query) + "-base-RF" + "-starting-" \
                 + str(opts.starting) + "-weighted_update-" + str(opts.weighted_update) + "-nruns-" + str(opts.n_runs)

    opts.prefix = prefix
    if not os.path.exists("./logs"):
        os.makedirs("./logs")
    if opts.log_file is "":
        opts.log_file = os.path.join("./logs", opts.prefix + ".log")
        print("log_file :\n" + opts.log_file)
    return opts


def process_options(opts):
    """Process the options based on the input parameters"""
    opts.results_f1 = {}
    opts.results_adr = {}
    opts.queried_indexes = {}
    if opts.query_strategy == 0:
        opts.query_strategy = uncertainty_sampling
        opts.qsname = "uncertainty_sampling"
    elif opts.query_strategy == 1:
        opts.query_strategy = greedy_query_strategy
        opts.qsname = "greedy_query_strategy"
    elif opts.query_strategy == 2:
        opts.query_strategy = diverse_query_strategy
        opts.qsname = "diverse_query_strategy"
    elif opts.query_strategy == 3:
        opts.query_strategy = entropy_sampling
        opts.qsname = "entropy_sampling"
    elif opts.query_strategy == 4:
        opts.query_strategy = margin_sampling
        opts.qsname = "margin_sampling"

    if opts.starting == 0:
        opts.starting_policy = "pretrain"
    elif opts.starting == 1:
        opts.starting_policy = "carry_feedback"
    elif opts.starting == 2:
        opts.starting_policy = "complete"

    if opts.weighted_update == 0:
        opts.weighted_update = False
    elif opts.weighted_update == 1:
        opts.weighted_update = True

    opts = prefix_preparation(opts)
    opts.save_fig_location = "./figures"
    opts.save_results_location = "./results"
    logger.debug("Figures will be save at %s" % opts.save_fig_location)

    return opts


def al_total_data_baseline():
    start = time.time()
    opts = parse_arguments()
    opts = process_options(opts)
    configure_logger(opts)
    for run_id in range(opts.n_runs):
        bsline_exp = BaselineExperiments(opts)
        bsline_exp.current_run_id = run_id
        bsline_exp.X_pool = bsline_exp.X_full
        bsline_exp.y_pool = bsline_exp.y_full
        init_idxs = bsline_exp.initial_data(n_labeled_data=100)
        bsline_exp.x_train = bsline_exp.X_full[init_idxs]
        bsline_exp.y_train = bsline_exp.y_full[init_idxs]

        bsline_exp.X_pool = np.delete(bsline_exp.X_pool, init_idxs, axis=0)
        bsline_exp.y_pool = np.delete(bsline_exp.y_pool, init_idxs)

        bsline_exp.prepare_learner(query_strategy=opts.query_strategy)  # default is "uncertainty sampling"


def al_baselines():
    """Active Learning Baselines"""
    start = time.time()
    opts = parse_arguments()

    opts = process_options(opts)
    configure_logger(opts)
    # logger.debug("started experiment")

    for run_id in range(opts.n_runs):
        bsline_exp = BaselineExperiments(opts)
        bsline_exp.current_run_id = run_id

        while not bsline_exp.stream.empty():
            # initializing the active learner
            bsline_exp.prepare_learner(query_strategy=opts.query_strategy)  # default is "uncertainty sampling"
            if bsline_exp.opts.train_years == -1:
                pool_data = bsline_exp.stream.read_next_from_stream(n=-1)
                init_idxs = bsline_exp.initial_data(n_labeled_data=100)
                logger.debug("pool data shape %d" % (pool_data.x.shape[0]))
                bsline_exp.x_train = pool_data.x[init_idxs]
                bsline_exp.y_train = pool_data.y[init_idxs]
                bsline_exp.prepare_learner(query_strategy=opts.query_strategy)  # default is "uncertainty sampling"
                pool_data.x = np.delete(pool_data.x, init_idxs, axis=0)
                pool_data.y = np.delete(pool_data.y, init_idxs)
                pool_data.ids = np.delete(pool_data.ids, init_idxs)
                bsline_exp.pool_data = pool_data
            else:
                pool_data = bsline_exp.stream.read_next_from_stream()
            X_pool = pool_data.x
            y_pool = pool_data.y
            id_pool = pool_data.ids
            tree_exceeding_count = None
            if opts.weighted_update is True:
                tree_exceeding_count = bsline_exp.compute_tree_exceeding(X_pool, y_pool)
            bsline_exp.evaluate_learner(feedbacks=0) # w/o feedbacks
            if bsline_exp.opts.with_feedback == 1:
                bsline_exp.train_with_feedbacks(opts, X_pool, y_pool, id_pool, tree_exceeding_count=tree_exceeding_count)
                logger.debug("After Feedbacks")
            # bsline_exp.evaluate_learner(feedbacks=bsline_exp.opts.n_asked_query)
            bsline_exp.cur_train_year += 1
            # break
        logger.debug("F1 scores")
        # print(bsline_exp.opts.results_f1)
        pprint(bsline_exp.opts.results_f1)
        print("ADR")
        # print(bsline_exp.opts.results_adr)
        pprint(bsline_exp.opts.results_adr)
        # bsline_exp.plot_results()
        bsline_exp.save_results()
        end = time.time()
        logger.debug("##Time needed for processing %d feed backs %.2f min(s)##" %(opts.n_query, (end - start)/60.0))
    logger.debug("##Finished all %d runs in %.2f min(s)##" % (opts.n_runs, (end - start) / 60.0))


if __name__ == '__main__':
    al_baselines()
