# Path: plur/eval/util.py
# def _get_lower_index(size, interval):
# def _get_upper_index(size, interval):
# def get_confidence_intervals(bootstraps, intervals):
#   def __str__(self, sort_keys=False):
# class Results():


import abc
import glob
import operator
import random
from plur.eval import util as eval_util
    Args:
      num_bootstraps: Number of times to run bootstrap resampling.
      seed: Random seed
      ci_intervals: list of confidence intervals desired in percentages.

    Returns:
      results: A results object containing metrics from task specific
          evaluators.
    """
    grouped_prediction_lines, target_lines = (
        self.read_prediction_and_target_file())

    results = self.evaluate_once(grouped_prediction_lines, target_lines)

    if num_bootstraps > 1:
      bootstraps = []
      rng = random.Random(seed)
      results.lower_confidence_interval = {}
      results.upper_confidence_interval = {}
      for unused_bootstrap_id in range(num_bootstraps):
        resamples = rng.choices(
            list(zip(grouped_prediction_lines, target_lines)),
            k=len(target_lines))
        resampled_grouped_prediction_lines = list(map(
            operator.itemgetter(0), resamples))
        resampled_target_lines = list(map(operator.itemgetter(1), resamples))
        bootstraps.append(
            self.evaluate_once(resampled_grouped_prediction_lines,
                               resampled_target_lines))
      # Add confidence intervals to results.

# Path: plur/eval/util.py
# def _get_lower_index(size, interval):
# def _get_upper_index(size, interval):
# def get_confidence_intervals(bootstraps, intervals):
#   def __str__(self, sort_keys=False):
# class Results():


# Path: plur/eval/plur_eval.py
import abc
import glob
import operator
import random
from plur.eval import util as eval_util

    Args:
      num_bootstraps: Number of times to run bootstrap resampling.
      seed: Random seed
      ci_intervals: list of confidence intervals desired in percentages.

    Returns:
      results: A results object containing metrics from task specific
          evaluators.
    """
    grouped_prediction_lines, target_lines = (
        self.read_prediction_and_target_file())

    results = self.evaluate_once(grouped_prediction_lines, target_lines)

    if num_bootstraps > 1:
      bootstraps = []
      rng = random.Random(seed)
      results.lower_confidence_interval = {}
      results.upper_confidence_interval = {}
      for unused_bootstrap_id in range(num_bootstraps):
        resamples = rng.choices(
            list(zip(grouped_prediction_lines, target_lines)),
            k=len(target_lines))
        resampled_grouped_prediction_lines = list(map(
            operator.itemgetter(0), resamples))
        resampled_target_lines = list(map(operator.itemgetter(1), resamples))
        bootstraps.append(
            self.evaluate_once(resampled_grouped_prediction_lines,
                               resampled_target_lines))
      # Add confidence intervals to results.

      cints = eval_util.get_confidence_intervals(