# Path: algo/const.py
# NUM_SETS = 5
# 
# CONF_QT = 0.975
# 
# RANK_SIZE = 5
# 
# REP = 3
# 
# Path: perf/metrics.py
# def calculate_rmse(result, truth):
#   """ Calculates RMSE (Root Mean Squared Error) of a prediction task.
# 
#       Args:
#         result: a list of predicted values assigned to the instances.
#         truth: a list with the corrected values.
# 
#       Returns:
#         A real value representing the RMSE.
#   """
#   errors = calculate_squared_errors(result, truth)
#   rmse = sqrt(float(sum(errors)) / len(errors))
#   return rmse
# 
# def calculate_ndcg(pred, truth, k):
#   """ Calculates the normalized Discounted Cumulative Gain until certain
#       position K (nDCG@K). This metrics normalizes the DCG by dividing to the
#       DCG of the best ranking, which sorts considering the true relevances.
# 
#       Observation:
#       - If the best DGC equals 0 (which may occur with a ranking with all
#         relevances equal to zero), then the value of the nDCG is considered to
#         be 1.0, since there is no way to be worse than the best ranking.
# 
#       Args:
#         pred: list of floats with predicted relevances.
#         truth: list of floats (or integers) with true relevances.
#         k: an integer with the limit position of the ranking to calculate the
#       score.
# 
#       Returns:
#         A float in range [0, 1] with nDCG@K score.
#   """
#   curr_dcg = calculate_dcg(pred, truth, k)
#   best_dcg = calculate_dcg(truth, truth, k)
#   return curr_dcg / best_dcg if best_dcg != 0.0 else 0.0 
#       # zero-relevance on top is bad (conservative)
# 
# def calculate_ap(pred, truth, k):
#   """ Gets the average precision.
# 
#       Observation:
#       - We consider the top-k in true ranking as relevant items. Consquently,
#       there are always k relevant items.
# 
#       Args:
#         pred: list of floats with predicted relevances.
#         truth: list of floats (or integers) with true relevances.
#         k: integer with the size of ranking to consider.
# 
#       Returns:
#         A real value with the average precition at k.      
#   """
#   i_ranking = sorted(range(len(truth)), key=lambda p: pred[p], reverse=True)
#  # p_ranking = sorted(range(len(pred)), key=lambda p: pred[p], reverse=True)
#  # top = t_ranking[:k]
#  # worst_rel = truth[top[-1]]
#  # for pos in xrange(k, len(t_ranking)):
#  #   index = t_ranking[pos]
#  #   if truth[index] == worst_rel:
#  #     top.append(index)
#  #   else:
#  #     break
#   avg_prec = 0.0 
#   acc = 0.0
#   for i in xrange(k):
#    # if p_ranking[i] in top:
#     if truth[i_ranking[i]] == 5:
#       acc += 1.0
#       avg_prec += (acc / (i+1))
#   avg_prec /= k
#   return avg_prec 
# 
# def calculate_err(pred, truth):
#   """ Gets the expected reciprocal rank.
# 
#       Args:
#         pred: list of floats with predicted relevances.
#         truth: list of floats (or integers) with true relevances.
# 
#       Returns:
#         A real value with the ERR.
#   """
#   p_ranking = sorted(range(len(pred)), key=lambda p: pred[p], reverse=True)
#   err = 0.0 
#   p = 1.0
#   for i in xrange(len(p_ranking)):
#     relevance = truth[p_ranking[i]]
#     local_p = probability_stop(relevance)
#     err += p * local_p / (i + 1.0)
#     p *= (1.0 - local_p)
#   return err 


from math import sqrt
from pickle import load
from os.path import isfile
from sys import argv, exit
from numpy import mean, std, array
from scipy.stats import t 
from sklearn.metrics import average_precision_score
from algo.const import NUM_SETS, CONF_QT, RANK_SIZE, REP
from perf.metrics import calculate_rmse, calculate_ndcg, calculate_ap, \
    calculate_err
  """ Main function.

      Args:
        None.

      Returns:
        None. Results are output to 
      /out/res/<predictor>-<set_type>-<set_index>.dat file and to stdout.
  """
  predictor, set_type, rep = parse_args()
  rmse = []
  ndcg = []
  err = []
  output = open('out/res/%s-%s.dat' % (predictor, set_type), 'w')
  for i in xrange(NUM_SETS):
    rmse_sum = 0
    ndcg_sum = array([0] * RANK_SIZE)
    err_sum = 0
    repetitions = REP if rep else 1
    print >> output, 'Results on Set Type %d' % (i+1)
    for j in xrange(repetitions):
      votes, pred, reviews = load_data(predictor, set_type, i, j)
      rmse_sum += evaluate_regression(pred, votes, output)
      ndcg_sum = ndcg_sum + array(evaluate_ranking(pred, votes, reviews, output))
      err_sum += evaluate_ranking_err(pred, votes, reviews, output)
    rmse.append(rmse_sum / repetitions)
    ndcg.append(ndcg_sum / repetitions)
    err.append(err_sum / repetitions)
    print >> output, '-----'
  if rep:

# Path: algo/const.py
# NUM_SETS = 5
# 
# CONF_QT = 0.975
# 
# RANK_SIZE = 5
# 
# REP = 3
# 
# Path: perf/metrics.py
# def calculate_rmse(result, truth):
#   """ Calculates RMSE (Root Mean Squared Error) of a prediction task.
# 
#       Args:
#         result: a list of predicted values assigned to the instances.
#         truth: a list with the corrected values.
# 
#       Returns:
#         A real value representing the RMSE.
#   """
#   errors = calculate_squared_errors(result, truth)
#   rmse = sqrt(float(sum(errors)) / len(errors))
#   return rmse
# 
# def calculate_ndcg(pred, truth, k):
#   """ Calculates the normalized Discounted Cumulative Gain until certain
#       position K (nDCG@K). This metrics normalizes the DCG by dividing to the
#       DCG of the best ranking, which sorts considering the true relevances.
# 
#       Observation:
#       - If the best DGC equals 0 (which may occur with a ranking with all
#         relevances equal to zero), then the value of the nDCG is considered to
#         be 1.0, since there is no way to be worse than the best ranking.
# 
#       Args:
#         pred: list of floats with predicted relevances.
#         truth: list of floats (or integers) with true relevances.
#         k: an integer with the limit position of the ranking to calculate the
#       score.
# 
#       Returns:
#         A float in range [0, 1] with nDCG@K score.
#   """
#   curr_dcg = calculate_dcg(pred, truth, k)
#   best_dcg = calculate_dcg(truth, truth, k)
#   return curr_dcg / best_dcg if best_dcg != 0.0 else 0.0 
#       # zero-relevance on top is bad (conservative)
# 
# def calculate_ap(pred, truth, k):
#   """ Gets the average precision.
# 
#       Observation:
#       - We consider the top-k in true ranking as relevant items. Consquently,
#       there are always k relevant items.
# 
#       Args:
#         pred: list of floats with predicted relevances.
#         truth: list of floats (or integers) with true relevances.
#         k: integer with the size of ranking to consider.
# 
#       Returns:
#         A real value with the average precition at k.      
#   """
#   i_ranking = sorted(range(len(truth)), key=lambda p: pred[p], reverse=True)
#  # p_ranking = sorted(range(len(pred)), key=lambda p: pred[p], reverse=True)
#  # top = t_ranking[:k]
#  # worst_rel = truth[top[-1]]
#  # for pos in xrange(k, len(t_ranking)):
#  #   index = t_ranking[pos]
#  #   if truth[index] == worst_rel:
#  #     top.append(index)
#  #   else:
#  #     break
#   avg_prec = 0.0 
#   acc = 0.0
#   for i in xrange(k):
#    # if p_ranking[i] in top:
#     if truth[i_ranking[i]] == 5:
#       acc += 1.0
#       avg_prec += (acc / (i+1))
#   avg_prec /= k
#   return avg_prec 
# 
# def calculate_err(pred, truth):
#   """ Gets the expected reciprocal rank.
# 
#       Args:
#         pred: list of floats with predicted relevances.
#         truth: list of floats (or integers) with true relevances.
# 
#       Returns:
#         A real value with the ERR.
#   """
#   p_ranking = sorted(range(len(pred)), key=lambda p: pred[p], reverse=True)
#   err = 0.0 
#   p = 1.0
#   for i in xrange(len(p_ranking)):
#     relevance = truth[p_ranking[i]]
#     local_p = probability_stop(relevance)
#     err += p * local_p / (i + 1.0)
#     p *= (1.0 - local_p)
#   return err 


# Path: perf/main.py
from math import sqrt
from pickle import load
from os.path import isfile
from sys import argv, exit
from numpy import mean, std, array
from scipy.stats import t 
from sklearn.metrics import average_precision_score
from algo.const import NUM_SETS, CONF_QT, RANK_SIZE, REP
from perf.metrics import calculate_rmse, calculate_ndcg, calculate_ap, \
    calculate_err

  """ Main function.

      Args:
        None.

      Returns:
        None. Results are output to 
      /out/res/<predictor>-<set_type>-<set_index>.dat file and to stdout.
  """
  predictor, set_type, rep = parse_args()
  rmse = []
  ndcg = []
  err = []
  output = open('out/res/%s-%s.dat' % (predictor, set_type), 'w')
  for i in xrange(NUM_SETS):
    rmse_sum = 0
    ndcg_sum = array([0] * RANK_SIZE)
    err_sum = 0
    repetitions = REP if rep else 1
    print >> output, 'Results on Set Type %d' % (i+1)
    for j in xrange(repetitions):
      votes, pred, reviews = load_data(predictor, set_type, i, j)
      rmse_sum += evaluate_regression(pred, votes, output)
      ndcg_sum = ndcg_sum + array(evaluate_ranking(pred, votes, reviews, output))
      err_sum += evaluate_ranking_err(pred, votes, reviews, output)
    rmse.append(rmse_sum / repetitions)
    ndcg.append(ndcg_sum / repetitions)
    err.append(err_sum / repetitions)
    print >> output, '-----'
  if rep:

    for i in xrange(NUM_SETS):