<commit_before>"""
Evaluate the prediction results
"""
import numpy as np
import pandas as pd
from pathlib import Path
import re, os
from PIL import Image
from breastcancer.visualization import Shape, add_mark
from sklearn.cluster import DBSCAN

GROUND_TRUTH_FILE_ID_RE = "\d+/\d+.csv"

def prepare_f1_inputs(prediction, ground_truth, threshold=30, file_id=None):
  """ Prepare the input variables (TP, FP, and PN) for computing F1
  score.

  TP: its Eucledian distance to a ground truth location is less than
    threshold;
  FP: not within the threshold distance of a ground truth location;
  FN: all ground truth locations that do not have a detection within
    the threshold.
  file_id: file id for the input prediction and ground truth data,
    which should have the same file id

  Args:
    prediction: prediction result, a list of point locations, e.g.
      [(r0, c0, p0), (r1, c1, p1), (r2, c2, p2), ...].
    ground_truth: groud truth data, a list of point locations, e.g.
      [(r0, c0), (r1, c1), (r2, c2), ...].
    threshold: Eucledian distance to see if the predict and ground truth
      points are at the same circle.

  Return:
    A tuple of (FP, TP, FN), where TP will be a list of list of tuples like
    [[(file_id0, r0, c0, prob0, 'TP'),..., (file_id_n, r_n, c_n, prob_n,
     'TP')], [.....], ......, [......]], FP will be like [(file_id, r,
     c, prob, 'FP'), .....], FN will be like [(file_id, r, c, prob,
     'FN'), .....]
  """
  if len(prediction) == 0 and len(ground_truth) != 0:
    FP = []
    TP = []
    FN = [(file_id, r, c, 999, 'FN') for (r, c) in ground_truth]
  elif len(prediction) != 0 and len(ground_truth) == 0:
    FP = [(file_id, r, c, p, 'FP') for (r, c, p) in prediction]
    TP = []
    FN = []
  elif len(prediction) != 0 and len(ground_truth) != 0:
    # initialize the values.
    FP = [(file_id, r, c, p, 'FP') for (r, c, p) in prediction]
    TP = []
    FN = []
    # check which points in the ground truth are located with each other
    # less than `threshold`.
    for gt_r, gt_c in ground_truth:
      for r, c in ground_truth:
        dist = np.sqrt((gt_r - r) ** 2 + (gt_c - c) ** 2)
        if dist > 0 and dist < threshold * 2:
          print(f"Point ({r} , {c}) has multiple points in the circle")

    for gt_r, gt_c in ground_truth:
      # if several points fall within a single ground truth location,
      # they will be counted as one true positive. Here we use a list
      # to collect this kind of points for each ground truth point.
      tp = []
      for pred_r, pred_c, pre_prob in prediction:
        dist = np.sqrt((gt_r - pred_r) ** 2 + (gt_c - pred_c) ** 2)
        if dist <= threshold:
          tp.append((file_id, pred_r, pred_c, pre_prob, 'TP'))
          # remove the TP point from the FP list
          if (file_id, pred_r, pred_c, pre_prob, 'FP') in FP:
            FP.remove((file_id, pred_r, pred_c, pre_prob, 'FP'))
      if len(tp) == 0:
        # if no points fall within this ground truth point, the ground
        # truth point will be treated as FN. 999 does not mean the
        # prediction probability, it will indicate this is FN
        FN.append((file_id, gt_r, gt_c, 999, 'FN'))
      else:
        # the reason of using .append() here instead of += is to count
        # several points falling in a single ground truth location as
        # one true positive
        TP.append(tp)
  else:
    FP = []
    TP = []
    FN = []

  return (FP, TP, FN)

def compute_f1(FP, TP, FN):
  """ Compute the F1 score. The calculation equation can be found at
  http://amida13.isi.uu.nl/?q=node/4.

  Args:
    FP: a list of false positive predictions.
    TP: a list of true positive predictions.
    FN: a list of false negative predictions.

  Return:
    F1 score.
  """

  precision = len(TP) / (len(TP) + len(FP)) if len(TP) + len(FP) > 0 else 0
  recall = len(TP) / (len(TP) + len(FN)) if len(TP) + len(FN) > 0 else 0
  f1 = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0

  print (f"TP: {len(TP)}; FP: {len(FP)}; FN: {len(FN)}")
  print (f"precision: {precision}; recall: {recall}")

  return f1

def list_files(dir, file_suffix):
  """ recursively list all the files that have the same input file
    suffix under the input directory.

  Args
    dir: file directory.
    file_suffix: file suffix.

  Return:
    a list of file path.
  """
  dir = Path(dir)
  files = [str(x) for x in dir.rglob(file_suffix)]
  return files

def get_file_id(files, file_id_re):
  """ get the file id using the file id regular expression
  Args:
    files: list of input file paths
    file_id_re: regular expression string used to detect the file ID
      from the file path

  Return:
    a dictionary of file id and its full file path
  """
  id_files = {re.findall(file_id_re, x)[0]: x for x in files}

  # the id generated in the above will be like "/12/01.csv". The prefix
  # `/` and suffix '.cvs' need to be removed because this id will be
  # used as the part of file path in other places.
  id_files_update = {id[0 : len(id) - 4] : file for id, file in id_files.items()}
  return id_files_update


def get_locations_from_csv(file, hasHeader=False, hasProb=True):<commit_msg>Fix the errors when getting file ids (#29)<commit_after>"""
Evaluate the prediction results
"""
import numpy as np
import pandas as pd
from pathlib import Path
import re, os
from PIL import Image
from breastcancer.visualization import Shape, add_mark
from sklearn.cluster import DBSCAN

GROUND_TRUTH_FILE_ID_RE = "(\d+/\d+)[.csv|.tif]"

def prepare_f1_inputs(prediction, ground_truth, threshold=30, file_id=None):
  """ Prepare the input variables (TP, FP, and PN) for computing F1
  score.

  TP: its Eucledian distance to a ground truth location is less than
    threshold;
  FP: not within the threshold distance of a ground truth location;
  FN: all ground truth locations that do not have a detection within
    the threshold.
  file_id: file id for the input prediction and ground truth data,
    which should have the same file id

  Args:
    prediction: prediction result, a list of point locations, e.g.
      [(r0, c0, p0), (r1, c1, p1), (r2, c2, p2), ...].
    ground_truth: groud truth data, a list of point locations, e.g.
      [(r0, c0), (r1, c1), (r2, c2), ...].
    threshold: Eucledian distance to see if the predict and ground truth
      points are at the same circle.

  Return:
    A tuple of (FP, TP, FN), where TP will be a list of list of tuples like
    [[(file_id0, r0, c0, prob0, 'TP'),..., (file_id_n, r_n, c_n, prob_n,
     'TP')], [.....], ......, [......]], FP will be like [(file_id, r,
     c, prob, 'FP'), .....], FN will be like [(file_id, r, c, prob,
     'FN'), .....]
  """
  if len(prediction) == 0 and len(ground_truth) != 0:
    FP = []
    TP = []
    FN = [(file_id, r, c, 999, 'FN') for (r, c) in ground_truth]
  elif len(prediction) != 0 and len(ground_truth) == 0:
    FP = [(file_id, r, c, p, 'FP') for (r, c, p) in prediction]
    TP = []
    FN = []
  elif len(prediction) != 0 and len(ground_truth) != 0:
    # initialize the values.
    FP = [(file_id, r, c, p, 'FP') for (r, c, p) in prediction]
    TP = []
    FN = []
    # check which points in the ground truth are located with each other
    # less than `threshold`.
    for gt_r, gt_c in ground_truth:
      for r, c in ground_truth:
        dist = np.sqrt((gt_r - r) ** 2 + (gt_c - c) ** 2)
        if dist > 0 and dist < threshold * 2:
          print(f"Point ({r} , {c}) has multiple points in the circle")

    for gt_r, gt_c in ground_truth:
      # if several points fall within a single ground truth location,
      # they will be counted as one true positive. Here we use a list
      # to collect this kind of points for each ground truth point.
      tp = []
      for pred_r, pred_c, pre_prob in prediction:
        dist = np.sqrt((gt_r - pred_r) ** 2 + (gt_c - pred_c) ** 2)
        if dist <= threshold:
          tp.append((file_id, pred_r, pred_c, pre_prob, 'TP'))
          # remove the TP point from the FP list
          if (file_id, pred_r, pred_c, pre_prob, 'FP') in FP:
            FP.remove((file_id, pred_r, pred_c, pre_prob, 'FP'))
      if len(tp) == 0:
        # if no points fall within this ground truth point, the ground
        # truth point will be treated as FN. 999 does not mean the
        # prediction probability, it will indicate this is FN
        FN.append((file_id, gt_r, gt_c, 999, 'FN'))
      else:
        # the reason of using .append() here instead of += is to count
        # several points falling in a single ground truth location as
        # one true positive
        TP.append(tp)
  else:
    FP = []
    TP = []
    FN = []

  return (FP, TP, FN)

def compute_f1(FP, TP, FN):
  """ Compute the F1 score. The calculation equation can be found at
  http://amida13.isi.uu.nl/?q=node/4.

  Args:
    FP: a list of false positive predictions.
    TP: a list of true positive predictions.
    FN: a list of false negative predictions.

  Return:
    F1 score.
  """

  precision = len(TP) / (len(TP) + len(FP)) if len(TP) + len(FP) > 0 else 0
  recall = len(TP) / (len(TP) + len(FN)) if len(TP) + len(FN) > 0 else 0
  f1 = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0

  print (f"TP: {len(TP)}; FP: {len(FP)}; FN: {len(FN)}")
  print (f"precision: {precision}; recall: {recall}")

  return f1

def list_files(dir, file_suffix):
  """ recursively list all the files that have the same input file
    suffix under the input directory.

  Args
    dir: file directory.
    file_suffix: file suffix.

  Return:
    a list of file path.
  """
  dir = Path(dir)
  files = [str(x) for x in dir.rglob(file_suffix)]
  return files

def get_file_id(files, file_id_re):
  """ get the file id using the file id regular expression
  Args:
    files: list of input file paths
    file_id_re: regular expression string used to detect the file ID
      from the file path

  Return:
    a dictionary of file id and its full file path
  """
  id_files = {re.findall(file_id_re, x)[0]: x for x in files}
  return id_files


def get_locations_from_csv(file, hasHeader=False, hasProb=True):
{'fraction_non_alphanumeric': 0.12180495477782147, 'fraction_numerical': 0.012780180888714117, 'mean_word_length': 2.741449062155204, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4708050', 'n_tokens_mistral': 3616, 'n_tokens_neox': 3372, 'n_words': 1368}