<commit_before>#!/usr/bin/env python
# vim: set sw=2 ts=2 softtabstop=2 expandtab:
"""
This script generates a "quantile" style plot
inspired by the quantile plots used SV-COMP.
"""
import argparse
import logging
import os
import pprint
import sys
import yaml
from br_util import FinalResultType, classifyResult, validateMappingFile
from count_by_label_then_result_type import groupByResultTypeThenLabel
import matplotlib.pyplot as plt

try:
  # Try to use libyaml which is faster
  from yaml import CLoader as Loader, CDumper as Dumper
except ImportError:
  # fall back on python implementation
  from yaml import Loader, Dumper

class ComputeScoreException(Exception):
  pass

class ScoredResultList:
  def __init__(self, listName, correctnessMapping):
    self.listName = listName
    self._correctnessMapping = correctnessMapping
    validateMappingFile(self._correctnessMapping)
    self._positiveResults = []
    self._zeroResults = []
    self._negativeResults = []

  def reset(self):
    self._positiveResults.clear()
    self._zeroResults.clear()
    self._negativeResult.clear()

  def addResult(self, r):
    score = self.computeScore(r)
    if score > 0:
      self._positiveResults.append( (score, r) )
    elif score == 0:
      self._zeroResults.append( r )
    else:
      assert score < 0
      self._negativeResults.append( (score, r) )

    return score

  def computeScore(self, r):
    # Get expected correctness
    programName = r['program']
    expectedCorrect = self._correctnessMapping[programName]['expected_correct']

    if expectedCorrect == None:
      # Can't handle results where the expected correctness is not known
      raise ComputeScoreException('Correctness not known for program {}'.format(programName))

    assert isinstance(expectedCorrect, bool)

    rType = classifyResult(r)

    # FIXME: We need to decide on what the most sensible scoring scheme is
    if expectedCorrect:
      if rType == FinalResultType.FULLY_EXPLORED:
        return 1
      elif rType == FinalResultType.BUG_FOUND:
        return -1 # Penalise wrong result
      elif rType == FinalResultType.BOUND_HIT:
        return 0
      else:
        return 0
    else:
      if rType == FinalResultType.FULLY_EXPLORED:
        return -1 # Penalise wrong result
      elif rType == FinalResultType.BUG_FOUND:
        return 1
      elif rType == FinalResultType.BOUND_HIT:
        return 0
      else:
        return 0


  @property
  def positiveResults(self):
    """
    Returns list that iterates over (score, result) tuples
    where the tuples are ordered (increasing) 'total_time' attribute
    """
    pResults = self._positiveResults.copy()
    pResults.sort(key=lambda e: e[1]['total_time'], reverse=False)
    return pResults

  @property
  def numOfPositiveResults(self):
    return len(self._positiveResults)

  @property
  def zeroResults(self):
    """
    Returns list that iterates over results that were
    given a zero score. The order of results is not defined
    """
    zResults = self._zeroResults.copy()
    return zResults

  @property
  def numOfZeroResults(self):
    return len(self._zeroResults)

  @property
  def negativeResults(self):
    """
    Returns list that iterates over (score, result) tuples
    where the tuples are ordered (increasing) 'total_time' attribute
    """
    nResults = self._negativeResults.copy()
    nResults.sort(key=lambda e: e[1]['total_time'], reverse=False)
    return nResults

  @property
  def numOfNegativeResults(self):
    return len(self._negativeResults)

  @property
  def totalNegativeScore(self):
    totalScore = 0
    for score, _ in self._negativeResults:
      assert isinstance(score, int)
      totalScore += score

    return totalScore

  @property
  def totalPositiveScore(self):
    totalScore = 0
    for score, _ in self._positiveResult:
      assert isinstance(score, int)
      totalScore += score

    return totalScore

  def totalScore(self):
    return self.totalNegativeScore + self.totalPositiveScore

def computeProgramNameToResults(data, resultListNames):
  output = { }

  for resultListName in resultListNames:
    for r in data[resultListName]:
      program = r['program']
      if program in output:
        output[program][resultListName] = r
      else:
        output[program] = { resultListName:r}

  for program, results in output.items():
    if len(results) != len(resultListNames):
      logging.warning('There are missing results for program {}'.format(program))

  return output


def main(args):
  parser = argparse.ArgumentParser(description=__doc__)
  parser.add_argument("-l","--log-level",type=str, default="info", dest="log_level", choices=['debug','info','warning','error'])
  parser.add_argument('label_mapping_file', type=argparse.FileType('r'))
  parser.add_argument('result_ymls', nargs='+', help='Input YAML files')
  parser.add_argument('--title', default="", type=str)
  parser.add_argument('--legend-name-map',dest='legend_name_map', default=None, type=str)
  parser.add_argument('--legend-position',dest='legend_position', choices=['outside_bottom', 'outside_right', 'inner', 'none'])

  actionGroup = parser.add_mutually_exclusive_group()
  actionGroup.add_argument('--ipython', action='store_true')
  actionGroup.add_argument('--pdf', help='Write graph to PDF')

  plotGroup = parser.add_mutually_exclusive_group()
  plotGroup.add_argument("--points", action='store_true')
  plotGroup.add_argument("--error-bars", action='store_true', dest='error_bars')

  pargs = parser.parse_args(args)

  logLevel = getattr(logging, pargs.log_level.upper(),None)
  logging.basicConfig(level=logLevel)

  if len(pargs.result_ymls) < 2:
    logging.error('Need at least two YAML files')

  if pargs.pdf != None:
    if not pargs.pdf.endswith('.pdf'):
      logging.error('--pdf argument must end with .pdf')<commit_msg>Set a default legend position else we hit assertion failure<commit_after>#!/usr/bin/env python
# vim: set sw=2 ts=2 softtabstop=2 expandtab:
"""
This script generates a "quantile" style plot
inspired by the quantile plots used SV-COMP.
"""
import argparse
import logging
import os
import pprint
import sys
import yaml
from br_util import FinalResultType, classifyResult, validateMappingFile
from count_by_label_then_result_type import groupByResultTypeThenLabel
import matplotlib.pyplot as plt

try:
  # Try to use libyaml which is faster
  from yaml import CLoader as Loader, CDumper as Dumper
except ImportError:
  # fall back on python implementation
  from yaml import Loader, Dumper

class ComputeScoreException(Exception):
  pass

class ScoredResultList:
  def __init__(self, listName, correctnessMapping):
    self.listName = listName
    self._correctnessMapping = correctnessMapping
    validateMappingFile(self._correctnessMapping)
    self._positiveResults = []
    self._zeroResults = []
    self._negativeResults = []

  def reset(self):
    self._positiveResults.clear()
    self._zeroResults.clear()
    self._negativeResult.clear()

  def addResult(self, r):
    score = self.computeScore(r)
    if score > 0:
      self._positiveResults.append( (score, r) )
    elif score == 0:
      self._zeroResults.append( r )
    else:
      assert score < 0
      self._negativeResults.append( (score, r) )

    return score

  def computeScore(self, r):
    # Get expected correctness
    programName = r['program']
    expectedCorrect = self._correctnessMapping[programName]['expected_correct']

    if expectedCorrect == None:
      # Can't handle results where the expected correctness is not known
      raise ComputeScoreException('Correctness not known for program {}'.format(programName))

    assert isinstance(expectedCorrect, bool)

    rType = classifyResult(r)

    # FIXME: We need to decide on what the most sensible scoring scheme is
    if expectedCorrect:
      if rType == FinalResultType.FULLY_EXPLORED:
        return 1
      elif rType == FinalResultType.BUG_FOUND:
        return -1 # Penalise wrong result
      elif rType == FinalResultType.BOUND_HIT:
        return 0
      else:
        return 0
    else:
      if rType == FinalResultType.FULLY_EXPLORED:
        return -1 # Penalise wrong result
      elif rType == FinalResultType.BUG_FOUND:
        return 1
      elif rType == FinalResultType.BOUND_HIT:
        return 0
      else:
        return 0


  @property
  def positiveResults(self):
    """
    Returns list that iterates over (score, result) tuples
    where the tuples are ordered (increasing) 'total_time' attribute
    """
    pResults = self._positiveResults.copy()
    pResults.sort(key=lambda e: e[1]['total_time'], reverse=False)
    return pResults

  @property
  def numOfPositiveResults(self):
    return len(self._positiveResults)

  @property
  def zeroResults(self):
    """
    Returns list that iterates over results that were
    given a zero score. The order of results is not defined
    """
    zResults = self._zeroResults.copy()
    return zResults

  @property
  def numOfZeroResults(self):
    return len(self._zeroResults)

  @property
  def negativeResults(self):
    """
    Returns list that iterates over (score, result) tuples
    where the tuples are ordered (increasing) 'total_time' attribute
    """
    nResults = self._negativeResults.copy()
    nResults.sort(key=lambda e: e[1]['total_time'], reverse=False)
    return nResults

  @property
  def numOfNegativeResults(self):
    return len(self._negativeResults)

  @property
  def totalNegativeScore(self):
    totalScore = 0
    for score, _ in self._negativeResults:
      assert isinstance(score, int)
      totalScore += score

    return totalScore

  @property
  def totalPositiveScore(self):
    totalScore = 0
    for score, _ in self._positiveResult:
      assert isinstance(score, int)
      totalScore += score

    return totalScore

  def totalScore(self):
    return self.totalNegativeScore + self.totalPositiveScore

def computeProgramNameToResults(data, resultListNames):
  output = { }

  for resultListName in resultListNames:
    for r in data[resultListName]:
      program = r['program']
      if program in output:
        output[program][resultListName] = r
      else:
        output[program] = { resultListName:r}

  for program, results in output.items():
    if len(results) != len(resultListNames):
      logging.warning('There are missing results for program {}'.format(program))

  return output


def main(args):
  parser = argparse.ArgumentParser(description=__doc__)
  parser.add_argument("-l","--log-level",type=str, default="info", dest="log_level", choices=['debug','info','warning','error'])
  parser.add_argument('label_mapping_file', type=argparse.FileType('r'))
  parser.add_argument('result_ymls', nargs='+', help='Input YAML files')
  parser.add_argument('--title', default="", type=str)
  parser.add_argument('--legend-name-map',dest='legend_name_map', default=None, type=str)
  parser.add_argument('--legend-position',dest='legend_position', default='outside_bottom', choices=['outside_bottom', 'outside_right', 'inner', 'none'])

  actionGroup = parser.add_mutually_exclusive_group()
  actionGroup.add_argument('--ipython', action='store_true')
  actionGroup.add_argument('--pdf', help='Write graph to PDF')

  plotGroup = parser.add_mutually_exclusive_group()
  plotGroup.add_argument("--points", action='store_true')
  plotGroup.add_argument("--error-bars", action='store_true', dest='error_bars')

  pargs = parser.parse_args(args)

  logLevel = getattr(logging, pargs.log_level.upper(),None)
  logging.basicConfig(level=logLevel)

  if len(pargs.result_ymls) < 2:
    logging.error('Need at least two YAML files')

  if pargs.pdf != None:
    if not pargs.pdf.endswith('.pdf'):
      logging.error('--pdf argument must end with .pdf')
{'fraction_non_alphanumeric': 0.10282121006118287, 'fraction_numerical': 0.003229095853161115, 'mean_word_length': 3.865233567589913, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1519320', 'n_tokens_mistral': 3554, 'n_tokens_neox': 3381, 'n_words': 925}