<commit_before>#!/usr/bin/env python
# vim: set sw=2 ts=2 softtabstop=2 expandtab:
import argparse
import os
import logging
import sys
import yaml
from br_util import FinalResultType, classifyResult, validateMappingFile

try:
  # Try to use libyaml which is faster
  from yaml import CLoader as Loader, CDumper as Dumper
except ImportError:
  # fall back on python implementation
  from yaml import Loader, Dumper


def main(args):
  parser = argparse.ArgumentParser()
  parser.add_argument("-l","--log-level",type=str, default="info", dest="log_level", choices=['debug','info','warning','error'])
  parser.add_argument("--uncommon", action='store_true')
  parser.add_argument("-s", "--proper-supersets", dest='proper_supersets', action='store_true', help='show information about results in supersets')
  parser.add_argument("-e", "--label-mapping", default=None, type=argparse.FileType('r'), dest="label_mapping",
    help="Group by expected result type from a mapping file")
  parser.add_argument('result_ymls', nargs='+', help='Input YAML files')
  pargs = parser.parse_args(args)

  logLevel = getattr(logging, pargs.log_level.upper(),None)
  logging.basicConfig(level=logLevel)

  correctnessMapping = None
  if pargs.label_mapping != None:
    logging.info('Loading correctness mapping file')
    correctnessMapping = yaml.load(pargs.label_mapping, Loader=Loader)
    validateMappingFile(correctnessMapping)
    benchmarkLabels = ['correct', 'incorrect', 'unknown']
    def labelFieldToString(expected_correct):
      if expected_correct:
        return 'correct'
      elif expected_correct == False:
        return 'incorrect'
      elif expected_correct == None:
        return 'unknown'
      else:
        raise Exception('"expected_correct" must be None or boolean')
  else:
    benchmarkLabels = ['all']

  # Check that each yml file exists
  categorised = { }
  data = { }
  resultSetLabels = [ ]
  for f in pargs.result_ymls:
    if not os.path.exists(f):
      logging.error('YAML file {} does not exist'.format(f))
      return 1

    # Compute result set label
    resultSetLabel = os.path.basename(f)
    resultSetLabels.append(resultSetLabel)
    if resultSetLabel in data:
      logging.error('Can\'t use {} as label name because it is already used'.format(resultSetLabel))
      return 1

    data[resultSetLabel] = None # Will be filled with loaded YAML data

    # Initialise data
    categorised[resultSetLabel] = {}
    for benchmarkLabel in benchmarkLabels:
      categorised[resultSetLabel][benchmarkLabel] = {}
      for name, _ in FinalResultType.__members__.items():
        categorised[resultSetLabel][benchmarkLabel][name] = { 'raw':[], 'program_set': set() }

  # Now load YAML
  length = 0
  for f in pargs.result_ymls:
    logging.info('Loading YAML file {}'.format(f))
    with open(f, 'r') as openFile:
      results = yaml.load(openFile, Loader=Loader)
    logging.info('Loading complete')
    assert isinstance(results, list)
    resultSetLabel = os.path.basename(f)
    data[resultSetLabel] = results
    length = len(results)

  # Check the lengths are the same
  for name, rList in data.items():
    if len(rList) != length:
      logging.error('There is a length mismatch for {}, expected {} entries but was'.format(name, length, len(rList)))
      return 1

  programToRawResultMap = { }
  # Put results into buckets based on labels
  for resultSetLabel in resultSetLabels:
    for r in data[resultSetLabel]:
      if not 'bug_found' in r:
        logging.error('Key "bug_found" not in result')
        return 1

      if pargs.label_mapping != None:
        expectedCorrect = correctnessMapping[ r['program'] ]['expected_correct']<commit_msg>Remove restriction where the basename of a file to a result file was used as the result set label. This restriction causes problems if you have something like<commit_after>#!/usr/bin/env python
# vim: set sw=2 ts=2 softtabstop=2 expandtab:
import argparse
import os
import logging
import sys
import yaml
from br_util import FinalResultType, classifyResult, validateMappingFile

try:
  # Try to use libyaml which is faster
  from yaml import CLoader as Loader, CDumper as Dumper
except ImportError:
  # fall back on python implementation
  from yaml import Loader, Dumper


def main(args):
  parser = argparse.ArgumentParser()
  parser.add_argument("-l","--log-level",type=str, default="info", dest="log_level", choices=['debug','info','warning','error'])
  parser.add_argument("--uncommon", action='store_true')
  parser.add_argument("-s", "--proper-supersets", dest='proper_supersets', action='store_true', help='show information about results in supersets')
  parser.add_argument("-e", "--label-mapping", default=None, type=argparse.FileType('r'), dest="label_mapping",
    help="Group by expected result type from a mapping file")
  parser.add_argument('result_ymls', nargs='+', help='Input YAML files')
  pargs = parser.parse_args(args)

  logLevel = getattr(logging, pargs.log_level.upper(),None)
  logging.basicConfig(level=logLevel)

  correctnessMapping = None
  if pargs.label_mapping != None:
    logging.info('Loading correctness mapping file')
    correctnessMapping = yaml.load(pargs.label_mapping, Loader=Loader)
    validateMappingFile(correctnessMapping)
    benchmarkLabels = ['correct', 'incorrect', 'unknown']
    def labelFieldToString(expected_correct):
      if expected_correct:
        return 'correct'
      elif expected_correct == False:
        return 'incorrect'
      elif expected_correct == None:
        return 'unknown'
      else:
        raise Exception('"expected_correct" must be None or boolean')
  else:
    benchmarkLabels = ['all']

  # Check that each yml file exists
  categorised = { }
  data = { }
  resultSetLabels = [ ]
  for f in pargs.result_ymls:
    if not os.path.exists(f):
      logging.error('YAML file {} does not exist'.format(f))
      return 1

    # Compute result set label
    resultSetLabel = f
    resultSetLabels.append(resultSetLabel)
    if resultSetLabel in data:
      logging.error('Can\'t use {} as label name because it is already used'.format(resultSetLabel))
      return 1

    data[resultSetLabel] = None # Will be filled with loaded YAML data

    # Initialise data
    categorised[resultSetLabel] = {}
    for benchmarkLabel in benchmarkLabels:
      categorised[resultSetLabel][benchmarkLabel] = {}
      for name, _ in FinalResultType.__members__.items():
        categorised[resultSetLabel][benchmarkLabel][name] = { 'raw':[], 'program_set': set() }

  # Now load YAML
  length = 0
  for f in pargs.result_ymls:
    logging.info('Loading YAML file {}'.format(f))
    with open(f, 'r') as openFile:
      results = yaml.load(openFile, Loader=Loader)
    logging.info('Loading complete')
    assert isinstance(results, list)
    resultSetLabel = f
    data[resultSetLabel] = results
    length = len(results)

  # Check the lengths are the same
  for name, rList in data.items():
    if len(rList) != length:
      logging.error('There is a length mismatch for {}, expected {} entries but was'.format(name, length, len(rList)))
      return 1

  programToRawResultMap = { }
  # Put results into buckets based on labels
  for resultSetLabel in resultSetLabels:
    for r in data[resultSetLabel]:
      if not 'bug_found' in r:
        logging.error('Key "bug_found" not in result')
        return 1

      if pargs.label_mapping != None:
        expectedCorrect = correctnessMapping[ r['program'] ]['expected_correct']
{'fraction_non_alphanumeric': 0.11102204408817636, 'fraction_numerical': 0.0021376085504342017, 'mean_word_length': 3.9543348775645266, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5512726', 'n_tokens_mistral': 2236, 'n_tokens_neox': 2183, 'n_words': 642}