<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Measure performance for a run file, an intervalType (day,week, all), and a judgmentLevel (1 or 2).

Creates the output next to the input file, appending "-eval-week.tsv" (for weeks)

 example usage
 python measure-slice.py --intervalType week --judgmentLevel 1 -f ~/kba-evaluation/taia/data/umass-runs/UMass_CIIR-PC_RM20_1500.gz
"""

import os.path
import numpy as np
from numpy.lib import recfunctions
import gzip
from metrics import *
import string
from utils import *
from truthutil import *
from argparse import ArgumentParser
from targetentities import *
import csv
import tempfile


DEBUG = False

metrics = {
    'nDCG@R': ndcgREval, 'Prec@R': precREval, 'MAP': mapEval}

runFile = '~/kba-evaluation/taia/data/umass-runs/UMass_CIIR-PC_RM20_1500.gz'

parser = ArgumentParser()
parser.add_argument('--intervalType', help='slice segmentation: day, week, or all', default='all')
parser.add_argument('--judgmentLevel', type=int, help='Judgement level', default=1)
parser.add_argument('--trec_eval', action='store_true', help='Also create output for evaluation with trec_eval',
                    default=False)
parser.add_argument('-f', '--runFile', metavar='FILE', default=runFile)
args = parser.parse_args()

judgmentLevel = args.judgmentLevel
intervalType = args.intervalType
runFile = args.runFile
DUMP_TREC_EVAL = args.trec_eval  # also create output for trec_eval.

entry_dtype = np.dtype([('runname', '50a'), ('docid', '50a'), ('query', '50a'),
                        ('confidence', 'd4')])
judg_dtype = np.dtype([('docid', '50a'), ('query', '50a'),
                       ('label', 'd4')])
eval_dtype = np.dtype(
    [('team', '50a'), ('runname', '50a'), ('query', '50a'), ('intervalLow', 'd4'), ('intervalUp', 'd4'),
     ('unjudged', '50a'), ('judgmentLevel', 'd4'), ('metric', '50a'), ('value', 'f4')])


class Annotations(object):
    """ represents a set of annotations that have been sorted by entity.
        We do this on disk to keep the memory footprint down. """
    def __init__(self, fname):
        print 'generating prediction cache for', fname.filename
        self.entity_files = {}

        annotation_file = csv.reader(fname, delimiter='\t')
        for line in annotation_file:
            if len(line) < 4: continue
            entity = line[3]
            if entity not in self.entity_files:
                 self.entity_files[entity] = tempfile.NamedTemporaryFile(prefix='kba', delete=True)
            self.entity_files[entity].write('\t'.join(line)+'\n')

        print 'indexed predictions for %d entities' % (len(self.entity_files))

    def get_predictions(self, entity):
        if entity not in self.entity_files:
            return []
        f = self.entity_files[entity]
        f.seek(0)
        a = np.genfromtxt(f, dtype=entry_dtype, usecols=[1,2,3,4])
        print 'read %d predictions' % len(a)
        times = [int(t) for t in np.core.defchararray.partition(a['docid'], '-')[:, 0]]
        return recfunctions.append_fields(a, 'time', times)

def read_judgments(fname):
    a = np.genfromtxt(fname, dtype=judg_dtype, usecols=[2, 3, 5])
    times = [int(t) for t in np.core.defchararray.partition(a['docid'], '-')[:, 0]]
    return recfunctions.append_fields(a, 'time', times)


def read_zipped_predictions(fname):
    with gzip.open(fname) as f:
        return Annotations(f)


def readPredictionsHeader(fname):
    f = gzip.open(fname, mode='r')
    firstline = f.readline()
    secondline = f.readline()
    team, runname, rest = string.split(secondline, maxsplit=2)
    f.close()
    return (team, runname)
<commit_msg>handle case of no predictions for one particular entity<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Measure performance for a run file, an intervalType (day,week, all), and a judgmentLevel (1 or 2).

Creates the output next to the input file, appending "-eval-week.tsv" (for weeks)

 example usage
 python measure-slice.py --intervalType week --judgmentLevel 1 -f ~/kba-evaluation/taia/data/umass-runs/UMass_CIIR-PC_RM20_1500.gz
"""

import os.path
import numpy as np
from numpy.lib import recfunctions
import gzip
from metrics import *
import string
from utils import *
from truthutil import *
from argparse import ArgumentParser
from targetentities import *
import csv
import tempfile


DEBUG = False

metrics = {
    'nDCG@R': ndcgREval, 'Prec@R': precREval, 'MAP': mapEval}

runFile = '~/kba-evaluation/taia/data/umass-runs/UMass_CIIR-PC_RM20_1500.gz'

parser = ArgumentParser()
parser.add_argument('--intervalType', help='slice segmentation: day, week, or all', default='all')
parser.add_argument('--judgmentLevel', type=int, help='Judgement level', default=1)
parser.add_argument('--trec_eval', action='store_true', help='Also create output for evaluation with trec_eval',
                    default=False)
parser.add_argument('-f', '--runFile', metavar='FILE', default=runFile)
args = parser.parse_args()

judgmentLevel = args.judgmentLevel
intervalType = args.intervalType
runFile = args.runFile
DUMP_TREC_EVAL = args.trec_eval  # also create output for trec_eval.

entry_dtype = np.dtype([('runname', '50a'), ('docid', '50a'), ('query', '50a'),
                        ('confidence', 'd4')])
judg_dtype = np.dtype([('docid', '50a'), ('query', '50a'),
                       ('label', 'd4')])
eval_dtype = np.dtype(
    [('team', '50a'), ('runname', '50a'), ('query', '50a'), ('intervalLow', 'd4'), ('intervalUp', 'd4'),
     ('unjudged', '50a'), ('judgmentLevel', 'd4'), ('metric', '50a'), ('value', 'f4')])


class Annotations(object):
    """ represents a set of annotations that have been sorted by entity.
        We do this on disk to keep the memory footprint down. """
    def __init__(self, fname):
        print 'generating prediction cache for', fname.filename
        self.entity_files = {}

        annotation_file = csv.reader(fname, delimiter='\t')
        for line in annotation_file:
            if len(line) < 4: continue
            entity = line[3]
            if entity not in self.entity_files:
                 self.entity_files[entity] = tempfile.NamedTemporaryFile(prefix='kba', delete=True)
            self.entity_files[entity].write('\t'.join(line)+'\n')

        print 'indexed predictions for %d entities' % (len(self.entity_files))

    def get_predictions(self, entity):
        if entity not in self.entity_files:
            return np.array([], dtype=entry_dtype)
        f = self.entity_files[entity]
        f.seek(0)
        a = np.genfromtxt(f, dtype=entry_dtype, usecols=[1,2,3,4])
        print 'read %d predictions' % len(a)
        times = [int(t) for t in np.core.defchararray.partition(a['docid'], '-')[:, 0]]
        return recfunctions.append_fields(a, 'time', times)

def read_judgments(fname):
    a = np.genfromtxt(fname, dtype=judg_dtype, usecols=[2, 3, 5])
    times = [int(t) for t in np.core.defchararray.partition(a['docid'], '-')[:, 0]]
    return recfunctions.append_fields(a, 'time', times)


def read_zipped_predictions(fname):
    with gzip.open(fname) as f:
        return Annotations(f)


def readPredictionsHeader(fname):
    f = gzip.open(fname, mode='r')
    firstline = f.readline()
    secondline = f.readline()
    team, runname, rest = string.split(secondline, maxsplit=2)
    f.close()
    return (team, runname)

{'fraction_non_alphanumeric': 0.1497755407427561, 'fraction_numerical': 0.01523602230988981, 'mean_word_length': 3.5978736710444026, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2318796', 'n_tokens_mistral': 2491, 'n_tokens_neox': 2401, 'n_words': 624}