<commit_before>from collections import defaultdict, namedtuple, OrderedDict

from colorama import Fore, Style

from iepy import db
from iepy.human_validation import Answers
from iepy.pycompatibility import py_compatible_csv


def certainty(p):
    return 0.5 + abs(p - 0.5) if p is not None else 0.5


class Knowledge(OrderedDict):
    """Maps evidence to a score in [0...1]

    None is also a valid score for cases when no score information is available
    """
    CSV_COLUMNS = [
        u'entity a kind', u'entity a key', u'entity b kind', u'entity b key',
        u'relation name', u'document name', u'segment offset',
        u'entity a tokens offset', u'entity b tokens offset', u'label']
    __slots__ = ()

    def __eq__(self, other):
        """Only comparing the mapping evidence -> score
        and just that, not the ordering, so...
        """
        return dict(self) == dict(other)

    def by_certainty(self):
        """
        Returns an iterable over the evidence, with the most certain evidence
        at the front and the least certain evidence at the back. "Certain"
        means a score close to 0 or 1, and "uncertain" a score closer to 0.5.
        Note that a score of 'None' is considered as 0.5 here
        """
        def key_funct(e_s):
            e = e_s[0]
            return (certainty(self[e]) if self[e] is not None else 0, e)
        return sorted(self.items(), key=key_funct, reverse=True)

    def by_score(self, reverse=False):
        """
        Returns an iterable over the evidence sorted by score, in increasing
        order if reverse=False (default), in decreasing order if reverse=True.
        """
        return sorted(self.items(), key=lambda x: x[1], reverse=reverse)

    def per_relation(self):
        """
        Returns a dictionary: relation -> Knowledge, where each value is only
        the knowledge for that specific relation
        """
        result = defaultdict(Knowledge)
        for e, s in self.items():
            result[e.fact.relation][e] = s
        return result

    def save_to_csv(self, filepath):
        """Writes labeled evidence to a CSV file encoded in UTF-8.

        The output CSV format can be seen on CSV_COLUMNS.
        """
        with py_compatible_csv.writer(filepath) as evidence_writer:
            for (evidence, label) in sorted(self.items(),
                                            key=lambda x: (x[0].segment.document,
                                                           x[0].segment,
                                                           x[0].fact)):
                fact = evidence.fact
                segm = evidence.segment
                entity_a = fact.e1
                entity_b = fact.e2
                row = [entity_a.kind, entity_a.key,
                       entity_b.kind, entity_b.key,
                       fact.relation]
                if segm:
                    row.extend(
                        [segm.document.human_identifier,
                         segm.offset,
                         segm.entities[evidence.o1].offset,
                         segm.entities[evidence.o2].offset,
                         label
                         ]<commit_msg>changed sort-evidence-function so it doent brake when dealing with facts unevidenced (seed facts in the iepy bootstrap)<commit_after>from collections import defaultdict, namedtuple, OrderedDict

from colorama import Fore, Style

from iepy import db
from iepy.human_validation import Answers
from iepy.pycompatibility import py_compatible_csv


def certainty(p):
    return 0.5 + abs(p - 0.5) if p is not None else 0.5


class Knowledge(OrderedDict):
    """Maps evidence to a score in [0...1]

    None is also a valid score for cases when no score information is available
    """
    CSV_COLUMNS = [
        u'entity a kind', u'entity a key', u'entity b kind', u'entity b key',
        u'relation name', u'document name', u'segment offset',
        u'entity a tokens offset', u'entity b tokens offset', u'label']
    __slots__ = ()

    def __eq__(self, other):
        """Only comparing the mapping evidence -> score
        and just that, not the ordering, so...
        """
        return dict(self) == dict(other)

    def by_certainty(self):
        """
        Returns an iterable over the evidence, with the most certain evidence
        at the front and the least certain evidence at the back. "Certain"
        means a score close to 0 or 1, and "uncertain" a score closer to 0.5.
        Note that a score of 'None' is considered as 0.5 here
        """
        def key_funct(e_s):
            e = e_s[0]
            return (certainty(self[e]) if self[e] is not None else 0, e)
        return sorted(self.items(), key=key_funct, reverse=True)

    def by_score(self, reverse=False):
        """
        Returns an iterable over the evidence sorted by score, in increasing
        order if reverse=False (default), in decreasing order if reverse=True.
        """
        return sorted(self.items(), key=lambda x: x[1], reverse=reverse)

    def per_relation(self):
        """
        Returns a dictionary: relation -> Knowledge, where each value is only
        the knowledge for that specific relation
        """
        result = defaultdict(Knowledge)
        for e, s in self.items():
            result[e.fact.relation][e] = s
        return result

    def save_to_csv(self, filepath):
        """Writes labeled evidence to a CSV file encoded in UTF-8.

        The output CSV format can be seen on CSV_COLUMNS.
        """
        def sort_funct(x):
            ev, score = x
            if not ev.segment:
                return (None, None, ev.fact)
            else:
                return (ev.segment.document, ev.segment, ev.fact)
        with py_compatible_csv.writer(filepath) as evidence_writer:
            for (evidence, label) in sorted(self.items(),
                                            key=sort_funct):
                fact = evidence.fact
                segm = evidence.segment
                entity_a = fact.e1
                entity_b = fact.e2
                row = [entity_a.kind, entity_a.key,
                       entity_b.kind, entity_b.key,
                       fact.relation]
                if segm:
                    row.extend(
                        [segm.document.human_identifier,
                         segm.offset,
                         segm.entities[evidence.o1].offset,
                         segm.entities[evidence.o2].offset,
                         label
                         ]
{'fraction_non_alphanumeric': 0.08616227736337342, 'fraction_numerical': 0.007154818084944436, 'mean_word_length': 1.7547169811320755, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7345233', 'n_tokens_mistral': 1804, 'n_tokens_neox': 1710, 'n_words': 613}