<commit_before>import random
import pattern.en as pen
import snode
from unidecode import unidecode
import simplejson
import syntax_rewriter
import sys
import argument_preprocess

def Reload():
    snode.Reload()
    syntax_rewriter.Reload()
    argument_preprocess.Reload()
    return reload(sys.modules[__name__])

P = snode.SNode

def PT(*nodes):
    if len(nodes) == 1:
        if isinstance(nodes[0],str):
            return P({"type":snode.SYNTAX}, nodes[0])
        else:
            return nodes[0]
    nodes = [PT(n) if isinstance(n,str) else n for n in nodes]
    return P({"type":snode.SYNTAX}, *nodes)

ONGOING = "ongoing"
RECURRING = "recurring"
EVENT = "event"

PAST = "past"
PRESENT = "present"
FUTURE = "future"

DID = "did"
BE = "be"

SIMPLE = "simple"
CONTINUOUS = "continuous"
PERFECT = "perfect"
PERFECT_CONTINUOUS = "perfect_continuous"

INDICATIVE = "indicative"
CONDITIONAL_COULD = "conditional_could"
CONDITIONAL_SHOULD = "conditional_should"
CONDITIONAL_WOULD = "conditional_would"
SUBJUNCTIVE = "subjunctive"

WHY = "why"
HOW = "how"
EVIDENCE = "evidence"
SIMILAR = "similar"
IFTHEN = "ifthen"

MALE = "male"
FEMALE = "female"

THING = "thing"
PERSON = "person"
GROUP = "group"
UNCOUNTABLE_THING = "uncountable_thing"

POSSESSIVE = "possessive"
SET = "set"

class MtrException(Exception):
    def __init__(self, msg):
        self.msg = msg

    def __str__(self):
        return "MtrException(%s)" % self.msg

def OpenContext(fname):
    ctx = Context()
    json = simplejson.load(open(fname,"r"))
    json = argument_preprocess.Preprocess(json)
    ctx.FromJSON(json)
    return ctx

class Context:
    def __init__(self, debug=False):
        self.entities  = {}
        self.facts     = {}
        self.relations = {}
        self.reverse_relations = {}
        self.justifies = {}
        self.debug = debug

    def AddEntity(self, json):
        self.entities[json["name"]] = Entity(json)

    def GetEntity(self, name):
        if name in self.entities:
            return self.entities[name]
        return Entity({"name":name, "type":THING})

    def AddFact(self, json):
        tid = json["id"]
        assert tid not in self.facts, (tid, self.facts)
        self.facts[tid] = Fact(json)

    def GetFact(self, tid):
        return self.facts[tid]

    def GetFactsByOriginalId(self, tid):
        return [self.GetFact(f) for f in self.facts if self.GetFact(f).OriginalId() == tid]
    
    def AddRelation(self, json):
        if json["gov"] not in self.relations:
            self.relations[json["gov"]] = []
        self.relations[json["gov"]].append(Relation(json))
        if json["dep"] not in self.reverse_relations:
            self.reverse_relations[json["dep"]] = []
        self.reverse_relations[json["dep"]].append(Relation(json))
        if "justifies" in json:
            for j in json["justifies"]:
                if j not in self.justifies:
                    self.justifies[j] = []
                self.justifies[j].append(Relation(json))
        
    def FromJSON(self, json):
        for k in ["entities","facts","relations"]:
            if k not in json:
                json[k] = []
        for e in json["entities"]:
            self.AddEntity(e)
        for f in json["facts"]:
            self.AddFact(f)
        for r in json["relations"]:
            self.AddRelation(r)

    def SubjectPronoun(self, name):
        return self.GetEntity(name).SubjectPronoun(self)

    def ObjectPronoun(self, name):
        return self.GetEntity(name).ObjectPronoun(self)

    def SelfPronoun(self, name):
        return self.GetEntity(name).SelfPronoun(self)
    
    def PrintAllFacts(self, toText=True):
        for i, f in self.facts.iteritems():
            print "FACT", i
            f.PrintAllSimpleSentences(self, toText)

    def PrintAllRelations(self, toText=True, rewrite=True, involving=None, times=1):
        for k,v in self.relations.iteritems():
            for vi in v:
                if involving is None or involving in [vi.json["gov"], vi.json["dep"], vi.Gov(self).OriginalId(), vi.Dep(self).OriginalId()]:
                    for _ in xrange(times):
                        snode = vi.Mtr(self)
                        if rewrite:
                            snode = syntax_rewriter.RewriteSyntax(self, snode)
                        if toText:
                            print snode.ToText()
                        else:
                            print snode
                    if times > 1:
                        print ""

    def Justifications(self, factId):
        result = []
        if factId in self.relations:
            result.extend(self.relations[factId])
        if factId in self.reverse_relations:
            result.extend([r for r in self.reverse_relations[factId] if r.Type() in [SIMILAR, IFTHEN]])
        if factId in self.justifies:
            result.extend(self.justifies[factId])
        return result

    def RandomRelation(self):
        return random.choice([r for _, rs in self.relations.iteritems() for r in rs])

    def RandomFact(self, entityName=None):
        facts = []
        if entityName is not None:
            facts = [f for f in self.facts if f.ReferencesEntity(entityName)]<commit_msg>Fix Justifications<commit_after>import random
import pattern.en as pen
import snode
from unidecode import unidecode
import simplejson
import syntax_rewriter
import sys
import argument_preprocess

def Reload():
    snode.Reload()
    syntax_rewriter.Reload()
    argument_preprocess.Reload()
    return reload(sys.modules[__name__])

P = snode.SNode

def PT(*nodes):
    if len(nodes) == 1:
        if isinstance(nodes[0],str):
            return P({"type":snode.SYNTAX}, nodes[0])
        else:
            return nodes[0]
    nodes = [PT(n) if isinstance(n,str) else n for n in nodes]
    return P({"type":snode.SYNTAX}, *nodes)

ONGOING = "ongoing"
RECURRING = "recurring"
EVENT = "event"

PAST = "past"
PRESENT = "present"
FUTURE = "future"

DID = "did"
BE = "be"

SIMPLE = "simple"
CONTINUOUS = "continuous"
PERFECT = "perfect"
PERFECT_CONTINUOUS = "perfect_continuous"

INDICATIVE = "indicative"
CONDITIONAL_COULD = "conditional_could"
CONDITIONAL_SHOULD = "conditional_should"
CONDITIONAL_WOULD = "conditional_would"
SUBJUNCTIVE = "subjunctive"

WHY = "why"
HOW = "how"
EVIDENCE = "evidence"
SIMILAR = "similar"
IFTHEN = "ifthen"

MALE = "male"
FEMALE = "female"

THING = "thing"
PERSON = "person"
GROUP = "group"
UNCOUNTABLE_THING = "uncountable_thing"

POSSESSIVE = "possessive"
SET = "set"

class MtrException(Exception):
    def __init__(self, msg):
        self.msg = msg

    def __str__(self):
        return "MtrException(%s)" % self.msg

def OpenContext(fname):
    ctx = Context()
    json = simplejson.load(open(fname,"r"))
    json = argument_preprocess.Preprocess(json)
    ctx.FromJSON(json)
    return ctx

class Context:
    def __init__(self, debug=False):
        self.entities  = {}
        self.facts     = {}
        self.originals = {}
        self.relations = {}
        self.reverse_relations = {}
        self.justifies = {}
        self.debug = debug

    def AddEntity(self, json):
        self.entities[json["name"]] = Entity(json)

    def GetEntity(self, name):
        if name in self.entities:
            return self.entities[name]
        return Entity({"name":name, "type":THING})

    def AddFact(self, json):
        tid = json["id"]
        assert tid not in self.facts, (tid, self.facts)
        self.facts[tid] = Fact(json)
        oid = self.facts[tid].OriginalId()
        if oid not in self.originals:
            self.originals[oid] = []
        self.originals[oid].append(tid)

    def GetFact(self, tid):
        return self.facts[tid]

    def GetFactsByOriginalId(self, tid):
        return [self.GetFact(f) for f in self.facts if self.GetFact(f).OriginalId() == tid]
    
    def AddRelation(self, json):
        if json["gov"] not in self.relations:
            self.relations[json["gov"]] = []
        self.relations[json["gov"]].append(Relation(json))
        if json["dep"] not in self.reverse_relations:
            self.reverse_relations[json["dep"]] = []
        self.reverse_relations[json["dep"]].append(Relation(json))
        if "justifies" in json:
            for j in json["justifies"]:
                if j not in self.justifies:
                    self.justifies[j] = []
                self.justifies[j].append(Relation(json))
        
    def FromJSON(self, json):
        for k in ["entities","facts","relations"]:
            if k not in json:
                json[k] = []
        for e in json["entities"]:
            self.AddEntity(e)
        for f in json["facts"]:
            self.AddFact(f)
        for r in json["relations"]:
            self.AddRelation(r)

    def SubjectPronoun(self, name):
        return self.GetEntity(name).SubjectPronoun(self)

    def ObjectPronoun(self, name):
        return self.GetEntity(name).ObjectPronoun(self)

    def SelfPronoun(self, name):
        return self.GetEntity(name).SelfPronoun(self)
    
    def PrintAllFacts(self, toText=True):
        for i, f in self.facts.iteritems():
            print "FACT", i
            f.PrintAllSimpleSentences(self, toText)

    def PrintAllRelations(self, toText=True, rewrite=True, involving=None, times=1):
        for k,v in self.relations.iteritems():
            for vi in v:
                if involving is None or involving in [vi.json["gov"], vi.json["dep"], vi.Gov(self).OriginalId(), vi.Dep(self).OriginalId()]:
                    for _ in xrange(times):
                        snode = vi.Mtr(self)
                        if rewrite:
                            snode = syntax_rewriter.RewriteSyntax(self, snode)
                        if toText:
                            print snode.ToText()
                        else:
                            print snode
                    if times > 1:
                        print ""

    def Justifications(self, originalFactId):
        result = []
        for factId in self.originals[originalFactId]:
            if factId in self.relations:
                result.extend(self.relations[factId])
            if factId in self.reverse_relations:
                result.extend([r for r in self.reverse_relations[factId] if r.Type() in [SIMILAR, IFTHEN]])
            if factId in self.justifies:
                result.extend(self.justifies[factId])
        return result

    def RandomRelation(self):
        return random.choice([r for _, rs in self.relations.iteritems() for r in rs])

    def RandomFact(self, entityName=None):
        facts = []
        if entityName is not None:
            facts = [f for f in self.facts if f.ReferencesEntity(entityName)]
{'fraction_non_alphanumeric': 0.11887197684190867, 'fraction_numerical': 0.0011205528060509852, 'mean_word_length': 2.308619091751622, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '771248', 'n_tokens_mistral': 3465, 'n_tokens_neox': 3299, 'n_words': 796}