<commit_before>#!/usr/bin/env python2

# Take 2 arguments
#
# 1. the instance to type filename
# 2. sumo filename
#
# and export the file to atomese scheme

import sys
import kifparser
from collections import defaultdict
from opencog.atomspace import AtomSpace, TruthValue, types, get_type

DEFAULT_NODE_TV = TruthValue(0.01, 1)
DEFAULT_LINK_TV = TruthValue(1, 1)
DEFAULT_PREDICATE_TV = TruthValue(0.1, 1)

global atomspace
atomspace=None

def load_instance2type(filename):
    """
    Load a filename generated by sumo-instance-types.py that
    associates each SUMO instance and its atom type and construct a
    dictionary that maps each instance to its atom type.
    """
    i2t = defaultdict(lambda: types.ConceptNode)
    with open(filename, 'rt') as i2tfile:
        for line in i2tfile:
            tokens = line.split()
            instance = tokens[0]
            atom_type = get_type(tokens[1])
            i2t[instance] = atom_type
    return i2t

def convert_multiple_expressions(i2t, expressions):
    for expression in expressions:
        convert_expression(i2t, expression)

def convert_expression(i2t, expression, link_tv=DEFAULT_LINK_TV):
    if isinstance(expression, str):
        return convert_token(i2t, expression)
    else:
        return convert_list(i2t, expression, link_tv)

def convert_token(i2t, token):
    if token.startswith('?') or token.startswith('@'):
        return atomspace.add_node(types.VariableNode, token)
    if token.startswith('"'):
        token = token[1:-2]
    atom_type = i2t[token]
    return atomspace.add_node(atom_type, token, tv=DEFAULT_NODE_TV)

def convert_variables(i2t, variables):
    var_atoms = [convert_token(i2t, var) for var in variables]
    return atomspace.add_link(types.VariableList, var_atoms)

def convert_list(i2t, expression, link_tv):
    oper = expression[0]
    args = expression[1:]

    # If it is a quantifier then the first argument is a variable list
    if oper in {"forall", "exists"}:
        args_atoms = [convert_variables(i2t, args[0])]
        args_atoms += [convert_expression(i2t, expr, link_tv=None)
                       for expr in args[1:]]
    else:
        args_atoms = [convert_expression(i2t, expr, link_tv=None) for expr in args]

    return link(i2t, oper, args_atoms, link_tv)

def link(i2t, oper, args_atoms, link_tv):
    # # If starts with a variable it is a variable list
    # if oper.startswith("?") or oper.startswith("@"):
    #     args_atoms = [convert_token(i2t, oper)] + args_atoms
    #     return atomspace.add_link(types.VariableList, args_atoms, tv=link_tv)

    # Map special operator to Atomese link
    link_type = special_link_type(oper)

    if link_type:
        return atomspace.add_link(link_type, args_atoms, tv=link_tv)

    # Otherwise assume that it is either a schema execution or
    # predicate evaluation
    link_type = i2t[oper]
    node_type = None
    if link_type == types.SchemaNode:
        link_type = types.ExecutionOutputLink
        node_type = types.SchemaNode
    elif link_type == types.PredicateNode:
        link_type = types.EvaluationLink
        node_type = types.PredicateNode
    elif oper.endswith('Fn'):   # This is wrong but may happen if the
                                # SUMO term has not been properly
                                # defined in SUMO. This is here till
                                # it gets fixed on the SUMO side
        link_type = types.ExecutionOutputLink
        node_type = types.SchemaNode
    elif oper.startswith('?') or oper.startswith('?'): # It's a
                                                       # variable,
                                                       # we're gonna
                                                       # assume it is a
                                                       # relation,
                                                       # which seems to
                                                       # be always
                                                       # treated as a
                                                       # predicate
        link_type = types.EvaluationLink
        node_type = types.VariableNode
    else:                       # By default we assume it is a
                                # predicate, which is surely wrong in
                                # practice but seems to work in
                                # practice
        link_type = types.EvaluationLink
        node_type = types.PredicateNode

    node = atomspace.add_node(node_type, oper, tv=DEFAULT_PREDICATE_TV)

    # Wrap in a ListLink if there is more than one argument
    if len(args_atoms) == 1:
        args_atoms = args_atoms[0]
    else:
        args_atoms = atomspace.add_link(types.ListLink, args_atoms)

    return atomspace.add_link(link_type, [node, args_atoms], tv=link_tv)

def special_link_type(oper):
    mapping = {
        '=>':types.ImplicationScopeLink,
        '<=>':types.EquivalenceScopeLink,
        'and':types.AndLink,
        'or':types.OrLink,
        'not':types.NotLink,
        'instance':types.MemberLink,
        # This might break some of the formal precision of SUMO, but who cares
        'attribute':types.MemberLink,
        'member':types.MemberLink,
        'subclass':types.InheritanceLink,
        'exists':types.ExistsLink,
        'forall':types.ForAllLink,
        'causes':types.ImplicationLink # Should probably be a
                                       # PredictiveImplicationLink but
                                       # that one is not known by the
                                       # python atomspace API
        }
<commit_msg>Variable lists are now typed<commit_after>#!/usr/bin/env python2

# Take 2 arguments
#
# 1. the instance to type filename
# 2. sumo filename
#
# and export the file to atomese scheme

import sys
import kifparser
from collections import defaultdict
from opencog.atomspace import AtomSpace, TruthValue, types, get_type

DEFAULT_NODE_TV = TruthValue(0.01, 1)
DEFAULT_LINK_TV = TruthValue(1, 1)
DEFAULT_PREDICATE_TV = TruthValue(0.1, 1)

global atomspace
atomspace=None

def load_instance2type(filename):
    """
    Load a filename generated by sumo-instance-types.py that
    associates each SUMO instance and its atom type and construct a
    dictionary that maps each instance to its atom type.
    """
    i2t = defaultdict(lambda: types.ConceptNode)
    with open(filename, 'rt') as i2tfile:
        for line in i2tfile:
            tokens = line.split()
            instance = tokens[0]
            atom_type = get_type(tokens[1])
            i2t[instance] = atom_type
    return i2t

def convert_multiple_expressions(i2t, expressions):
    for expression in expressions:
        convert_expression(i2t, expression)

def convert_expression(i2t, expression, link_tv=DEFAULT_LINK_TV):
    if isinstance(expression, str):
        return convert_token(i2t, expression)
    else:
        return convert_list(i2t, expression, link_tv)

def convert_token(i2t, token):
    if token.startswith('?') or token.startswith('@'):
        return atomspace.add_node(types.VariableNode, token)
    if token.startswith('"'):
        token = token[1:-2]
    atom_type = i2t[token]
    return atomspace.add_node(atom_type, token, tv=DEFAULT_NODE_TV)

def convert_variable(i2t, variable):
    # This type is gonna work for any translated atom
    predicate_type = atomspace.add_node(types.TypeNode, "PredicateNode")
    schema_type = atomspace.add_node(types.TypeNode, "SchemaNode")
    concept_type = atomspace.add_node(types.TypeNode, "ConceptNode")
    atom_type = atomspace.add_link(types.TypeChoice,
                                   [predicate_type, schema_type, concept_type])
    return atomspace.add_link(types.TypedVariableLink,
                              [convert_token(i2t, variable), atom_type])

def convert_variables(i2t, variables):
    if len(variables) == 1:
        return convert_variable(i2t, variables[0])
    var_atoms = [convert_variable(i2t, variable) for variable in variables]
    return atomspace.add_link(types.VariableList, var_atoms)

def convert_quantifier(i2t, expression, link_tv):
    oper = expression[0]
    args = expression[1:]

    if oper == "forall":
        var_atoms = convert_variables(i2t, args[0])

        # (forall (=> is translated into ImplicationScopeLink
        if args[1][0] == "=>":
            args_atoms = [var_atoms] + \
                         [convert_expression(i2t, expr, link_tv=None)
                          for expr in args[1][1:]]
            return atomspace.add_link(types.ImplicationScopeLink, args_atoms, link_tv)

        # (forall (<=> is translated into EquivalenceScopeLink
        if args[1][0] == "<=>":
            args_atoms = [var_atoms] + \
                         [convert_expression(i2t, expr, link_tv=None)
                          for expr in args[1][1:]]
            return atomspace.add_link(types.EquivalenceScopeLink, args_atoms, link_tv)

        # Otherwise (forall ... is translated into ForAllLink
        args_atoms = [var_atoms] + \
                     [convert_expression(i2t, expr, link_tv=None)
                      for expr in args[1:]]
        return atomspace.add_link(types.ForAllLink, args_atoms, link_tv)

    elif oper == "exists":
        var_atoms = convert_variables(i2t, args[0])

        # (exists ... is translated into ExistsLink
        args_atoms = [var_atoms] + \
                     [convert_expression(i2t, expr, link_tv=None)
                      for expr in args[1:]]
        return atomspace.add_link(types.ExistsLink, args_atoms, link_tv)
    else:
        return None

def convert_list(i2t, expression, link_tv):
    # First attempt to convert as a quantifier
    quantifier_atom = convert_quantifier(i2t, expression, link_tv)
    if quantifier_atom:
        return quantifier_atom

    # If failed then call default link converter
    oper = expression[0]
    args = expression[1:]
    args_atoms = [convert_expression(i2t, expr, link_tv=None) for expr in args]
    return link(i2t, oper, args_atoms, link_tv)

def link(i2t, oper, args_atoms, link_tv):
    # # If starts with a variable it is a variable list
    # if oper.startswith("?") or oper.startswith("@"):
    #     args_atoms = [convert_token(i2t, oper)] + args_atoms
    #     return atomspace.add_link(types.VariableList, args_atoms, tv=link_tv)

    # Map special operator to Atomese link
    link_type = special_link_type(oper)

    if link_type:
        return atomspace.add_link(link_type, args_atoms, tv=link_tv)

    # Otherwise assume that it is either a schema execution or
    # predicate evaluation
    link_type = i2t[oper]
    node_type = None
    if link_type == types.SchemaNode:
        link_type = types.ExecutionOutputLink
        node_type = types.SchemaNode
    elif link_type == types.PredicateNode:
        link_type = types.EvaluationLink
        node_type = types.PredicateNode
    elif oper.endswith('Fn'):   # This is wrong but may happen if the
                                # SUMO term has not been properly
                                # defined in SUMO. This is here till
                                # it gets fixed on the SUMO side
        link_type = types.ExecutionOutputLink
        node_type = types.SchemaNode
    elif oper.startswith('?') or oper.startswith('?'): # It's a
                                                       # variable,
                                                       # we're gonna
                                                       # assume it is a
                                                       # relation,
                                                       # which seems to
                                                       # be always
                                                       # treated as a
                                                       # predicate
        link_type = types.EvaluationLink
        node_type = types.VariableNode
    else:                       # By default we assume it is a
                                # predicate, which is surely wrong in
                                # practice but seems to work in
                                # practice
        link_type = types.EvaluationLink
        node_type = types.PredicateNode

    node = atomspace.add_node(node_type, oper, tv=DEFAULT_PREDICATE_TV)

    # Wrap in a ListLink if there is more than one argument
    if len(args_atoms) == 1:
        args_atoms = args_atoms[0]
    else:
        args_atoms = atomspace.add_link(types.ListLink, args_atoms)

    return atomspace.add_link(link_type, [node, args_atoms], tv=link_tv)

def special_link_type(oper):
    mapping = {
        '=>':types.ImplicationLink,
        '<=>':types.EquivalenceLink,
        'and':types.AndLink,
        'or':types.OrLink,
        'not':types.NotLink,
        'instance':types.MemberLink,
        # This might break some of the formal precision of SUMO, but who cares
        'attribute':types.MemberLink,
        'member':types.MemberLink,
        'subclass':types.InheritanceLink,
        'exists':types.ExistsLink,
        'forall':types.ForAllLink,
        'causes':types.ImplicationLink # Should probably be a
                                       # PredictiveImplicationLink but
                                       # that one is not known by the
                                       # python atomspace API
        }

{'fraction_non_alphanumeric': 0.08193793257423532, 'fraction_numerical': 0.00855845798913448, 'mean_word_length': 2.0069366748713358, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 61, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6061685', 'n_tokens_mistral': 3814, 'n_tokens_neox': 3714, 'n_words': 1010}