<commit_before>from opencog.atomspace import AtomSpace, Atom, get_type, types
from copy import copy, deepcopy
from functools import *
import sys
from itertools import permutations
from util import *

def coerce_tree(x):
    assert type(x) != type(None)
    if isinstance(x, tree):
        return x
    else:
        return tree(x)

class tree:
    def __init__(self, op, *args):
        # Transparently allow using passing a list or using it in the more streamlined way
        # (better for constructing trees by hand)
        if len(args) and isinstance(args[0], list):
            args = args[0]
        # Transparently record Links as strings rather than Handles
        assert type(op) != type(None)
        if len(args):
            if isinstance(op, Atom):
                assert not op.is_a(types.Link)
                self.op = op.type_name
            else:
                self.op = op
            self.args = [coerce_tree(x) for x in args]
        else:
            self.op = op
            self.args = args

    def __str__(self):
        if self.is_leaf():
            if isinstance(self.op, Atom):
                return self.op.name+':'+self.op.type_name
            else:
                return 'tree:'+str(self.op)
        else:
            return '(' + str(self.op) + ' '+ ' '.join(map(str, self.args)) + ')'

    # TODO add doctests
    def __repr__(self):
        return str(self)
        
        if self.is_variable():
            return "a.add(t.VariableNode, name='$%s')" % (str(self.op),)
        elif self.is_leaf() and isinstance(self.op, Atom):
            # a.add(t.ConceptNode, name='cat')
            return "a.add(t.%s, '%s')" % (self.op.type_name, self.op.name)
        else:
            # a.add(t.ListLink, out=[ a.add(t.ConceptNode, name='eat') ])
            return "a.add(t.%s, out=%s)" % (self.op, repr(self.args))

    def __hash__(self):
        return hash( self.to_tuple() )

    def is_variable(self):
        "A variable is an int starting from 0"
        return isinstance(self.op, int)
    
    def get_type(self):
        if isinstance(self.op, int):
            return types.Atom
        elif isinstance(self.op, Atom):
            return self.op.t
        else:
            return get_type(self.op)
    
    def is_leaf(self):
        return len(self.args) == 0
    
    def __cmp__(self, other):
        if not isinstance(other, tree):
            return cmp(tree, type(other))
        #print self.to_tuple(), other.to_tuple()
        return cmp(self.to_tuple(), other.to_tuple())
    
    def to_tuple(self):
        # Atom doesn't support comparing to different types in the Python-standard way.
        if isinstance(self.op, Atom):
            #assert type(self.op.h) != type(None)
            return self.op.h.value()
            #return self.op.type_name+':'+self.op.name # Easier to understand, though a bit less efficient
        else:
            return tuple([self.op]+[x.to_tuple() for x in self.args])

def tree_from_atom(atom):
    if atom.is_node():
        return tree(atom)
    else:
        args = [tree_from_atom(x) for x in atom.out]
        return tree(atom.type_name, args)

def atom_from_tree(tree, a):
    if tree.is_variable():
        return a.add(types.VariableNode, name='$'+str(tree.op))
    elif tree.is_leaf():
        # Node (simply the handle)
        if isinstance (tree.op, Atom):
            return tree.op
        # Empty Link
        else:
            return a.add(get_type(tree.op), out = [])
    else:
        out = [atom_from_tree(x, a) for x in tree.args]
        return a.add(get_type(tree.op), out=out)

def find(template, atoms):
    return [a for a in atoms if unify(tree_from_atom(a), template, {}) != None]

class Match(object):
    def __init__(self, subst = {}, atoms = [], conj = ()):<commit_msg>Simple efficiency change<commit_after>from opencog.atomspace import AtomSpace, Atom, get_type, types
from copy import copy, deepcopy
from functools import *
import sys
from itertools import permutations
from util import *

def coerce_tree(x):
    assert type(x) != type(None)
    if isinstance(x, tree):
        return x
    else:
        return tree(x)

class tree:
    def __init__(self, op, *args):
        # Transparently allow using passing a list or using it in the more streamlined way
        # (better for constructing trees by hand)
        if len(args) and isinstance(args[0], list):
            args = args[0]
        # Transparently record Links as strings rather than Handles
        assert type(op) != type(None)
        if len(args):
            if isinstance(op, Atom):
                assert not op.is_a(types.Link)
                self.op = op.type_name
            else:
                self.op = op
            self.args = [coerce_tree(x) for x in args]
        else:
            self.op = op
            self.args = args

    def __str__(self):
        if self.is_leaf():
            if isinstance(self.op, Atom):
                return self.op.name+':'+self.op.type_name
            else:
                return 'tree:'+str(self.op)
        else:
            return '(' + str(self.op) + ' '+ ' '.join(map(str, self.args)) + ')'

    # TODO add doctests
    def __repr__(self):
        return str(self)
        
        if self.is_variable():
            return "a.add(t.VariableNode, name='$%s')" % (str(self.op),)
        elif self.is_leaf() and isinstance(self.op, Atom):
            # a.add(t.ConceptNode, name='cat')
            return "a.add(t.%s, '%s')" % (self.op.type_name, self.op.name)
        else:
            # a.add(t.ListLink, out=[ a.add(t.ConceptNode, name='eat') ])
            return "a.add(t.%s, out=%s)" % (self.op, repr(self.args))

    def __hash__(self):
        return hash( self.to_tuple() )

    def is_variable(self):
        "A variable is an int starting from 0"
        return isinstance(self.op, int)
    
    def get_type(self):
        if isinstance(self.op, int):
            return types.Atom
        elif isinstance(self.op, Atom):
            return self.op.t
        else:
            return get_type(self.op)
    
    def is_leaf(self):
        return len(self.args) == 0
    
    def __cmp__(self, other):
        if not isinstance(other, tree):
            return cmp(tree, type(other))
        #print self.to_tuple(), other.to_tuple()
        return cmp(self.to_tuple(), other.to_tuple())
    
    def to_tuple(self):
        # Simply cache the tuple.
        # TODO: A more efficient alternative would be to adapt the hash function and compare function
        # to work on Trees directly.
        try:
            return self._tuple
        except:
            # Atom doesn't support comparing to different types in the Python-standard way.
            if isinstance(self.op, Atom):
                #assert type(self.op.h) != type(None)
                self._tuple = self.op.h.value()
                return self._tuple
                #return self.op.type_name+':'+self.op.name # Easier to understand, though a bit less efficient
            else:
                self._tuple = tuple([self.op]+[x.to_tuple() for x in self.args])
                return self._tuple

def tree_from_atom(atom):
    if atom.is_node():
        return tree(atom)
    else:
        args = [tree_from_atom(x) for x in atom.out]
        return tree(atom.type_name, args)

def atom_from_tree(tree, a):
    if tree.is_variable():
        return a.add(types.VariableNode, name='$'+str(tree.op))
    elif tree.is_leaf():
        # Node (simply the handle)
        if isinstance (tree.op, Atom):
            return tree.op
        # Empty Link
        else:
            return a.add(get_type(tree.op), out = [])
    else:
        out = [atom_from_tree(x, a) for x in tree.args]
        return a.add(get_type(tree.op), out=out)

def find(template, atoms):
    return [a for a in atoms if unify(tree_from_atom(a), template, {}) != None]

class Match(object):
    def __init__(self, subst = {}, atoms = [], conj = ()):
{'fraction_non_alphanumeric': 0.12264150943396226, 'fraction_numerical': 0.0010062893081761006, 'mean_word_length': 2.006049149338374, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7283000', 'n_tokens_mistral': 2522, 'n_tokens_neox': 2469, 'n_words': 678}