<commit_before>from sqlalchemy.mapper import *
from sqlalchemy.schema import *
from sqlalchemy.sql import *
import sqlalchemy.util as util
import tables
import string, sys

"""a more advanced example of basic_tree.py.  illustrates MapperExtension objects which
add application-specific functionality to a Mapper object."""

class NodeList(util.OrderedDict):
    """extends an Ordered Dictionary, which is just a dictionary that iterates its keys and values
    in the order they were inserted.  Adds an "append" method, which appends a node to the 
    dictionary as though it were a list, and also within append automatically associates 
    the parent of a TreeNode with itself."""
    def __init__(self, parent):
        util.OrderedDict.__init__(self)
        self.parent = parent
    def append(self, node):
        node.parent = self.parent
        node._set_root(self.parent.root)
        self[node.name] = node
    def __iter__(self):
        return iter(self.values())

class TreeNode(object):
    """a hierarchical Tree class, which adds the concept of a "root node".  The root is 
    the topmost node in a tree, or in other words a node whose parent ID is NULL.  
    All child nodes that are decendents of a particular root, as well as a root node itself, 
    reference this root node.  
    this is useful as a way to identify all nodes in a tree as belonging to a single
    identifiable root.  Any node can return its root node and therefore the "tree" that it 
    belongs to, and entire trees can be selected from the database in one query, by 
    identifying their common root ID."""
    
    def __init__(self, name):
        """for data integrity, a TreeNode requires its name to be passed as a parameter
        to its constructor, so there is no chance of a TreeNode that doesnt have a name."""
        self.name = name
        self.children = NodeList(self)
        self.root = self
        self.parent = None
        self.id = None
        self.parent_id = None
        self.root_id=None
    def _set_root(self, root):
        self.root = root
        for c in self.children:
            c._set_root(root)
    def append(self, node):
        if isinstance(node, str):
            self.children.append(TreeNode(node))
        else:
            self.children.append(node)
    def __str__(self):
        return self._getstring(0, False)
    def _getstring(self, level, expand = False):
        s = ('  ' * level) + "%s (%s,%s,%s, %d)" % (self.name, self.id,self.parent_id,self.root_id, id(self)) + '\n'
        if expand:
            s += string.join([n._getstring(level+1, True) for n in self.children.values()], '')
        return s
    def print_nodes(self):
        return self._getstring(0, True)
        
class TreeLoader(MapperExtension):
    """an extension that will plug-in additional functionality to the Mapper."""
    def create_instance(self, mapper, row, imap, class_):
        """creates an instance of a TreeNode.  since the TreeNode constructor requires
        the 'name' argument, this method pulls the data from the database row directly."""
        return TreeNode(row[mapper.c.name.label], _mapper_nohistory=True)
    def after_insert(self, mapper, instance):
        """runs after the insert of a new TreeNode row.  The primary key of the row is not determined
        until the insert is complete, since most DB's use autoincrementing columns.  If this node is
        the root node, we will take the new primary key and update it as the value of the node's 
        "root ID" as well, since its root node is itself."""
        if instance.root is instance:
            mapper.primarytable.update(TreeNode.c.id==instance.id, values=dict(root_node_id=instance.id)).execute()
            instance.root_id = instance.id
    def append_result(self, mapper, row, imap, result, instance, populate_existing=False):
        """runs as results from a SELECT statement are processed, and newly created or already-existing
        instances that correspond to each row are appended to result lists.  This method will only
        append root nodes to the result list, and will attach child nodes to their appropriate parent
        node as they arrive from the select results.  This allows a SELECT statement which returns
        both root and child nodes in one query to return a list of "roots"."""
        if instance.parent_id is None:
            result.append(instance)
        else:
            parentnode = imap[mapper.identity_key(instance.parent_id)]
            parentnode.children.append(instance, _mapper_nohistory=True)
        return False
            
# define the mapper.  we will make "convenient" property
# names vs. the more verbose names in the table definition

TreeNode.mapper=assignmapper(tables.trees, properties=dict(
    id=tables.trees.c.node_id,
    name=tables.trees.c.node_name,
    parent_id=tables.trees.c.parent_node_id,
    root_id=tables.trees.c.root_node_id,
    root=relation(TreeNode, primaryjoin=tables.trees.c.root_node_id==tables.trees.c.node_id, thiscol=tables.trees.c.root_node_id, lazy=None, uselist=False),
    children=relation(TreeNode, primaryjoin=tables.trees.c.parent_node_id==tables.trees.c.node_id, thiscol=tables.trees.c.node_id, lazy=None, uselist=True, private=True),
), extension = TreeLoader())
TreeNode.mapper


node2 = TreeNode('node2')
node2.append('subnode1')
node = TreeNode('rootnode')
node.append('node1')
node.append(node2)
node.append('node3')
node.children['node2'].append('subnode2')

print "\n\n\n----------------------------"
print "Created new tree structure:"
print "----------------------------"

print node.print_nodes()

print "\n\n\n----------------------------"
print "Committing:"
print "----------------------------"

objectstore.commit()

print "\n\n\n----------------------------"
print "Tree After Save:"
print "----------------------------"

print node.print_nodes()

node.append('node4')
node.children['node4'].append('subnode3')
node.children['node4'].append('subnode4')
node.children['node4'].children['subnode3'].append('subsubnode1')
del node.children['node1']

print "\n\n\n----------------------------"
print "Modified the tree"
print "(added node4, node4/subnode3, node4/subnode4,"
print "node4/subnode3/subsubnode1, deleted node1):"
print "----------------------------"

print node.print_nodes()

print "\n\n\n----------------------------"
print "Committing:"
print "----------------------------"
objectstore.commit()
<commit_msg>w00p<commit_after>from sqlalchemy.mapper import *
from sqlalchemy.schema import *
from sqlalchemy.sql import *
import sqlalchemy.util as util
import tables
import string, sys

"""a more advanced example of basic_tree.py.  illustrates MapperExtension objects which
add application-specific functionality to a Mapper object."""

class NodeList(util.OrderedDict):
    """extends an Ordered Dictionary, which is just a dictionary that iterates its keys and values
    in the order they were inserted.  Adds an "append" method, which appends a node to the 
    dictionary as though it were a list, and also within append automatically associates 
    the parent of a TreeNode with itself."""
    def __init__(self, parent):
        util.OrderedDict.__init__(self)
        self.parent = parent
    def append(self, node):
        node.parent = self.parent
        node._set_root(self.parent.root)
        self[node.name] = node
    def __iter__(self):
        return iter(self.values())

class TreeNode(object):
    """a hierarchical Tree class, which adds the concept of a "root node".  The root is 
    the topmost node in a tree, or in other words a node whose parent ID is NULL.  
    All child nodes that are decendents of a particular root, as well as a root node itself, 
    reference this root node.  
    this is useful as a way to identify all nodes in a tree as belonging to a single
    identifiable root.  Any node can return its root node and therefore the "tree" that it 
    belongs to, and entire trees can be selected from the database in one query, by 
    identifying their common root ID."""
    
    def __init__(self, name):
        """for data integrity, a TreeNode requires its name to be passed as a parameter
        to its constructor, so there is no chance of a TreeNode that doesnt have a name."""
        self.name = name
        self.children = NodeList(self)
        self.root = self
        self.parent = None
        self.id = None
        self.parent_id = None
        self.root_id=None
    def _set_root(self, root):
        self.root = root
        for c in self.children:
            c._set_root(root)
    def append(self, node):
        if isinstance(node, str):
            self.children.append(TreeNode(node))
        else:
            self.children.append(node)
    def __str__(self):
        return self._getstring(0, False)
    def _getstring(self, level, expand = False):
        s = ('  ' * level) + "%s (%s,%s,%s, %d)" % (self.name, self.id,self.parent_id,self.root_id, id(self)) + '\n'
        if expand:
            s += string.join([n._getstring(level+1, True) for n in self.children.values()], '')
        return s
    def print_nodes(self):
        return self._getstring(0, True)
        
class TreeLoader(MapperExtension):
    """an extension that will plug-in additional functionality to the Mapper."""
    def create_instance(self, mapper, row, imap, class_):
        """creates an instance of a TreeNode.  since the TreeNode constructor requires
        the 'name' argument, this method pulls the data from the database row directly."""
        return TreeNode(row[mapper.c.name.label], _mapper_nohistory=True)
    def after_insert(self, mapper, instance):
        """runs after the insert of a new TreeNode row.  The primary key of the row is not determined
        until the insert is complete, since most DB's use autoincrementing columns.  If this node is
        the root node, we will take the new primary key and update it as the value of the node's 
        "root ID" as well, since its root node is itself."""
        if instance.root is instance:
            mapper.primarytable.update(TreeNode.c.id==instance.id, values=dict(root_node_id=instance.id)).execute()
            instance.root_id = instance.id
    def append_result(self, mapper, row, imap, result, instance, populate_existing=False):
        """runs as results from a SELECT statement are processed, and newly created or already-existing
        instances that correspond to each row are appended to result lists.  This method will only
        append root nodes to the result list, and will attach child nodes to their appropriate parent
        node as they arrive from the select results.  This allows a SELECT statement which returns
        both root and child nodes in one query to return a list of "roots"."""
        if instance.parent_id is None:
            result.append(instance)
        else:
            parentnode = imap[mapper.identity_key(instance.parent_id)]
            parentnode.children.append(instance, _mapper_nohistory=True)
        return False
            
# define the mapper.  we will make "convenient" property
# names vs. the more verbose names in the table definition

TreeNode.mapper=assignmapper(tables.trees, properties=dict(
    id=tables.trees.c.node_id,
    name=tables.trees.c.node_name,
    parent_id=tables.trees.c.parent_node_id,
    root_id=tables.trees.c.root_node_id,
    root=relation(TreeNode, primaryjoin=tables.trees.c.root_node_id==tables.trees.c.node_id, thiscol=tables.trees.c.root_node_id, lazy=None, uselist=False),
    children=relation(TreeNode, primaryjoin=tables.trees.c.parent_node_id==tables.trees.c.node_id, thiscol=tables.trees.c.node_id, lazy=None, uselist=True, private=True),
), extension = TreeLoader())
TreeNode.mapper


node2 = TreeNode('node2')
node2.append('subnode1')
node = TreeNode('rootnode')
node.append('node1')
node.append(node2)
node.append('node3')
node.children['node2'].append('subnode2')

print "\n\n\n----------------------------"
print "Created new tree structure:"
print "----------------------------"

print node.print_nodes()

print "\n\n\n----------------------------"
print "Committing:"
print "----------------------------"

objectstore.commit()

print "\n\n\n----------------------------"
print "Tree After Save:"
print "----------------------------"

print node.print_nodes()

#node.name='newname'
node.append('node4')
node.children['node4'].append('subnode3')
node.children['node4'].append('subnode4')
node.children['node4'].children['subnode3'].append('subsubnode1')
del node.children['node1']

print "\n\n\n----------------------------"
print "Modified the tree"
print "(added node4, node4/subnode3, node4/subnode4,"
print "node4/subnode3/subsubnode1, deleted node1):"
print "----------------------------"

print node.print_nodes()

print "\n\n\n----------------------------"
print "Committing:"
print "----------------------------"
objectstore.commit()

{'fraction_non_alphanumeric': 0.13734734889472872, 'fraction_numerical': 0.004792085330035554, 'mean_word_length': 2.7624309392265194, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4136225', 'n_tokens_mistral': 3675, 'n_tokens_neox': 3590, 'n_words': 1259}