<commit_before>#    This file is part of rhizi, a collaborative knowledge graph editor.
#    Copyright (C) 2014-2015  Rhizi
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.


from copy import deepcopy
import hashlib
import logging
import re
from time import time

from flask import current_app # for user_db

from .model.graph import Attr_Diff
from .model.graph import Topo_Diff
from .model.model import Link, RZDoc, RZCommit
from .neo4j_cypher import DB_Query, DB_result_set, DB_Raw_Query
from .neo4j_util import cfmt
from .neo4j_util import generate_random_id__uuid, rzdoc__ns_label, \
    quote__singlequote, rzdoc__meta_ns_label, quote__backtick
from . import neo4j_util
from . import neo4j_schema
from . import neo4j_util as db_util


log = logging.getLogger('rhizi')


class DBO_factory__default(object):
    """
    Default configuration DB op factory, providing the following function:
       - gen_op__rzdb__init_DB(): DB init op
    """

    def gen_op__rzdb__init_DB(self):
        return DBO_rzdb__init_DB(neo4j_schema.RZDOC__DEFAULT_MAINPAGE_NAME,
                                 neo4j_schema.RZDOC__NAME__MAX_LENGTH)


class DB_op(object):
    """
    Transaction (tx) wrapped DB operation possibly composing multiple DB queries
    """
    def __init__(self):
        self.query_set = []
        self.result_set = []
        self.error_set = None
        self.tx_id = None  # int
        self.tx_commit_url = None  # cached from response to tx begin

    def __iter__(self):
        for dbq in self.query_set:
            yield dbq

    def __repr__(self):
        return '%s: tx-id: %s' % (self.__class__.__name__, str(self.tx_id))  # id may be null if Tx open failed

    def iter__r_set(self):
        """
        iterate over (DB_Query index, DB_Query, result | error)
        where result & error are mutually exclusive

        note: index is zero based

        TODO: handle partial iteration due to error_set being non-empty
        """
        r_set_len = len(self.result_set)

        #
        # q_idx: statement index
        # dbq: db query
        # row-set: query result
        #
        for q_idx, dbq in enumerate(self.query_set):
            r_set = None
            if q_idx < r_set_len:  # support partial result recovery
                r_set = DB_result_set(self.result_set[q_idx])
            yield (q_idx, dbq, r_set)

    def add_db_query(self, db_q):

        assert type(db_q) == DB_Query

        self.query_set.append(db_q)
        return len(self.query_set)

    def add_statement(self, q_str_or_array, query_params={}):
        """
        add a DB query language statement

        @return: statement index (zero based)
        """
        if type(q_str_or_array) is str:
            q_arr = [q_str_or_array]
        else:
            q_arr = q_str_or_array
        db_q = DB_Query(q_arr, query_params)
        return self.add_db_query(db_q)

    @property
    def name(self):
        return self.__class__.__name__

    def process_result_set(self):
        """
        DB op can issue complex sets of queries all at once - this helper method
        assists in parsing response data from a single query.
        """
        ret = []
        for _q_idx, _dbq, r_set in self.iter__r_set():
            for row in r_set:
                for col in row:
                    ret.append(col)
        return ret

    def parse_tx_id(self, tx_commit_url):
        m = re.search('/(?P<id>\d+)/commit$', tx_commit_url)
        id_str = m.group('id')
        self.tx_id = int(id_str)

class DBO_add_node_set(DB_op):

    def __init__(self, node_map):
        """
        DB op: add node set

        @param node_map: node-type to node-set map
        @return: set of new node DB ids
        """
        super(DBO_add_node_set, self).__init__()
        for q, q_param_set in db_util.gen_query_create_from_node_map(node_map):
            self.add_statement(q, q_param_set)

    def process_result_set(self):
        n_id_set = []
        for _, _, r_set in self.iter__r_set():
            for row in r_set:
                for ret_dict in row:
                    n_id_set.append(ret_dict['id'])

        return n_id_set

class DBO_add_link_set(DB_op):

    def __init__(self, link_map):
        """
        @param link_map: is a link-type to link-set map - see model.link
        @return: set of new link DB ids
        """
        super(DBO_add_link_set, self).__init__()
        for q, q_param_set in db_util.gen_query_create_from_link_map(link_map):
            self.add_statement(q, q_param_set)

    def process_result_set(self):
        l_id_set = []
        for _, _, r_set in self.iter__r_set():
            for row in r_set:
                for ret_dict in row:
                    l_id_set.append(ret_dict['id'])

        return l_id_set

class DB_composed_op(DB_op):
    """
    A DB_op composed of sup-operations with the intention of being able to
    partially succeed in sub-op execution. This op class will reject addition
    of direct query statements.

    Note: this class may be removed in future releases.
    """
    def __init__(self):
        super(DB_composed_op, self).__init__()
        self.sub_op_set = []

    def __assert_false_statement_access(self):
        assert False, "composed_op may not contain statements, only sub-ops"

    def __getattribute__(self, attr):
        """
        intercept 'query_set' attr get
        """
        if attr == 'query_set':
            self.__assert_false_statement_access()

        return object.__getattribute__(self, attr)

    def __iter__(self):
        for s_op in self.sub_op_set:
            for dbq in s_op:
                yield dbq

    def add_statement(self, query, query_params={}):
        self.__assert_false_statement_access()

    def add_sub_op(self, op):
        self.sub_op_set.append(op)

    def iter__sub_op(self):
        for s_op in self.sub_op_set:
            yield s_op

    def post_sub_op_exec_hook(self, prv_sub_op, prv_sub_op_ret):
        """
        Called after each successful sub-op execution. Throwing an exception here
        will prevent the execution of the subsequent sub_op's.
        """
        pass

    def process_result_set(self):
        ret = []
        for s_op in self.sub_op_set:
            s_result_set = s_op.process_result_set()
            ret.append(s_result_set)
        return ret

class DBO_block_chain__commit(DB_op):
    """
    Rhizi version control
    """

    @staticmethod
    def calc_blob_hash(blob=b''):
        """
        Calculate blog hash value
        """
        sha1 = hashlib.sha1()
        sha1.update(blob)
        ret = sha1.hexdigest()
        return ret

    def __init__(self, commit_obj=None, ctx=None, meta=None):
        """
        @param commit_obj: serializable blob

        @return: old_head, new_head, new_head.hash_value
        """
        super(DBO_block_chain__commit, self).__init__()

        blob = RZCommit.blob_from_diff_obj(commit_obj)
        hash_value = self.calc_blob_hash(blob.encode('utf-8'))

        l_id = generate_random_id__uuid()

        q_arr = ['match (old_head:%s:%s)' % (neo4j_schema.META_LABEL__VC_HEAD,
                                             neo4j_schema.META_LABEL__VC_COMMIT),
                 'create (new_head:%s:%s {commit_attr})' % (neo4j_schema.META_LABEL__VC_HEAD,
                                                            neo4j_schema.META_LABEL__VC_COMMIT),
                 'create (new_head)-[r:%s {link_attr}]->(old_head)' % (neo4j_schema.META_LABEL__VC_PARENT),
                 'remove old_head:%s' % (neo4j_schema.META_LABEL__VC_HEAD),
                 'set new_head.ts_created={ts_created}',
                 'return {head_parent_commit: old_head, head_commit: new_head, ts_created: new_head.ts_created}'
                 ]

        ts_created = meta['ts_created'] if meta is not None and 'ts_created' in meta else int(time() * 1000)
        q_param_set = {'commit_attr': {'blob': blob,
                                       'hash': hash_value,
                                       'id': hash_value},
                       'link_attr': {'id': l_id},
                       'ts_created': ts_created
                       }

        self.add_statement(q_arr, q_param_set)

        # cache values necessary to generate op result
        self.commit_obj = commit_obj
        self.n_id = hash_value
        self.l_id = l_id

        # create commit-[:__Authored-by]->__User link if possible
        if None != ctx and None != ctx.user_name:
            self.add_statement(self._add_statement__authored_by(ctx.user_name))

        if None != meta and 'sentence' in meta and meta['sentence'] != '':
            result_of_param_set = {'result_of_attr': {'sentence': meta['sentence']}}
            self.add_statement(self._add_statement__result_of_sentence(ctx.user_name),
                               result_of_param_set)

    def _add_statement__authored_by(self, user_name):
        return ['merge (n:%s {user_name: \'%s\'})' % (neo4j_schema.META_LABEL__USER, user_name),
                'with n',
                'match (m:%s)' % (neo4j_schema.META_LABEL__VC_HEAD),  # FIXME: specify commit-label index
                'create (m)-[r:`%s`]->(n)' % (neo4j_schema.META_LABEL__VC_COMMIT_AUTHOR),
                ]

    def _add_statement__result_of_sentence(self, user_name):
        return ['match (head:%s:%s)' % (neo4j_schema.META_LABEL__VC_HEAD,
                                        neo4j_schema.META_LABEL__VC_COMMIT),
                'create (head)-[r:%s]->(result_of:%s {result_of_attr} )' % (
                neo4j_schema.META_LABEL__VC_COMMIT_RESULT_OF,
                neo4j_schema.META_LABEL__VC_OPERATION,
                )]

    def process_result_set(self):
        """
        @return: a Topo_Diff object consisting of the commit node and parent link
        """
        ret = Topo_Diff()

        hash_parent = None
        hash_child = None
        ts_created = None
        for _, _, r_set in self.iter__r_set():
            for row in r_set:
                for ret_dict in row:

                    assert None == hash_parent  # assert hash values set once only
                    assert None == hash_child
                    assert None == ts_created

                    hash_parent = ret_dict['head_parent_commit']['hash']
                    hash_child = ret_dict['head_commit']['hash']
                    ts_created = ret_dict['ts_created']

        ret.node_set_add = [{'id': self.n_id,
                             '__label_set': ['__Commit']}
                           ]
        l = Link.Link_Ptr(src_id=hash_parent, dst_id=hash_child)
        l['id'] = self.l_id
        l['__type'] = '__Parent'
        ret.link_set_add = [l]
        ret.meta['ts_created'] = ts_created
        return ret

class DBO_block_chain__init(DB_op):

    def __init__(self, rzdoc):
        """
        Rhizi version control - initialize meta namespace block chain
        """
        super(DBO_block_chain__init, self).__init__()

        #
        # setup meta rzdoc NS
        #
        meta_ns_label_q = quote__backtick(rzdoc__meta_ns_label(rzdoc))
        q_arr = ['create (n:%s:%s:%s {commit_attr})' % (meta_ns_label_q,
                                                        neo4j_schema.META_LABEL__VC_HEAD,
                                                        neo4j_schema.META_LABEL__VC_COMMIT,
                                                        ),
                 'set n.ts_created=timestamp()', ]

        hash_value = neo4j_schema.META_LABEL__VC_EMPTY_RZDOC_HASH
        param_set = {'commit_attr': {
                                       'blob': '',
                                       'hash': hash_value,
                                       'id': hash_value,
                                       'name': 'root-commit'},
                       }

        db_q = DB_Query(q_arr, param_set)
        self.add_db_query(db_q)

class DBO_block_chain__list(DB_op):
    """
    Return block chain hash list

    @return: hash list where last list item corresponds to earliest commit
    """

    def __init__(self, length_lim=None):
        """
        @param blob_obj: serializable blob
        """
        super(DBO_block_chain__list, self).__init__()

        # FIXME: use cleaner query:
        # match p=(n:HEAD)-[r:Parent*]->(m) return extract(n in nodes(p) | n.hash);
        q_arr = ['match (n:__HEAD)-[r:__PARENT*]->(m)',
                 'return [n.hash] + collect(m.hash)'
                 ]

        if None != length_lim:
            # inject maxHops limit if available
            q_arr[0] = "match (n:__HEAD)-[r:__PARENT*..%d]->m" % (length_lim),

        self.add_statement(q_arr)

    def process_result_set(self):
        # optimize for single statement
        for _, _, r_set in self.iter__r_set():
            for row in r_set:
                for col in row:
                    return col

class DBO_raw_query_set(DB_op):
    """
    Freeform set of DB query statements

    [!] use of this class is discouraged and should be done
        only when no other DB_op is able to handle the task
        at hand
    """
    def __init__(self, q_arr=None, q_params={}):
        super(DBO_raw_query_set, self).__init__()

        if q_arr is not None:
            self.add_statement(q_arr, q_params)

    def add_statement(self, q_arr, query_params={}):
        """
        super.add_statement() override: use raw queries
        """
        db_q = DB_Raw_Query(q_arr, query_params)
        self.query_set.append(db_q)
        return len(self.query_set)

    def add_db_query(self, db_q):
        assert False, 'DBO_raw_query_set only supports raw queries - use add_statement()'

class DBO_diff_commit__topo(DB_composed_op):

    """
    commit a Topo_Diff

    @return: a Topo_Diff of the actual committed changes
    """
    def __init__(self, topo_diff):
        super(DBO_diff_commit__topo, self).__init__()

        n_add_map = db_util.meta_attr_list_to_meta_attr_map(topo_diff.node_set_add)
        l_add_map = db_util.meta_attr_list_to_meta_attr_map(topo_diff.link_set_add, meta_attr='__type')
        l_rm_set = topo_diff.link_id_set_rm
        n_rm_set = topo_diff.node_id_set_rm

        self.n_add_map = len(n_add_map) > 0
        self.l_add_map = len(l_add_map) > 0
        self.l_rm_set = len(l_rm_set) > 0
        self.n_rm_set = len(n_rm_set) > 0

        #
        # [!] order critical
        #
        if len(n_add_map) > 0:
            op = DBO_add_node_set(n_add_map)
            self.add_sub_op(op)

        if len(l_add_map) > 0:
            op = DBO_add_link_set(l_add_map)
            self.add_sub_op(op)

        if len(l_rm_set) > 0:
            op = DBO_rm_link_set(l_rm_set)
            self.add_sub_op(op)

        if len(n_rm_set) > 0:
            op = DBO_rm_node_set(n_rm_set)
            self.add_sub_op(op)

    def process_result_set(self):
        ret_nid_set_add = []
        ret_lid_set_add = []
        ret_nid_set_rm = []
        ret_lid_set_rm = []
        it = self.iter__sub_op()

        if self.n_add_map:
            for _, _, r_set in it.next().iter__r_set():  # iterate over result sets
                for row in r_set:
                    for ret_dict in row:
                        n_id = ret_dict['id']  # see query return statement
                        ret_nid_set_add.append(n_id)

        if self.l_add_map:
            for _, _, r_set in it.next().iter__r_set():  # iterate over result sets
                for row in r_set:
                    for ret_dict in row:
                        l_id = ret_dict['id']  # see query return statement
                        ret_lid_set_add.append(l_id)

        if self.l_rm_set:
            for _, _, row_set in it.next().iter__r_set():
                for l_id in row_set:
                    ret_lid_set_rm.extend(l_id)

        if self.n_rm_set:
            for _, _, row_set in it.next().iter__r_set():
                for n_id in row_set:
                    ret_nid_set_rm.extend(n_id)

        ret = Topo_Diff.Commit_Result_Type(node_id_set_add=ret_nid_set_add,
                                           link_id_set_add=ret_lid_set_add,
                                           node_id_set_rm=ret_nid_set_rm,
                                           link_id_set_rm=ret_lid_set_rm)
        return ret


class DBO_diff_commit__attr(DB_op):
    """
    commit a Attr_Diff, return an Attr_Diff

    @return: an Attr_Diff upon success - it is important to note that written-
             to attributes do not imply necessarily that the attribute has
             actually changed, only that it has been 'touched'
    """
    def __init__(self, attr_diff):
        super(DBO_diff_commit__attr, self).__init__()

        self.op_return_value__attr_diff = deepcopy(attr_diff)  # cache copy as return value on success

        for id_attr, n_attr_diff in attr_diff.type__node.items():
            # TODO parameterize multiple attr removal<commit_msg>python 3 uses next, no it/gen.next (python 2.6 supports it as well)<commit_after>#    This file is part of rhizi, a collaborative knowledge graph editor.
#    Copyright (C) 2014-2015  Rhizi
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.


from copy import deepcopy
import hashlib
import logging
import re
from time import time

from flask import current_app # for user_db

from .model.graph import Attr_Diff
from .model.graph import Topo_Diff
from .model.model import Link, RZDoc, RZCommit
from .neo4j_cypher import DB_Query, DB_result_set, DB_Raw_Query
from .neo4j_util import cfmt
from .neo4j_util import generate_random_id__uuid, rzdoc__ns_label, \
    quote__singlequote, rzdoc__meta_ns_label, quote__backtick
from . import neo4j_util
from . import neo4j_schema
from . import neo4j_util as db_util


log = logging.getLogger('rhizi')


class DBO_factory__default(object):
    """
    Default configuration DB op factory, providing the following function:
       - gen_op__rzdb__init_DB(): DB init op
    """

    def gen_op__rzdb__init_DB(self):
        return DBO_rzdb__init_DB(neo4j_schema.RZDOC__DEFAULT_MAINPAGE_NAME,
                                 neo4j_schema.RZDOC__NAME__MAX_LENGTH)


class DB_op(object):
    """
    Transaction (tx) wrapped DB operation possibly composing multiple DB queries
    """
    def __init__(self):
        self.query_set = []
        self.result_set = []
        self.error_set = None
        self.tx_id = None  # int
        self.tx_commit_url = None  # cached from response to tx begin

    def __iter__(self):
        for dbq in self.query_set:
            yield dbq

    def __repr__(self):
        return '%s: tx-id: %s' % (self.__class__.__name__, str(self.tx_id))  # id may be null if Tx open failed

    def iter__r_set(self):
        """
        iterate over (DB_Query index, DB_Query, result | error)
        where result & error are mutually exclusive

        note: index is zero based

        TODO: handle partial iteration due to error_set being non-empty
        """
        r_set_len = len(self.result_set)

        #
        # q_idx: statement index
        # dbq: db query
        # row-set: query result
        #
        for q_idx, dbq in enumerate(self.query_set):
            r_set = None
            if q_idx < r_set_len:  # support partial result recovery
                r_set = DB_result_set(self.result_set[q_idx])
            yield (q_idx, dbq, r_set)

    def add_db_query(self, db_q):

        assert type(db_q) == DB_Query

        self.query_set.append(db_q)
        return len(self.query_set)

    def add_statement(self, q_str_or_array, query_params={}):
        """
        add a DB query language statement

        @return: statement index (zero based)
        """
        if type(q_str_or_array) is str:
            q_arr = [q_str_or_array]
        else:
            q_arr = q_str_or_array
        db_q = DB_Query(q_arr, query_params)
        return self.add_db_query(db_q)

    @property
    def name(self):
        return self.__class__.__name__

    def process_result_set(self):
        """
        DB op can issue complex sets of queries all at once - this helper method
        assists in parsing response data from a single query.
        """
        ret = []
        for _q_idx, _dbq, r_set in self.iter__r_set():
            for row in r_set:
                for col in row:
                    ret.append(col)
        return ret

    def parse_tx_id(self, tx_commit_url):
        m = re.search('/(?P<id>\d+)/commit$', tx_commit_url)
        id_str = m.group('id')
        self.tx_id = int(id_str)

class DBO_add_node_set(DB_op):

    def __init__(self, node_map):
        """
        DB op: add node set

        @param node_map: node-type to node-set map
        @return: set of new node DB ids
        """
        super(DBO_add_node_set, self).__init__()
        for q, q_param_set in db_util.gen_query_create_from_node_map(node_map):
            self.add_statement(q, q_param_set)

    def process_result_set(self):
        n_id_set = []
        for _, _, r_set in self.iter__r_set():
            for row in r_set:
                for ret_dict in row:
                    n_id_set.append(ret_dict['id'])

        return n_id_set

class DBO_add_link_set(DB_op):

    def __init__(self, link_map):
        """
        @param link_map: is a link-type to link-set map - see model.link
        @return: set of new link DB ids
        """
        super(DBO_add_link_set, self).__init__()
        for q, q_param_set in db_util.gen_query_create_from_link_map(link_map):
            self.add_statement(q, q_param_set)

    def process_result_set(self):
        l_id_set = []
        for _, _, r_set in self.iter__r_set():
            for row in r_set:
                for ret_dict in row:
                    l_id_set.append(ret_dict['id'])

        return l_id_set

class DB_composed_op(DB_op):
    """
    A DB_op composed of sup-operations with the intention of being able to
    partially succeed in sub-op execution. This op class will reject addition
    of direct query statements.

    Note: this class may be removed in future releases.
    """
    def __init__(self):
        super(DB_composed_op, self).__init__()
        self.sub_op_set = []

    def __assert_false_statement_access(self):
        assert False, "composed_op may not contain statements, only sub-ops"

    def __getattribute__(self, attr):
        """
        intercept 'query_set' attr get
        """
        if attr == 'query_set':
            self.__assert_false_statement_access()

        return object.__getattribute__(self, attr)

    def __iter__(self):
        for s_op in self.sub_op_set:
            for dbq in s_op:
                yield dbq

    def add_statement(self, query, query_params={}):
        self.__assert_false_statement_access()

    def add_sub_op(self, op):
        self.sub_op_set.append(op)

    def iter__sub_op(self):
        for s_op in self.sub_op_set:
            yield s_op

    def post_sub_op_exec_hook(self, prv_sub_op, prv_sub_op_ret):
        """
        Called after each successful sub-op execution. Throwing an exception here
        will prevent the execution of the subsequent sub_op's.
        """
        pass

    def process_result_set(self):
        ret = []
        for s_op in self.sub_op_set:
            s_result_set = s_op.process_result_set()
            ret.append(s_result_set)
        return ret

class DBO_block_chain__commit(DB_op):
    """
    Rhizi version control
    """

    @staticmethod
    def calc_blob_hash(blob=b''):
        """
        Calculate blog hash value
        """
        sha1 = hashlib.sha1()
        sha1.update(blob)
        ret = sha1.hexdigest()
        return ret

    def __init__(self, commit_obj=None, ctx=None, meta=None):
        """
        @param commit_obj: serializable blob

        @return: old_head, new_head, new_head.hash_value
        """
        super(DBO_block_chain__commit, self).__init__()

        blob = RZCommit.blob_from_diff_obj(commit_obj)
        hash_value = self.calc_blob_hash(blob.encode('utf-8'))

        l_id = generate_random_id__uuid()

        q_arr = ['match (old_head:%s:%s)' % (neo4j_schema.META_LABEL__VC_HEAD,
                                             neo4j_schema.META_LABEL__VC_COMMIT),
                 'create (new_head:%s:%s {commit_attr})' % (neo4j_schema.META_LABEL__VC_HEAD,
                                                            neo4j_schema.META_LABEL__VC_COMMIT),
                 'create (new_head)-[r:%s {link_attr}]->(old_head)' % (neo4j_schema.META_LABEL__VC_PARENT),
                 'remove old_head:%s' % (neo4j_schema.META_LABEL__VC_HEAD),
                 'set new_head.ts_created={ts_created}',
                 'return {head_parent_commit: old_head, head_commit: new_head, ts_created: new_head.ts_created}'
                 ]

        ts_created = meta['ts_created'] if meta is not None and 'ts_created' in meta else int(time() * 1000)
        q_param_set = {'commit_attr': {'blob': blob,
                                       'hash': hash_value,
                                       'id': hash_value},
                       'link_attr': {'id': l_id},
                       'ts_created': ts_created
                       }

        self.add_statement(q_arr, q_param_set)

        # cache values necessary to generate op result
        self.commit_obj = commit_obj
        self.n_id = hash_value
        self.l_id = l_id

        # create commit-[:__Authored-by]->__User link if possible
        if None != ctx and None != ctx.user_name:
            self.add_statement(self._add_statement__authored_by(ctx.user_name))

        if None != meta and 'sentence' in meta and meta['sentence'] != '':
            result_of_param_set = {'result_of_attr': {'sentence': meta['sentence']}}
            self.add_statement(self._add_statement__result_of_sentence(ctx.user_name),
                               result_of_param_set)

    def _add_statement__authored_by(self, user_name):
        return ['merge (n:%s {user_name: \'%s\'})' % (neo4j_schema.META_LABEL__USER, user_name),
                'with n',
                'match (m:%s)' % (neo4j_schema.META_LABEL__VC_HEAD),  # FIXME: specify commit-label index
                'create (m)-[r:`%s`]->(n)' % (neo4j_schema.META_LABEL__VC_COMMIT_AUTHOR),
                ]

    def _add_statement__result_of_sentence(self, user_name):
        return ['match (head:%s:%s)' % (neo4j_schema.META_LABEL__VC_HEAD,
                                        neo4j_schema.META_LABEL__VC_COMMIT),
                'create (head)-[r:%s]->(result_of:%s {result_of_attr} )' % (
                neo4j_schema.META_LABEL__VC_COMMIT_RESULT_OF,
                neo4j_schema.META_LABEL__VC_OPERATION,
                )]

    def process_result_set(self):
        """
        @return: a Topo_Diff object consisting of the commit node and parent link
        """
        ret = Topo_Diff()

        hash_parent = None
        hash_child = None
        ts_created = None
        for _, _, r_set in self.iter__r_set():
            for row in r_set:
                for ret_dict in row:

                    assert None == hash_parent  # assert hash values set once only
                    assert None == hash_child
                    assert None == ts_created

                    hash_parent = ret_dict['head_parent_commit']['hash']
                    hash_child = ret_dict['head_commit']['hash']
                    ts_created = ret_dict['ts_created']

        ret.node_set_add = [{'id': self.n_id,
                             '__label_set': ['__Commit']}
                           ]
        l = Link.Link_Ptr(src_id=hash_parent, dst_id=hash_child)
        l['id'] = self.l_id
        l['__type'] = '__Parent'
        ret.link_set_add = [l]
        ret.meta['ts_created'] = ts_created
        return ret

class DBO_block_chain__init(DB_op):

    def __init__(self, rzdoc):
        """
        Rhizi version control - initialize meta namespace block chain
        """
        super(DBO_block_chain__init, self).__init__()

        #
        # setup meta rzdoc NS
        #
        meta_ns_label_q = quote__backtick(rzdoc__meta_ns_label(rzdoc))
        q_arr = ['create (n:%s:%s:%s {commit_attr})' % (meta_ns_label_q,
                                                        neo4j_schema.META_LABEL__VC_HEAD,
                                                        neo4j_schema.META_LABEL__VC_COMMIT,
                                                        ),
                 'set n.ts_created=timestamp()', ]

        hash_value = neo4j_schema.META_LABEL__VC_EMPTY_RZDOC_HASH
        param_set = {'commit_attr': {
                                       'blob': '',
                                       'hash': hash_value,
                                       'id': hash_value,
                                       'name': 'root-commit'},
                       }

        db_q = DB_Query(q_arr, param_set)
        self.add_db_query(db_q)

class DBO_block_chain__list(DB_op):
    """
    Return block chain hash list

    @return: hash list where last list item corresponds to earliest commit
    """

    def __init__(self, length_lim=None):
        """
        @param blob_obj: serializable blob
        """
        super(DBO_block_chain__list, self).__init__()

        # FIXME: use cleaner query:
        # match p=(n:HEAD)-[r:Parent*]->(m) return extract(n in nodes(p) | n.hash);
        q_arr = ['match (n:__HEAD)-[r:__PARENT*]->(m)',
                 'return [n.hash] + collect(m.hash)'
                 ]

        if None != length_lim:
            # inject maxHops limit if available
            q_arr[0] = "match (n:__HEAD)-[r:__PARENT*..%d]->m" % (length_lim),

        self.add_statement(q_arr)

    def process_result_set(self):
        # optimize for single statement
        for _, _, r_set in self.iter__r_set():
            for row in r_set:
                for col in row:
                    return col

class DBO_raw_query_set(DB_op):
    """
    Freeform set of DB query statements

    [!] use of this class is discouraged and should be done
        only when no other DB_op is able to handle the task
        at hand
    """
    def __init__(self, q_arr=None, q_params={}):
        super(DBO_raw_query_set, self).__init__()

        if q_arr is not None:
            self.add_statement(q_arr, q_params)

    def add_statement(self, q_arr, query_params={}):
        """
        super.add_statement() override: use raw queries
        """
        db_q = DB_Raw_Query(q_arr, query_params)
        self.query_set.append(db_q)
        return len(self.query_set)

    def add_db_query(self, db_q):
        assert False, 'DBO_raw_query_set only supports raw queries - use add_statement()'

class DBO_diff_commit__topo(DB_composed_op):

    """
    commit a Topo_Diff

    @return: a Topo_Diff of the actual committed changes
    """
    def __init__(self, topo_diff):
        super(DBO_diff_commit__topo, self).__init__()

        n_add_map = db_util.meta_attr_list_to_meta_attr_map(topo_diff.node_set_add)
        l_add_map = db_util.meta_attr_list_to_meta_attr_map(topo_diff.link_set_add, meta_attr='__type')
        l_rm_set = topo_diff.link_id_set_rm
        n_rm_set = topo_diff.node_id_set_rm

        self.n_add_map = len(n_add_map) > 0
        self.l_add_map = len(l_add_map) > 0
        self.l_rm_set = len(l_rm_set) > 0
        self.n_rm_set = len(n_rm_set) > 0

        #
        # [!] order critical
        #
        if len(n_add_map) > 0:
            op = DBO_add_node_set(n_add_map)
            self.add_sub_op(op)

        if len(l_add_map) > 0:
            op = DBO_add_link_set(l_add_map)
            self.add_sub_op(op)

        if len(l_rm_set) > 0:
            op = DBO_rm_link_set(l_rm_set)
            self.add_sub_op(op)

        if len(n_rm_set) > 0:
            op = DBO_rm_node_set(n_rm_set)
            self.add_sub_op(op)

    def process_result_set(self):
        ret_nid_set_add = []
        ret_lid_set_add = []
        ret_nid_set_rm = []
        ret_lid_set_rm = []
        it = self.iter__sub_op()

        if self.n_add_map:
            for _, _, r_set in next(it).iter__r_set():  # iterate over result sets
                for row in r_set:
                    for ret_dict in row:
                        n_id = ret_dict['id']  # see query return statement
                        ret_nid_set_add.append(n_id)

        if self.l_add_map:
            for _, _, r_set in next(it).iter__r_set():  # iterate over result sets
                for row in r_set:
                    for ret_dict in row:
                        l_id = ret_dict['id']  # see query return statement
                        ret_lid_set_add.append(l_id)

        if self.l_rm_set:
            for _, _, row_set in next(it).iter__r_set():
                for l_id in row_set:
                    ret_lid_set_rm.extend(l_id)

        if self.n_rm_set:
            for _, _, row_set in next(it).iter__r_set():
                for n_id in row_set:
                    ret_nid_set_rm.extend(n_id)

        ret = Topo_Diff.Commit_Result_Type(node_id_set_add=ret_nid_set_add,
                                           link_id_set_add=ret_lid_set_add,
                                           node_id_set_rm=ret_nid_set_rm,
                                           link_id_set_rm=ret_lid_set_rm)
        return ret


class DBO_diff_commit__attr(DB_op):
    """
    commit a Attr_Diff, return an Attr_Diff

    @return: an Attr_Diff upon success - it is important to note that written-
             to attributes do not imply necessarily that the attribute has
             actually changed, only that it has been 'touched'
    """
    def __init__(self, attr_diff):
        super(DBO_diff_commit__attr, self).__init__()

        self.op_return_value__attr_diff = deepcopy(attr_diff)  # cache copy as return value on success

        for id_attr, n_attr_diff in attr_diff.type__node.items():
            # TODO parameterize multiple attr removal
{'fraction_non_alphanumeric': 0.0889191334921175, 'fraction_numerical': 0.002977203130316434, 'mean_word_length': 1.978046103183315, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 48, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5236535', 'n_tokens_mistral': 11311, 'n_tokens_neox': 10971, 'n_words': 2596}