<filename>bbc1/core/bbc_app.py
# -*- coding: utf-8 -*-
"""
Copyright (c) 2017 beyond-blockchain.org.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import gevent
from gevent import monkey
monkey.patch_all()
from gevent import socket
import traceback
import queue
import hashlib
import bson
import os

import sys
sys.path.append("../../")

import bbclib
from bbc1.core import message_key_types, logger
from bbclib import MsgType
from bbc1.core.message_key_types import KeyType, PayloadType
from bbc1.core.bbc_error import *

DEFAULT_CORE_PORT = 9000
DEFAULT_P2P_PORT = 6641

MESSAGE_WITH_NO_RESPONSE = (MsgType.MESSAGE, MsgType.REGISTER, MsgType.UNREGISTER, MsgType.DOMAIN_PING,
                            MsgType.REQUEST_INSERT_NOTIFICATION, MsgType.CANCEL_INSERT_NOTIFICATION,
                            MsgType.REQUEST_REPAIR)


def _parse_one_level_list(dat):
    """Get list information from queued message
    Args:
        dat (bytes): received message data
    Returns:
        list: list of information
    """
    results = []
    count = int.from_bytes(dat[:2], 'big')
    for i in range(count):
        base = 2 + 32 * i
        results.append(dat[base:base + 32])
    return results


def _parse_two_level_dict(dat):
    """Get hierarchical list information from queued message
    Args:
        dat (bytes): received message data
    Returns:
        dict: dictionary of information list
    """
    results = dict()
    count = int.from_bytes(dat[:2], 'big')
    ptr = 2
    for i in range(count):
        first_id = dat[ptr:ptr+32]
        ptr += 32
        results[first_id] = list()
        count2 = int.from_bytes(dat[ptr:ptr+2], 'big')
        ptr += 2
        for j in range(count2):
            second_id = dat[ptr:ptr+32]
            ptr += 32
            results[first_id].append(second_id)
    return results


class BBcAppClient:
    """Basic functions for a client of bbc_core"""
    def __init__(self, host='127.0.0.1', port=DEFAULT_CORE_PORT, multiq=True, logname="-", loglevel="none"):
        self.logger = logger.get_logger(key="bbc_app", level=loglevel, logname=logname)
        self.connection = socket.create_connection((host, port))
        self.callback = Callback(log=self.logger)
        self.callback.set_client(self)
        self.keypair = None
        self.node_keypair = None
        self.use_query_id_based_message_wait = multiq
        self.user_id = None
        self.domain_id = None
        self.query_id = (0).to_bytes(2, 'little')
        self.privatekey_for_ecdh = None
        self.aes_key_name = None
        self.is_secure_connection = False
        self.cross_ref_list = list()
        self.start_receiver_loop()

    def set_callback(self, callback_obj):
        """Set callback object that implements message processing functions

        Args:
            callback_obj (obj): callback method object
        """
        self.callback = callback_obj
        self.callback.set_logger(self.logger)
        self.callback.set_client(self)

    def set_domain_id(self, domain_id):
        """Set domain_id to this client to include it in all messages

        Args:
            domain_id (bytes): domain_id to join in
        """
        self.domain_id = domain_id

    def set_user_id(self, identifier):
        """Set user_id of the object

        Args:
            identifier (bytes): user_id of this clients
        """
        self.user_id = identifier

    def set_keypair(self, keypair):
        """Set keypair for the user

        Args:
            keypair (KeyPair): KeyPair object for signing
        """
        self.keypair = keypair

    def set_node_key(self, pem_file=None):
        """Set node_key to this client

        Args:
            pem_file (str): path string for the pem file
        """
        if pem_file is None:
            self.node_keypair = None
        try:
            self.node_keypair = bbclib.KeyPair()
            with open(pem_file, "r") as f:
                self.node_keypair.mk_keyobj_from_private_key_pem(f.read())
        except:
            return

    def include_admin_info(self, dat, admin_info, keypair):
        if keypair is not None:
            dat[KeyType.admin_info] = message_key_types.make_TLV_formatted_message(admin_info)
            digest = hashlib.sha256(dat[KeyType.admin_info]).digest()
            dat[KeyType.nodekey_signature] = keypair.sign(digest)
        else:
            dat.update(admin_info)

    def _make_message_structure(self, cmd):
        """Make a base message structure for sending to the core node

        Args:
            cmd (bytes): command type defined in bbclib.MsgType class
        """
        self.query_id = ((int.from_bytes(self.query_id, 'little') + 1) % 65536).to_bytes(2, 'little')
        msg = {
            KeyType.command: cmd,
            KeyType.domain_id: self.domain_id,
            KeyType.source_user_id: self.user_id,
            KeyType.status: ESUCCESS,
        }
        if cmd not in MESSAGE_WITH_NO_RESPONSE:
            msg[KeyType.query_id] = self.query_id
            if self.use_query_id_based_message_wait:
                if self.query_id not in self.callback.query_queue:
                    self.callback.create_queue(self.query_id)
        return msg

    def _send_msg(self, dat):
        """Send the message to the core node

        Args:
            dat (dict): message object to send
        Returns:
            bytes: query ID for request/response type message
        """
        if KeyType.domain_id not in dat or KeyType.source_user_id not in dat:
            self.logger.warn("Message must include domain_id and source_id")
            return None
        try:
            if self.is_secure_connection:
                msg = message_key_types.make_message(PayloadType.Type_encrypted_msgpack, dat, key_name=self.aes_key_name)
            else:
                msg = message_key_types.make_message(PayloadType.Type_msgpack, dat)
            self.connection.sendall(msg)
        except Exception as e:
            self.logger.error(traceback.format_exc())
            return None
        return self.query_id

    def exchange_key(self):
        """Perform ECDH (key exchange algorithm)

        Returns:
            bytes: query_id
        """
        if self.domain_id is None:
            self.logger.error("Need to set domain first!")
            return None
        dat = self._make_message_structure(MsgType.REQUEST_ECDH_KEY_EXCHANGE)
        self.privatekey_for_ecdh, dat[KeyType.ecdh], self.aes_key_name = message_key_types.get_ECDH_parameters()
        dat[KeyType.nonce] = os.urandom(16)
        dat[KeyType.hint] = self.aes_key_name
        dat[KeyType.random] = os.urandom(8)
        return self._send_msg(dat)

    def domain_setup(self, domain_id, config=None):
        """Set up domain with the specified network module and storage

        This method should be used by a system administrator.

        Args:
            domain_id (bytes): domain_id to create
            config (str): system config in json format
        Returns:
            bytes: query_id
        """
        dat = self._make_message_structure(MsgType.REQUEST_SETUP_DOMAIN)
        admin_info = {
            KeyType.domain_id: domain_id,
            KeyType.random: bbclib.get_random_value(32)
        }
        if config is not None:
            admin_info[KeyType.bbc_configuration] = config
        self.include_admin_info(dat, admin_info, self.node_keypair)
        return self._send_msg(dat)

    def domain_close(self, domain_id=None):
        """Close domain leading to remove_domain in the core

        Args:
            domain_id (bytes): domain_id to delete
        Returns:
            bytes: query_id
        """
        if domain_id is None and self.domain_id is not None:
            domain_id = self.domain_id
        if domain_id is None:
            return None
        dat = self._make_message_structure(MsgType.REQUEST_CLOSE_DOMAIN)
        admin_info = {
            KeyType.domain_id: domain_id,
            KeyType.random: bbclib.get_random_value(32)
        }
        self.include_admin_info(dat, admin_info, self.node_keypair)
        return self._send_msg(dat)

    def get_node_id(self):
        """Get node_id of the connecting core node

        Returns:
            bytes: query_id
        """
        dat = self._make_message_structure(MsgType.REQUEST_GET_NODEID)
        return self._send_msg(dat)

    def get_domain_neighborlist(self, domain_id):
        """Get peer list of the domain from the core node

        This method should be used by a system administrator.

        Args:
            domain_id (bytes): domain_id of the neighbor list
        Returns:
            bytes: query_id
        """
        dat = self._make_message_structure(MsgType.REQUEST_GET_NEIGHBORLIST)
        dat[KeyType.domain_id] = domain_id
        admin_info = {
            KeyType.random: bbclib.get_random_value(32)
        }
        self.include_admin_info(dat, admin_info, self.node_keypair)
        return self._send_msg(dat)

    def set_domain_static_node(self, domain_id, node_id, ipv4, ipv6, port):
        """Set static node to the core node

        IPv6 is used for socket communication if both IPv4 and IPv6 is specified.
        This method should be used by a system administrator.

        Args:
            domain_id (bytes): target domain_id to set static neighbor entry
            node_id (bytes): node_id to register
            ipv4 (str): IPv4 address of the node
            ipv6 (str): IPv6 address of the node
            port (int): Port number to wait messages (UDP/TCP)
        Returns:
            bytes: query_id
        """
        dat = self._make_message_structure(MsgType.REQUEST_SET_STATIC_NODE)
        dat[KeyType.domain_id] = domain_id
        admin_info = {
            KeyType.node_info: [node_id, ipv4, ipv6, port]
        }
        self.include_admin_info(dat, admin_info, self.node_keypair)
        return self._send_msg(dat)

    def send_domain_ping(self, domain_id, ipv4=None, ipv6=None, port=DEFAULT_P2P_PORT):
        """ Send domain ping to notify the existence of the node

        This method should be used by a system administrator.

        Args:
            domain_id (bytes): target domain_id to send ping
            ipv4 (str): IPv4 address of the node
            ipv6 (str): IPv6 address of the node
            port (int): Port number to wait messages UDP
        Returns:
            bytes: query_id
        """
        if ipv4 is None and ipv6 is None:
            return
        dat = self._make_message_structure(MsgType.DOMAIN_PING)
        dat[KeyType.domain_id] = domain_id
        admin_info = dict()
        if ipv4 is not None and ipv4 != "0.0.0.0":
            admin_info[KeyType.ipv4_address] = ipv4
        if ipv6 is not None and ipv6 != "::":
            admin_info[KeyType.ipv6_address] = ipv6
        admin_info[KeyType.port_number] = port
        admin_info[KeyType.static_entry] = True
        self.include_admin_info(dat, admin_info, self.node_keypair)
        return self._send_msg(dat)

    def get_bbc_config(self):
        """Get config file of bbc_core

        This method should be used by a system administrator.

        Returns:
            bytes: query_id
        """
        dat = self._make_message_structure(MsgType.REQUEST_GET_CONFIG)
        admin_info = {
            KeyType.random: bbclib.get_random_value(32)
        }
        self.include_admin_info(dat, admin_info, self.node_keypair)
        return self._send_msg(dat)

    def notify_domain_key_update(self):
        """Notify update of bbc_core

        This method should be used by a system administrator.

        Returns:
            bytes: query_id
        """
        dat = self._make_message_structure(MsgType.NOTIFY_DOMAIN_KEY_UPDATE)
        admin_info = {
            KeyType.random: bbclib.get_random_value(32)
        }
        self.include_admin_info(dat, admin_info, self.node_keypair)
        return self._send_msg(dat)

    def get_domain_list(self):
        """Get domain_id list in bbc_core

        Returns:
            bytes: query_id
        """
        dat = self._make_message_structure(MsgType.REQUEST_GET_DOMAINLIST)
        admin_info = {
            KeyType.random: bbclib.get_random_value(32)
        }
        self.include_admin_info(dat, admin_info, self.node_keypair)
        return self._send_msg(dat)

    def get_user_list(self):
        """Get user_ids in the domain that are connecting to the core node

        Returns:
            bytes: query_id
        """
        dat = self._make_message_structure(MsgType.REQUEST_GET_USERS)
        admin_info = {
            KeyType.random: bbclib.get_random_value(32)
        }
        self.include_admin_info(dat, admin_info, self.node_keypair)
        return self._send_msg(dat)

    def get_forwarding_list(self):
        """Get forwarding_list of the domain in the core node

        Returns:
            bytes: query_id
        """
        dat = self._make_message_structure(MsgType.REQUEST_GET_FORWARDING_LIST)
        admin_info = {
            KeyType.random: bbclib.get_random_value(32)
        }
        self.include_admin_info(dat, admin_info, self.node_keypair)
        return self._send_msg(dat)

    def get_notification_list(self):
        """Get notification_list of the core node

        Returns:
            bytes: query_id
        """
        dat = self._make_message_structure(MsgType.REQUEST_GET_NOTIFICATION_LIST)
        admin_info = {
            KeyType.random: bbclib.get_random_value(32)
        }
        self.include_admin_info(dat, admin_info, self.node_keypair)
        return self._send_msg(dat)

    def manipulate_ledger_subsystem(self, enable=False, domain_id=None):
        """Start/stop ledger_subsystem on the bbc_core

        This method should be used by a system administrator.

        Args:
            enable (bool): True->start, False->stop
            domain_id (bytes): target domain_id to enable/disable ledger_subsystem
        Returns:
            bytes: query_id
        """
        dat = self._make_message_structure(MsgType.REQUEST_MANIP_LEDGER_SUBSYS)
        dat[KeyType.domain_id] = domain_id
        admin_info = {
            KeyType.ledger_subsys_manip: enable,
            KeyType.random: bbclib.get_random_value(32)
        }
        self.include_admin_info(dat, admin_info, self.node_keypair)
        return self._send_msg(dat)

    def register_to_core(self, on_multiple_nodes=False):
        """Register the client (user_id) to the core node

        After that, the client can communicate with the core node.

        Args:
            on_multiple_nodes (bool): True if this user_id is for multicast address
        Returns:
            bool: True
        """
        dat = self._make_message_structure(MsgType.REGISTER)
        if on_multiple_nodes:
            dat[KeyType.on_multinodes] = True
        self._send_msg(dat)
        return True

    def unregister_from_core(self):
        """Unregister and disconnect from the core node

        Returns:
            bool: True
        """
        dat = self._make_message_structure(MsgType.UNREGISTER)
        self._send_msg(dat)
        if self.aes_key_name is not None:
            message_key_types.unset_cipher(self.aes_key_name)
            self.privatekey_for_ecdh = None
            self.aes_key_name = None
            self.is_secure_connection = False
        return True

    def request_insert_completion_notification(self, asset_group_id):
        """Request notification when a transaction has been inserted (as a copy of transaction)

        Args:
            asset_group_id (bytes): asset_group_id for requesting notification about insertion
        Returns:
            bytes: query_id
        """
        dat = self._make_message_structure(MsgType.REQUEST_INSERT_NOTIFICATION)
        dat[KeyType.asset_group_id] = asset_group_id
        return self._send_msg(dat)

    def cancel_insert_completion_notification(self, asset_group_id):
        """Cancel notification when a transaction has been inserted (as a copy of transaction)

        Args:
            asset_group_id (bytes): asset_group_id for requesting notification about insertion
        Returns:
            bytes: query_id
        """
        dat = self._make_message_structure(MsgType.CANCEL_INSERT_NOTIFICATION)
        dat[KeyType.asset_group_id] = asset_group_id
        return self._send_msg(dat)

    def gather_signatures(self, txobj, reference_obj=None, asset_files=None, destinations=None, anycast=False):
        """Request to gather signatures from the specified user_ids

        Args:
            txobj (BBcTransaction):
            reference_obj (BBcReference): BBcReference object that includes the information about destinations
            asset_files (dict): mapping from asset_id to its file content
            destinations (list): list of destination user_ids
            anycast (bool): True if this message is for anycasting
        Returns:
            bytes: query_id
        """
        if reference_obj is None and destinations is None:
            return False
        dat = self._make_message_structure(MsgType.REQUEST_GATHER_SIGNATURE)
        dat[KeyType.transaction_data] = bbclib.serialize(txobj)
        dat[KeyType.transaction_id] = txobj.transaction_id
        if anycast:
            dat[KeyType.is_anycast] = True
        if reference_obj is not None:
            dat[KeyType.destination_user_ids] = reference_obj.get_destinations()
            referred_transactions = {reference_obj.transaction_id: bbclib.serialize(reference_obj.ref_transaction)}
            if len(referred_transactions) > 0:
                dat[KeyType.transactions] = referred_transactions
        elif destinations is not None:
            dat[KeyType.destination_user_ids] = destinations
        if isinstance(asset_files, dict):
            dat[KeyType.all_asset_files] = asset_files
        return self._send_msg(dat)

    def sendback_signature(self, dest_user_id=None, transaction_id=None, ref_index=-1, signature=None, query_id=None):
        """Send back the signed transaction to the source

        This method is called if the receiver (signer) approves the transaction.

        Args:
            dest_user_id (bytes): destination user_id to send back
            transaction_id (bytes):
            ref_index (int): (optional) which reference in transaction the signature is for
            signature (BBcSignature): Signature that expresses approval of the transaction with transaction_id
            query_id: The query_id that was in the received SIGN_REQUEST message
        Returns:
            bytes: query_id
        """
        dat = self._make_message_structure(MsgType.RESPONSE_SIGNATURE)
        dat[KeyType.destination_user_id] = dest_user_id
        dat[KeyType.transaction_id] = transaction_id
        dat[KeyType.ref_index] = ref_index
        dat[KeyType.signature] = signature.pack()
        if query_id is not None:
            dat[KeyType.query_id] = query_id
        return self._send_msg(dat)

    def sendback_denial_of_sign(self, dest_user_id=None, transaction_id=None, reason_text=None, query_id=None):
        """Send back the denial of sign the transaction

        This method is called if the receiver (signer) denies the transaction.

        Args:
            dest_user_id (bytes): destination user_id to send back
            transaction_id (bytes):
            reason_text (str): message to the requester about why the node denies the transaction
            query_id: The query_id that was in the received SIGN_REQUEST message
        Returns:
            bytes: query_id
        """
        dat = self._make_message_structure(MsgType.RESPONSE_SIGNATURE)
        dat[KeyType.destination_user_id] = dest_user_id
        dat[KeyType.transaction_id] = transaction_id
        dat[KeyType.status] = EOTHER
        dat[KeyType.reason] = reason_text
        if query_id is not None:
            dat[KeyType.query_id] = query_id
        return self._send_msg(dat)

    def insert_transaction(self, txobj):
        """Request to insert a legitimate transaction

        Args:
            txobj (BBcTransaction): Transaction object to insert
        Returns:
            bytes: query_id
        """
        if txobj.transaction_id is None:
            txobj.digest()
        dat = self._make_message_structure(MsgType.REQUEST_INSERT)
        dat[KeyType.transaction_data] = bbclib.serialize(txobj)
        ast = dict()
        for evt in txobj.events:
            if evt.asset is None:
                continue
            asset_digest, content = evt.asset.get_asset_file()
            if content is not None:
                ast[evt.asset.asset_id] = content
        for rtn in txobj.relations:
            if rtn.asset is None:
                continue
            asset_digest, content = rtn.asset.get_asset_file()
            if content is not None:
                ast[rtn.asset.asset_id] = content
        dat[KeyType.all_asset_files] = ast
        return self._send_msg(dat)

    def search_transaction_with_condition(self, asset_group_id=None, asset_id=None, user_id=None,
                                          start_from=None, until=None, direction=0, count=0):
        """Search transaction data by asset_group_id/asset_id/user_id

        If multiple conditions are specified, they are considered as AND condition.

        Args:
            asset_group_id (bytes): asset_group_id in BBcEvent and BBcRelations
            asset_id (bytes): asset_id in BBcAsset
            user_id (bytes): user_id in BBcAsset that means the owner of the asset
            start_from (int): the starting timestamp to search
            until (int): the end timestamp to search
            direction (int): 0: descend, 1: ascend
            count (int): the number of transactions to retrieve
        Returns:
            bytes: query_id
        """
        dat = self._make_message_structure(MsgType.REQUEST_SEARCH_WITH_CONDITIONS)
        if asset_group_id is not None:
            dat[KeyType.asset_group_id] = asset_group_id
        if asset_id is not None:
            dat[KeyType.asset_id] = asset_id
        if user_id is not None:
            dat[KeyType.user_id] = user_id
        if start_from is not None:
            dat[KeyType.start_from] = start_from
        if until is not None:
            dat[KeyType.until] = until
        dat[KeyType.direction] = direction
        dat[KeyType.count] = count
        return self._send_msg(dat)

    def search_transaction(self, transaction_id):
        """Search request for a transaction

        Args:
            transaction_id (bytes): the target transaction to retrieve
        Returns:
            bytes: query_id
        """
        dat = self._make_message_structure(MsgType.REQUEST_SEARCH_TRANSACTION)
        dat[KeyType.transaction_id] = transaction_id
        return self._send_msg(dat)

    def count_transactions(self, asset_group_id=None, asset_id=None, user_id=None, start_from=None, until=None):
        """Count transactions that matches the given conditions

        If multiple conditions are specified, they are considered as AND condition.

        Args:
            asset_group_id (bytes): asset_group_id in BBcEvent and BBcRelations
            asset_id (bytes): asset_id in BBcAsset
            user_id (bytes): user_id in BBcAsset that means the owner of the asset
            start_from (int): the starting timestamp to search
            until (int): the end timestamp to search
        Returns:
            int: the number of transactions
        """
        dat = self._make_message_structure(MsgType.REQUEST_COUNT_TRANSACTIONS)
        if asset_group_id is not None:
            dat[KeyType.asset_group_id] = asset_group_id
        if asset_id is not None:
            dat[KeyType.asset_id] = asset_id
        if user_id is not None:
            dat[KeyType.user_id] = user_id
        if start_from is not None:
            dat[KeyType.start_from] = start_from
        if until is not None:
            dat[KeyType.until] = until
        return self._send_msg(dat)

    def traverse_transactions(self, transaction_id, asset_group_id=None, user_id=None, start_from=None, until=None, direction=1, hop_count=3):
        """Search request for transactions

        The method traverses the transaction graph in the ledger.
        The response from the bbc_core includes the list of transactions.

        Args:
            transaction_id (bytes): the target transaction to retrieve
            asset_group_id (bytes): asset_group_id that target transactions should have
            user_id (bytes): user_id that target transactions should have
            start_from (int): the starting timestamp to search
            until (int): the end timestamp to search
            direction (int): 1:backforward, non-1:forward
            hop_count (int): hop count to traverse from the specified origin point
        Returns:
            bytes: query_id
        """
        dat = self._make_message_structure(MsgType.REQUEST_TRAVERSE_TRANSACTIONS)
        dat[KeyType.transaction_id] = transaction_id
        if asset_group_id is not None:
            dat[KeyType.asset_group_id] = asset_group_id
        if user_id is not None:
            dat[KeyType.user_id] = user_id
        if start_from is not None:
            dat[KeyType.start_from] = start_from
        if until is not None:
            dat[KeyType.until] = until
        dat[KeyType.direction] = direction
        dat[KeyType.hop_count] = hop_count
        return self._send_msg(dat)

    def request_to_repair_transaction(self, transaction_id):
        """Request to repair compromised transaction data

        Args:
            transaction_id (bytes): the target transaction to repair
        Returns:
            bytes: query_id
        """
        dat = self._make_message_structure(MsgType.REQUEST_REPAIR)
        dat[KeyType.transaction_id] = transaction_id
        return self._send_msg(dat)

    def request_to_repair_asset(self, asset_group_id, asset_id):
        """Request to repair compromised asset file

        Args:
            asset_group_id (bytes): the asset_group_id of the target asset
            asset_id (bytes): the target asset_id
        Returns:
            bytes: query_id
        """
        dat = self._make_message_structure(MsgType.REQUEST_REPAIR)
        dat[KeyType.asset_group_id] = asset_group_id
        dat[KeyType.asset_id] = asset_id
        return self._send_msg(dat)

    def request_verify_by_cross_ref(self, transaction_id):
        """Request to verify the transaction by Cross_ref in transaction of outer domain

        Args:
            transaction_id (bytes): the target transaction_id
        Returns:
            bytes: query_id
        """
        dat = self._make_message_structure(MsgType.REQUEST_CROSS_REF_VERIFY)
        dat[KeyType.transaction_id] = transaction_id
        return self._send_msg(dat)

    def request_cross_ref_holders_list(self):
        """Request the list of transaction_ids that are registered as cross_ref in outer domains

        Returns:
            bytes: query_id
        """
        dat = self._make_message_structure(MsgType.REQUEST_CROSS_REF_LIST)
        # TODO: need to limit the number of entries??
        return self._send_msg(dat)

    def register_in_ledger_subsystem(self, asset_group_id, transaction_id):
        """Register transaction_id in the ledger_subsystem

        Args:
            asset_group_id (bytes):
            transaction_id (bytes): the target transaction_id
        Returns:
            bytes: query_id
        """
        dat = self._make_message_structure(MsgType.REQUEST_REGISTER_HASH_IN_SUBSYS)
        dat[KeyType.transaction_id] = transaction_id
        dat[KeyType.asset_group_id] = asset_group_id
        return self._send_msg(dat)

    def verify_in_ledger_subsystem(self, asset_group_id, transaction_id):
        """Verify transaction_id in the ledger_subsystem

        Args:
            asset_group_id (bytes):
            transaction_id (bytes): the target transaction_id
        Returns:
            bytes: query_id
        """
        dat = self._make_message_structure(MsgType.REQUEST_VERIFY_HASH_IN_SUBSYS)
        dat[KeyType.transaction_id] = transaction_id
        dat[KeyType.asset_group_id] = asset_group_id
        return self._send_msg(dat)

    def get_stats(self):
        """Get statistics of bbc_core

        Returns:
            bytes: query_id
        """
        dat = self._make_message_structure(MsgType.REQUEST_GET_STATS)
        admin_info = {
            KeyType.random: bbclib.get_random_value(32)
        }
        self.include_admin_info(dat, admin_info, self.node_keypair)
        return self._send_msg(dat)

    def send_message(self, msg, dst_user_id, is_anycast=False):
        """Send a message to the specified user_id

        Args:
            msg (dict): message to send
            dst_user_id (bytes): destination user_id
            is_anycast (bool): If true, the message is treated as an anycast message.
        Returns:
            bytes: query_id
        """
        dat = self._make_message_structure(MsgType.MESSAGE)
        dat[KeyType.destination_user_id] = dst_user_id
        dat[KeyType.message] = msg
        if is_anycast:
            dat[KeyType.is_anycast] = True
        return self._send_msg(dat)

    def start_receiver_loop(self):
        jobs = [gevent.spawn(self.receiver_loop)]
        #gevent.joinall(jobs)

    def receiver_loop(self):
        msg_parser = message_key_types.Message()
        try:
            connected = True
            while connected:
                buf = self.connection.recv(8192)
                if len(buf) == 0:
                    connected = False
                    break
                msg_parser.recv(buf)
                while True:
                    msg = msg_parser.parse()
                    if msg is None:
                        break
                    self.callback.dispatch(msg, msg_parser.payload_type)
        except Exception as e:
            self.logger.info("TCP disconnect: %s" % e)
            print(traceback.format_exc())

    def include_cross_ref(self, txobj):
        """Include BBcCrossRef from other domains in the transaction

        If the client object has one or more cross_ref objects, one of them is included in the given transaction.
        This method should be voluntarily called for inter-domain weak collaboration.

        Args:
            txobj (BBcTransaction): Transaction object to include cross_ref
        """
        if len(self.cross_ref_list) > 0:
            txobj.add(cross_ref=self.cross_ref_list.pop(0))


class Callback:
    """Set of callback functions for processing received message

    If you want to implement your own way to process messages, inherit this class.
    """
    def __init__(self, log=None):
        self.logger = log
        self.client = None
        self.queue = queue.Queue()
        self.query_queue = dict()

    def set_logger(self, log):
        self.logger = log

    def set_client(self, client):
        self.client = client

    def create_queue(self, query_id):
        self.query_queue.setdefault(query_id, queue.Queue())

    def get_from_queue(self, query_id, timeout=None, no_delete=False):
        msg = self.query_queue[query_id].get(timeout=timeout)
        if not no_delete:
            del self.query_queue[query_id]
        return msg

    def dispatch(self, dat, payload_type):
        #self.logger.debug("Received: %s" % dat)
        if KeyType.command not in dat:
            self.logger.warn("No command exists")
            return
        if KeyType.query_id in dat and dat[KeyType.query_id] in self.query_queue:
            self.query_queue[dat[KeyType.query_id]].put(dat)
            return

        if dat[KeyType.command] == MsgType.RESPONSE_SEARCH_TRANSACTION:
            self.proc_resp_search_transaction(dat)
        elif dat[KeyType.command] == MsgType.RESPONSE_SEARCH_WITH_CONDITIONS:
            self.proc_resp_search_with_condition(dat)
        elif dat[KeyType.command] == MsgType.RESPONSE_COUNT_TRANSACTIONS:
            self.proc_resp_count_transactions(dat)
        elif dat[KeyType.command] == MsgType.RESPONSE_TRAVERSE_TRANSACTIONS:
            self.proc_resp_traverse_transactions(dat)
        elif dat[KeyType.command] == MsgType.RESPONSE_GATHER_SIGNATURE:
            self.proc_resp_gather_signature(dat)
        elif dat[KeyType.command] == MsgType.REQUEST_SIGNATURE:
            self.proc_cmd_sign_request(dat)
        elif dat[KeyType.command] == MsgType.RESPONSE_SIGNATURE:
            self.proc_resp_sign_request(dat)
        elif dat[KeyType.command] == MsgType.RESPONSE_INSERT:
            self.proc_resp_insert(dat)
        elif dat[KeyType.command] == MsgType.NOTIFY_INSERTED:
            self.proc_notify_inserted(dat)
        elif dat[KeyType.command] == MsgType.NOTIFY_CROSS_REF:
            self.proc_notify_cross_ref(dat)
        elif dat[KeyType.command] == MsgType.MESSAGE:
            self.proc_user_message(dat)
        elif dat[KeyType.command] == MsgType.RESPONSE_CROSS_REF_VERIFY:
            self.proc_resp_verify_cross_ref(dat)
        elif dat[KeyType.command] == MsgType.RESPONSE_CROSS_REF_LIST:
            self.proc_resp_cross_ref_list(dat)
        elif dat[KeyType.command] == MsgType.RESPONSE_REGISTER_HASH_IN_SUBSYS:
            self.proc_resp_register_hash(dat)
        elif dat[KeyType.command] == MsgType.RESPONSE_VERIFY_HASH_IN_SUBSYS:
            self.proc_resp_verify_hash(dat)
        elif dat[KeyType.command] == MsgType.RESPONSE_GET_STATS:
            self.proc_resp_get_stats(dat)
        elif dat[KeyType.command] == MsgType.RESPONSE_GET_NEIGHBORLIST:
            self.proc_resp_get_neighborlist(dat)
        elif dat[KeyType.command] == MsgType.RESPONSE_GET_DOMAINLIST:
            self.proc_resp_get_domainlist(dat)
        elif dat[KeyType.command] == MsgType.RESPONSE_GET_USERS:
            self.proc_resp_get_userlist(dat)
        elif dat[KeyType.command] == MsgType.RESPONSE_GET_FORWARDING_LIST:
            self.proc_resp_get_forwardinglist(dat)
        elif dat[KeyType.command] == MsgType.RESPONSE_GET_NOTIFICATION_LIST:
            self.proc_resp_get_notificationlist(dat)
        elif dat[KeyType.command] == MsgType.RESPONSE_GET_NODEID:
            self.proc_resp_get_node_id(dat)
        elif dat[KeyType.command] == MsgType.RESPONSE_GET_CONFIG:
            self.proc_resp_get_config(dat)
        elif dat[KeyType.command] == MsgType.RESPONSE_MANIP_LEDGER_SUBSYS:
            self.proc_resp_ledger_subsystem(dat)
        elif dat[KeyType.command] == MsgType.RESPONSE_SETUP_DOMAIN:
            self.proc_resp_domain_setup(dat)
        elif dat[KeyType.command] == MsgType.RESPONSE_SET_STATIC_NODE:
            self.proc_resp_set_neighbor(dat)
        elif dat[KeyType.command] == MsgType.RESPONSE_CLOSE_DOMAIN:
            self.proc_resp_domain_close(dat)
        elif dat[KeyType.command] == MsgType.RESPONSE_ECDH_KEY_EXCHANGE:
            self.proc_resp_ecdh_key_exchange(dat)
        else:
            self.logger.warn("No method to process for command=%d" % dat[KeyType.command])

    def synchronize(self, timeout=None):
        """Wait for receiving message with a common queue

        Args:
            timeout (int): timeout for waiting a message in seconds
        Returns:
            dict: a received message
        """
        try:
            return self.queue.get(timeout=timeout)
        except:
            return None

    def sync_by_queryid(self, query_id, timeout=None, no_delete_q=False):
        """Wait for the message with specified query_id

        This method creates a queue for the query_id and waits for the response

        Args:
            query_id (byte): timeout for waiting a message in seconds
            timeout (int): timeout for waiting a message in seconds
            no_delete_q (bool): If True, the queue for the query_id remains after popping a message
        Returns:
            dict: a received message
        """
        try:
            if query_id not in self.query_queue:
                self.create_queue(query_id)
            return self.get_from_queue(query_id, timeout=timeout)
        except:
            return None

    def proc_notify_cross_ref(self, dat):
        """Callback for message NOTIFY_CROSS_REF

        This method must not be overridden.

        Args:
            dat (dict): received message
        """
        cross_ref = bbclib.BBcCrossRef(domain_id=dat[KeyType.cross_ref][0], transaction_id=dat[KeyType.cross_ref][1])
        self.client.cross_ref_list.append(cross_ref)

    def proc_cmd_sign_request(self, dat):
        """Callback for message REQUEST_SIGNATURE

        This method should be overridden if you want to process the message asynchronously.

        Args:
            dat (dict): received message
        """
        self.queue.put(dat)

    def proc_resp_sign_request(self, dat):
        """Callback for message RESPONSE_SIGNATURE

        This method should be overridden if you want to process the message asynchronously.

        Args:
            dat (dict): received message
        """
        self.queue.put(dat)

    def proc_resp_gather_signature(self, dat):
        """Callback for message RESPONSE_GATHER_SIGNATURE

        This method should be overridden if you want to process the message asynchronously.

        Args:
            dat (dict): received message
        """
        if KeyType.status not in dat or dat[KeyType.status] < ESUCCESS:
            self.queue.put(dat)
            return
        sig = bbclib.recover_signature_object(dat[KeyType.signature])
        self.queue.put({KeyType.status: ESUCCESS, KeyType.result: (dat[KeyType.ref_index], dat[KeyType.source_user_id], sig)})

    def proc_resp_insert(self, dat):
        """Callback for message RESPONSE_INSERT

        This method should be overridden if you want to process the message asynchronously.

        Args:
            dat (dict): received message
        """
        self.queue.put(dat)

    def proc_notify_inserted(self, dat):
        """Callback for message NOTIFY_INSERTED

        This method should be overridden if you want to process the message asynchronously.

        Args:
            dat (dict): received message
        """
        self.queue.put(dat)

    def proc_resp_search_with_condition(self, dat):
        """Callback for message RESPONSE_SEARCH_WITH_CONDITIONS

        This method should be overridden if you want to process the message asynchronously.

        Args:
            dat (dict): received message
        """
        self.queue.put(dat)

    def proc_resp_search_transaction(self, dat):
        """Callback for message RESPONSE_SEARCH_TRANSACTION

        This method should be overridden if you want to process the message asynchronously.

        Args:
            dat (dict): received message
        """
        self.queue.put(dat)

    def proc_resp_count_transactions(self, dat):
        """Callback for message RESPONSE_COUNT_TRANSACTIONS

        This method should be overridden if you want to process the message asynchronously.

        Args:
            dat (dict): received message
        """
        self.queue.put(dat)

    def proc_resp_traverse_transactions(self, dat):
        """Callback for message RESPONSE_TRAVERSE_TRANSACTIONS

        This method should be overridden if you want to process the message asynchronously.

        Args:
            dat (dict): received message
        """
        self.queue.put(dat)

    def proc_user_message(self, dat):
        """Callback for message MESSAGE

        This method should be overridden if you want to process the message asynchronously.

        Args:
            dat (dict): received message
        """
        self.queue.put(dat)

    def proc_resp_verify_cross_ref(self, dat):
        """Callback for message RESPONSE_CROSS_REF_VERIFY

        This method should be overridden if you want to process the message asynchronously.

        Args:
            dat (dict): received message
        """
        self.queue.put(dat)

    def proc_resp_cross_ref_list(self, dat):
        """Callback for message RESPONSE_CROSS_REF_LIST

        This method should be overridden if you want to process the message asynchronously.

        Args:
            dat (dict): received message
        """
        self.queue.put(dat)

    def proc_resp_ledger_subsystem(self, dat):
        """Callback for message RESPONSE_MANIP_LEDGER_SUBSYS

        This method should be overridden if you want to process the message asynchronously.

        Args:
            dat (dict): received message
        """
        self.queue.put(dat)

    def proc_resp_register_hash(self, dat):
        """Callback for message RESPONSE_REGISTER_HASH_IN_SUBSYS

        This method should be overridden if you want to process the message asynchronously.

        Args:
            dat (dict): received message
        """
        self.queue.put(dat)

    def proc_resp_verify_hash(self, dat):
        """Callback for message RESPONSE_VERIFY_HASH_IN_SUBSYS

        This method should be overridden if you want to process the message asynchronously.

        Args:
            dat (dict): received message
        """
        self.queue.put(dat)

    def proc_resp_domain_setup(self, dat):
        """Callback for message RESPONSE_SETUP_DOMAIN

        This method should be overridden if you want to process the message asynchronously.

        Args:
            dat (dict): received message
        """
        self.queue.put(dat)

    def proc_resp_domain_close(self, dat):
        """Callback for message RESPONSE_CLOSE_DOMAIN

        This method should be overridden if you want to process the message asynchronously.

        Args:
            dat (dict): received message
        """
        self.queue.put(dat)

    def proc_resp_set_neighbor(self, dat):
        """Callback for message RESPONSE_SET_STATIC_NODE

        This method should be overridden if you want to process the message asynchronously.

        Args:
            dat (dict): received message
        """
        self.queue.put(dat)

    def proc_resp_get_config(self, dat):
        """Callback for message RESPONSE_GET_CONFIG

        This method should be overridden if you want to process the message asynchronously.

        Args:
            dat (dict): received message
        """
        self.queue.put(dat)

    def proc_resp_get_neighborlist(self, dat):
        """Callback for message RESPONSE_GET_NEIGHBORLIST

        List of neighbor node info (the first one is that of the connecting core) is queued rather than message itself.
        This method must not be overridden.

        Args:
            dat (dict): received message
        """
        if KeyType.neighbor_list not in dat:
            self.queue.put(None)
            return
        neighbor_list = dat[KeyType.neighbor_list]
        results = []
        count = int.from_bytes(neighbor_list[:4], 'big')
        for i in range(count):
            base = 4 + i*(32+4+16+2+1+8)
            node_id = neighbor_list[base:base+32]
            ipv4 = socket.inet_ntop(socket.AF_INET, neighbor_list[base + 32:base + 36])
            ipv6 = socket.inet_ntop(socket.AF_INET6, neighbor_list[base + 36:base + 52])
            port = socket.ntohs(int.from_bytes(neighbor_list[base + 52:base + 54], 'big'))
            domain0 = True if neighbor_list[base + 54] == 0x01 else False
            updated_at = neighbor_list[base+55:base+63]
            results.append([node_id, ipv4, ipv6, port, domain0])
        self.queue.put(results)

    def proc_resp_get_domainlist(self, dat):
        """Callback for message RESPONSE_GET_DOMAINLIST

        List of domain_ids is queued rather than message itself.
        This method should be overridden if you want to process the message asynchronously.

        Args:
            dat (dict): received message
        """
        if KeyType.domain_list not in dat:
            self.queue.put(None)
            return
        self.queue.put(_parse_one_level_list(dat[KeyType.domain_list]))

    def proc_resp_get_userlist(self, dat):
        """Callback for message RESPONSE_GET_USERS

        List of user_ids is queued rather than message itself.
        This method should be overridden if you want to process the message asynchronously.

        Args:
            dat (dict): received message
        """
        if KeyType.user_list not in dat:
            self.queue.put(None)
            return
        self.queue.put(_parse_one_level_list(dat[KeyType.user_list]))

    def proc_resp_get_forwardinglist(self, dat):
        """Callback for message RESPONSE_GET_FORWARDING_LIST

        List of user_ids in other core nodes is queued rather than message itself.
        This method should be overridden if you want to process the message asynchronously.

        Args:
            dat (dict): received message
        """
        if KeyType.forwarding_list not in dat:
            self.queue.put(None)
            return
        self.queue.put(_parse_two_level_dict(dat[KeyType.forwarding_list]))

    def proc_resp_get_notificationlist(self, dat):
        """Callback for message RESPONSE_GET_NOTIFICATION_LIST

        List of user_ids in other core nodes is queued rather than message itself.
        This method should be overridden if you want to process the message asynchronously.

        Args:
            dat (dict): received message
        """
        if KeyType.notification_list not in dat:
            self.queue.put(None)
            return
        self.queue.put(_parse_two_level_dict(dat[KeyType.notification_list]))

    def proc_resp_get_node_id(self, dat):
        """Callback for message RESPONSE_GET_NODEID

        Node_id is queued rather than message itself.
        This method should be overridden if you want to process the message asynchronously.

        Args:
            dat (dict): received message
        """
        if KeyType.node_id not in dat:
            self.queue.put(dat)
            return
        self.queue.put(dat[KeyType.node_id])

    def proc_resp_get_stats(self, dat):
        """Callback for message RESPONSE_GET_STATS

        This method should be overridden if you want to process the message asynchronously.

        Args:
            dat (dict): received message
        """
        self.queue.put(dat)

    def proc_resp_ecdh_key_exchange(self, dat):
        """Callback for message RESPONSE_ECDH_KEY_EXCHANGE

        This method must not be overridden.

        Args:
            dat (dict): received message
        """
        shared_key = message_key_types.derive_shared_key(self.client.privatekey_for_ecdh,
                                                         dat[KeyType.ecdh], dat[KeyType.random])
        message_key_types.set_cipher(shared_key, dat[KeyType.nonce], self.client.aes_key_name, dat[KeyType.hint])
        self.client.is_secure_connection = False
        self.queue.put(True)
