<commit_before># Copyright 2015, <NAME> <<EMAIL>>
#
# This file is part of Azrael (https://github.com/olitheolix/azrael)
#
# Azrael is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# Azrael is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Azrael. If not, see <http://www.gnu.org/licenses/>.

"""
Primary gateway into Azrael.

``Clerk`` is a stateless class that arbitrates between clients on the network
and the various Azrael services. It also acts as a prudent sanity checker that
will verify all incoming data. In turn, the other Azrael services mostly forgoe
sanity checks for performance reasons.

``Clerk`` is meant to run as an independent process. Its stateless design
implies that there can be many ``Clerk`` processes (eg behind an NginX
server) running simultaneously. Furthermore, not all ``Clerk`` processes need
to run on the same machine since Azrael's collective memory is contained
entirely within a database -- ``Clerk`` has no state.

``Clerk`` uses ZeroMQ sockets for the communication with clients. ZeroMQ has
bindings for virtually every language except JavaScript, which is why
``Clacks`` implements a web server bridge. In both cases the data exchange is
plain JSON; no Python specific data types are used to keep the interface
language agnostic.

"""
import io
import os
import sys
import zmq
import copy
import json
import logging
import traceback
import subprocess

import numpy as np

import azrael.igor
import azrael.database
import azrael.util as util
import azrael.types as types
import azrael.config as config
import azrael.leo_api as leoAPI
import azrael.dibbler as dibbler
import azrael.database as database
import azrael.protocol as protocol

from IPython import embed as ipshell
from azrael.types import typecheck, RetVal, Template, CollShapeMeta, CollShapeEmpty
from azrael.types import FragMeta, _FragMeta


class Clerk(config.AzraelProcess):
    """
    Administrate the simulation.

    There can only be one instance of Clerk per machine because it binds 0MQ
    sockets. These sockets are the only way to interact with Clerk.

    Clerk will always inspect and sanity check the received data to reject
    invalid client requests.

    Clerk relies on the `protocol` module to convert the ZeroMQ byte strings
    to meaningful quantities.

    :raises: None
    """
    @typecheck
    def __init__(self):
        super().__init__()

        # Dibbler is the interface to the geometry database.
        self.dibbler = dibbler.Dibbler()

        # Dibbler is the interface to the constraints database.
        self.igor = azrael.igor.Igor()

        # Specify the decoding-processing-encoding triplet functions for
        # (almost) every command supported by Clerk. The only exceptions are
        # administrative commands (eg. ping). This dictionary will be used
        # in the digest loop.
        self.codec = {
            'ping_clerk': (
                protocol.ToClerk_Ping_Decode,
                self.pingClerk,
                protocol.FromClerk_Ping_Encode),
            'add_templates': (
                protocol.ToClerk_AddTemplates_Decode,
                self.addTemplates,
                protocol.FromClerk_AddTemplates_Encode),
            'get_templates': (
                protocol.ToClerk_GetTemplates_Decode,
                self.getTemplates,
                protocol.FromClerk_GetTemplates_Encode),
            'get_template_id': (
                protocol.ToClerk_GetTemplateID_Decode,
                self.getTemplateID,
                protocol.FromClerk_GetTemplateID_Encode),
            'spawn': (
                protocol.ToClerk_Spawn_Decode,
                self.spawn,
                protocol.FromClerk_Spawn_Encode),
            'remove': (
                protocol.ToClerk_Remove_Decode,
                self.removeObject,
                protocol.FromClerk_Remove_Encode),
            'get_all_objids': (
                protocol.ToClerk_GetAllObjectIDs_Decode,
                self.getAllObjectIDs,
                protocol.FromClerk_GetAllObjectIDs_Encode),
            'get_object_states': (
                protocol.ToClerk_GetObjectStates_Decode,
                self.getObjectStates,
                protocol.FromClerk_GetObjectStates_Encode),
            'get_rigid_bodies': (
                protocol.ToClerk_GetRigidBodies_Decode,
                self.getRigidBodies,
                protocol.FromClerk_GetRigidBodies_Encode),
            'set_rigid_bodies': (
                protocol.ToClerk_SetRigidBodies_Decode,
                self.setRigidBodies,
                protocol.FromClerk_SetRigidBodies_Encode),
            'set_fragments': (
                protocol.ToClerk_SetFragments_Decode,
                self.setFragments,
                protocol.FromClerk_SetFragments_Encode),
            'get_fragments': (
                protocol.ToClerk_GetFragments_Decode,
                self.getFragments,
                protocol.FromClerk_GetFragments_Encode),
            'set_force': (
                protocol.ToClerk_SetForce_Decode,
                self.setForce,
                protocol.FromClerk_SetForce_Encode),
            'control_parts': (
                protocol.ToClerk_ControlParts_Decode,
                self.controlParts,
                protocol.FromClerk_ControlParts_Encode),
            'add_constraints': (
                protocol.ToClerk_AddConstraints_Decode,
                self.addConstraints,
                protocol.FromClerk_AddConstraints_Encode),
            'get_constraints': (
                protocol.ToClerk_GetConstraints_Decode,
                self.getConstraints,
                protocol.FromClerk_GetConstraints_Encode),
            'delete_constraints': (
                protocol.ToClerk_DeleteConstraints_Decode,
                self.deleteConstraints,
                protocol.FromClerk_DeleteConstraints_Encode),
        }

    def runCommand(self, fun_decode, fun_process, fun_encode):
        """
        Wrapper function to process a client request.

        This wrapper will use ``fun_decode`` to convert the ZeroMQ byte stream
        into  Python data types, then pass these data types to ``fun_process``
        for processing, and encodes the return values (also Python types) with
        ``fun_encode`` so that they can be sent back via ZeroMQ.

        The main purpose of this function is to establish a clear
        decode-process-encode work flow that all commands adhere to.

        To add a new command it is therefore necessary to specify the necessary
        {en,de}coding functions (usually in the `protocol` module) and add a
        method to this class for the actual processing.

        To make this work it is mandatory that all processing methods return an
        `(ok, some_tuple)` tuple. The Boolean ``ok`` flag indicates the
        success, and the ``some_tuple`` can contain arbitrarily many return
        types. Note however that ``some_tuple`` *must* always be a tuple, even
        if it contains only one entry, or no entry at all.

        :param callable fun_decode: converter function (bytes --> Python)
        :param callable fun_process: processes the client request.
        :param callable fun_encode: converter function (Python --> bytes)
        :return: **None**
        """
        # Decode the binary data.
        out = fun_decode(self.payload)

        # Decoding was successful. Pass all returned parameters directly
        # to the respective Clerk method.
        ret = fun_process(*out)

        if ret.ok:
            # Convert the output to a JSON string.
            enc = fun_encode(ret.data)
            ret = ret._replace(data=enc)
            self.returnToClient(self.last_addr, ret)
        else:
            # The processing method encountered an error.
            self.returnToClient(self.last_addr, ret)

    def run(self):
        """
        Initialise ZeroMQ and wait for client requests.

        This method will not return.

        :raises: None
        """
        # Call `run` method of `AzraelProcess` base class.
        super().run()

        # Initialise ZeroMQ and create the command socket. All client request
        # will come through this socket.
        addr = 'tcp://{}:{}'.format(config.addr_clerk, config.port_clerk)
        self.logit.info('Attempt to bind <{}>'.format(addr))
        ctx = zmq.Context()
        self.sock_cmd = ctx.socket(zmq.ROUTER)
        self.sock_cmd.bind(addr)
        poller = zmq.Poller()
        poller.register(self.sock_cmd, zmq.POLLIN)
        self.logit.info('Listening on <{}>'.format(addr))
        del addr

        # Digest loop.
        while True:
            # Wait for socket activity.
            sock = dict(poller.poll())
            if self.sock_cmd not in sock:
                continue

            # Read from ROUTER socket and perform sanity checks.
            data = self.sock_cmd.recv_multipart()
            assert len(data) == 3
            self.last_addr, empty, msg = data[0], data[1], data[2]
            if empty != b'':
                self.logit.error('Expected empty frame')
                continue
            del data, empty

            # The payload must be a valid JSON string.
            try:
                msg = json.loads(msg.decode('utf8'))
            except (ValueError, TypeError) as err:
                ret = RetVal(False, 'JSON decoding error in Clerk', None)
                self.returnToClient(self.last_addr, ret)
                del ret
                continue

            # Sanity check: every message must contain at least a command word.
            if not (('cmd' in msg) and ('data' in msg)):
                ret = RetVal(False, 'Invalid command format', None)
                self.returnToClient(self.last_addr, ret)
                del ret
                continue

            # Extract the command word and payload.
            cmd, self.payload = msg['cmd'], msg['data']

            # The command word determines the action...
            if cmd in self.codec:
                # Look up the decode-process-encode functions for the current
                # command word. The 'decode' part will interpret the JSON
                # string we just received, the 'process' part is a handle to a
                # method in this very Clerk instance, and 'encode' will convert
                # the values to a valid JSON string that will be returned to
                # the Client.
                dec, proc, enc = self.codec[cmd]

                # Run the Clerk function. The try/except is to intercept any
                # errors and prevent Clerk from dying.
                try:
                    self.runCommand(dec, proc, enc)
                except Exception:
                    # Basic error message.
                    msg = 'Client data for <{}> raised an error in Clerk'
                    msg = msg.format(cmd)

                    # Get the Python stack trace as string (this will be added
                    # to the log for reference).
                    buf = io.StringIO()
                    traceback.print_exc(file=buf)
                    buf.seek(0)
                    msg_st = [' -> ' + _ for _ in buf.readlines()]
                    msg_st = msg + '\n' + ''.join(msg_st)

                    # Log the error message with stack trace. However, only
                    # the basic error message to the client.
                    self.logit.error(msg_st)
                    ret = RetVal(False, msg, None),
                    self.returnToClient(self.last_addr, ret, addToLog=False)
                    del msg, buf, msg_st, ret
            else:
                ret = RetVal(False, 'Invalid command <{}>'.format(cmd), None)
                # Unknown command word.
                self.returnToClient(self.last_addr, ret)

    @typecheck
    def returnToClient(self, addr, ret: RetVal, addToLog: bool=True):
        """
        Send ``ret`` to back to Client via ZeroMQ.

        This is a convenience method to enhance readability.

        :param addr: ZeroMQ address as returned by the ROUTER socket.
        :param bool addToLog: logs a 'Warning' with ``msg`` if *True*
        :param str msg: the error message to send back.
        :return: None
        """
        if addToLog:
            self.logit.warning(ret.msg)

        # Convert the message to JSON.
        try:
            ret = json.dumps(ret._asdict())
        except (ValueError, TypeError) as err:
            msg = 'Could not convert Clerk return value to JSON'
            ret = json.dumps(RetVal(False, msg, None)._asdict())

        # Send the message via ZeroMQ.
        self.sock_cmd.send_multipart([addr, b'', ret.encode('utf8')])
        
    def pingClerk(self):
        """
        Return a 'pong'.

        :return: the string 'pong' to acknowledge the ping.
        :rtype: str
        :raises: None
        """
        return RetVal(True, None, 'pong clerk')

    # ----------------------------------------------------------------------
    # These methods service Client requests.
    # ----------------------------------------------------------------------
    @typecheck
    def addTemplates(self, templates: list):
        """
        Add all ``templates`` to Azrael so that they can be spawned.
<commit_msg>'Clerk.sendToClient' does not log 'None' errors.<commit_after># Copyright 2015, <NAME> <<EMAIL>>
#
# This file is part of Azrael (https://github.com/olitheolix/azrael)
#
# Azrael is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# Azrael is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Azrael. If not, see <http://www.gnu.org/licenses/>.

"""
Primary gateway into Azrael.

``Clerk`` is a stateless class that arbitrates between clients on the network
and the various Azrael services. It also acts as a prudent sanity checker that
will verify all incoming data. In turn, the other Azrael services mostly forgoe
sanity checks for performance reasons.

``Clerk`` is meant to run as an independent process. Its stateless design
implies that there can be many ``Clerk`` processes (eg behind an NginX
server) running simultaneously. Furthermore, not all ``Clerk`` processes need
to run on the same machine since Azrael's collective memory is contained
entirely within a database -- ``Clerk`` has no state.

``Clerk`` uses ZeroMQ sockets for the communication with clients. ZeroMQ has
bindings for virtually every language except JavaScript, which is why
``Clacks`` implements a web server bridge. In both cases the data exchange is
plain JSON; no Python specific data types are used to keep the interface
language agnostic.

"""
import io
import os
import sys
import zmq
import copy
import json
import logging
import traceback
import subprocess

import numpy as np

import azrael.igor
import azrael.database
import azrael.util as util
import azrael.types as types
import azrael.config as config
import azrael.leo_api as leoAPI
import azrael.dibbler as dibbler
import azrael.database as database
import azrael.protocol as protocol

from IPython import embed as ipshell
from azrael.types import typecheck, RetVal, Template, CollShapeMeta, CollShapeEmpty
from azrael.types import FragMeta, _FragMeta


class Clerk(config.AzraelProcess):
    """
    Administrate the simulation.

    There can only be one instance of Clerk per machine because it binds 0MQ
    sockets. These sockets are the only way to interact with Clerk.

    Clerk will always inspect and sanity check the received data to reject
    invalid client requests.

    Clerk relies on the `protocol` module to convert the ZeroMQ byte strings
    to meaningful quantities.

    :raises: None
    """
    @typecheck
    def __init__(self):
        super().__init__()

        # Dibbler is the interface to the geometry database.
        self.dibbler = dibbler.Dibbler()

        # Dibbler is the interface to the constraints database.
        self.igor = azrael.igor.Igor()

        # Specify the decoding-processing-encoding triplet functions for
        # (almost) every command supported by Clerk. The only exceptions are
        # administrative commands (eg. ping). This dictionary will be used
        # in the digest loop.
        self.codec = {
            'ping_clerk': (
                protocol.ToClerk_Ping_Decode,
                self.pingClerk,
                protocol.FromClerk_Ping_Encode),
            'add_templates': (
                protocol.ToClerk_AddTemplates_Decode,
                self.addTemplates,
                protocol.FromClerk_AddTemplates_Encode),
            'get_templates': (
                protocol.ToClerk_GetTemplates_Decode,
                self.getTemplates,
                protocol.FromClerk_GetTemplates_Encode),
            'get_template_id': (
                protocol.ToClerk_GetTemplateID_Decode,
                self.getTemplateID,
                protocol.FromClerk_GetTemplateID_Encode),
            'spawn': (
                protocol.ToClerk_Spawn_Decode,
                self.spawn,
                protocol.FromClerk_Spawn_Encode),
            'remove': (
                protocol.ToClerk_Remove_Decode,
                self.removeObject,
                protocol.FromClerk_Remove_Encode),
            'get_all_objids': (
                protocol.ToClerk_GetAllObjectIDs_Decode,
                self.getAllObjectIDs,
                protocol.FromClerk_GetAllObjectIDs_Encode),
            'get_object_states': (
                protocol.ToClerk_GetObjectStates_Decode,
                self.getObjectStates,
                protocol.FromClerk_GetObjectStates_Encode),
            'get_rigid_bodies': (
                protocol.ToClerk_GetRigidBodies_Decode,
                self.getRigidBodies,
                protocol.FromClerk_GetRigidBodies_Encode),
            'set_rigid_bodies': (
                protocol.ToClerk_SetRigidBodies_Decode,
                self.setRigidBodies,
                protocol.FromClerk_SetRigidBodies_Encode),
            'set_fragments': (
                protocol.ToClerk_SetFragments_Decode,
                self.setFragments,
                protocol.FromClerk_SetFragments_Encode),
            'get_fragments': (
                protocol.ToClerk_GetFragments_Decode,
                self.getFragments,
                protocol.FromClerk_GetFragments_Encode),
            'set_force': (
                protocol.ToClerk_SetForce_Decode,
                self.setForce,
                protocol.FromClerk_SetForce_Encode),
            'control_parts': (
                protocol.ToClerk_ControlParts_Decode,
                self.controlParts,
                protocol.FromClerk_ControlParts_Encode),
            'add_constraints': (
                protocol.ToClerk_AddConstraints_Decode,
                self.addConstraints,
                protocol.FromClerk_AddConstraints_Encode),
            'get_constraints': (
                protocol.ToClerk_GetConstraints_Decode,
                self.getConstraints,
                protocol.FromClerk_GetConstraints_Encode),
            'delete_constraints': (
                protocol.ToClerk_DeleteConstraints_Decode,
                self.deleteConstraints,
                protocol.FromClerk_DeleteConstraints_Encode),
        }

    def runCommand(self, fun_decode, fun_process, fun_encode):
        """
        Wrapper function to process a client request.

        This wrapper will use ``fun_decode`` to convert the ZeroMQ byte stream
        into  Python data types, then pass these data types to ``fun_process``
        for processing, and encodes the return values (also Python types) with
        ``fun_encode`` so that they can be sent back via ZeroMQ.

        The main purpose of this function is to establish a clear
        decode-process-encode work flow that all commands adhere to.

        To add a new command it is therefore necessary to specify the necessary
        {en,de}coding functions (usually in the `protocol` module) and add a
        method to this class for the actual processing.

        To make this work it is mandatory that all processing methods return an
        `(ok, some_tuple)` tuple. The Boolean ``ok`` flag indicates the
        success, and the ``some_tuple`` can contain arbitrarily many return
        types. Note however that ``some_tuple`` *must* always be a tuple, even
        if it contains only one entry, or no entry at all.

        :param callable fun_decode: converter function (bytes --> Python)
        :param callable fun_process: processes the client request.
        :param callable fun_encode: converter function (Python --> bytes)
        :return: **None**
        """
        # Decode the binary data.
        out = fun_decode(self.payload)

        # Decoding was successful. Pass all returned parameters directly
        # to the respective Clerk method.
        ret = fun_process(*out)

        if ret.ok:
            # Convert the output to a JSON string.
            enc = fun_encode(ret.data)
            ret = ret._replace(data=enc)
            self.returnToClient(self.last_addr, ret)
        else:
            # The processing method encountered an error.
            self.returnToClient(self.last_addr, ret)

    def run(self):
        """
        Initialise ZeroMQ and wait for client requests.

        This method will not return.

        :raises: None
        """
        # Call `run` method of `AzraelProcess` base class.
        super().run()

        # Initialise ZeroMQ and create the command socket. All client request
        # will come through this socket.
        addr = 'tcp://{}:{}'.format(config.addr_clerk, config.port_clerk)
        self.logit.info('Attempt to bind <{}>'.format(addr))
        ctx = zmq.Context()
        self.sock_cmd = ctx.socket(zmq.ROUTER)
        self.sock_cmd.bind(addr)
        poller = zmq.Poller()
        poller.register(self.sock_cmd, zmq.POLLIN)
        self.logit.info('Listening on <{}>'.format(addr))
        del addr

        # Digest loop.
        while True:
            # Wait for socket activity.
            sock = dict(poller.poll())
            if self.sock_cmd not in sock:
                continue

            # Read from ROUTER socket and perform sanity checks.
            data = self.sock_cmd.recv_multipart()
            assert len(data) == 3
            self.last_addr, empty, msg = data[0], data[1], data[2]
            if empty != b'':
                self.logit.error('Expected empty frame')
                continue
            del data, empty

            # The payload must be a valid JSON string.
            try:
                msg = json.loads(msg.decode('utf8'))
            except (ValueError, TypeError) as err:
                ret = RetVal(False, 'JSON decoding error in Clerk', None)
                self.returnToClient(self.last_addr, ret)
                del ret
                continue

            # Sanity check: every message must contain at least a command word.
            if not (('cmd' in msg) and ('data' in msg)):
                ret = RetVal(False, 'Invalid command format', None)
                self.returnToClient(self.last_addr, ret)
                del ret
                continue

            # Extract the command word and payload.
            cmd, self.payload = msg['cmd'], msg['data']

            # The command word determines the action...
            if cmd in self.codec:
                # Look up the decode-process-encode functions for the current
                # command word. The 'decode' part will interpret the JSON
                # string we just received, the 'process' part is a handle to a
                # method in this very Clerk instance, and 'encode' will convert
                # the values to a valid JSON string that will be returned to
                # the Client.
                dec, proc, enc = self.codec[cmd]

                # Run the Clerk function. The try/except is to intercept any
                # errors and prevent Clerk from dying.
                try:
                    self.runCommand(dec, proc, enc)
                except Exception:
                    # Basic error message.
                    msg = 'Client data for <{}> raised an error in Clerk'
                    msg = msg.format(cmd)

                    # Get the Python stack trace as string (this will be added
                    # to the log for reference).
                    buf = io.StringIO()
                    traceback.print_exc(file=buf)
                    buf.seek(0)
                    msg_st = [' -> ' + _ for _ in buf.readlines()]
                    msg_st = msg + '\n' + ''.join(msg_st)

                    # Log the error message with stack trace. However, only
                    # the basic error message to the client.
                    self.logit.error(msg_st)
                    ret = RetVal(False, msg, None),
                    self.returnToClient(self.last_addr, ret, addToLog=False)
                    del msg, buf, msg_st, ret
            else:
                ret = RetVal(False, 'Invalid command <{}>'.format(cmd), None)
                # Unknown command word.
                self.returnToClient(self.last_addr, ret)

    @typecheck
    def returnToClient(self, addr, ret: RetVal, addToLog: bool=True):
        """
        Send ``ret`` to back to Client via ZeroMQ.

        This is a convenience method to enhance readability.

        :param addr: ZeroMQ address as returned by the ROUTER socket.
        :param bool addToLog: logs a 'Warning' with ``msg`` if *True*
        :param str msg: the error message to send back.
        :return: None
        """
        if addToLog and ret.msg is not None:
            self.logit.warning(ret.msg)

        # Convert the message to JSON.
        try:
            ret = json.dumps(ret._asdict())
        except (ValueError, TypeError) as err:
            msg = 'Could not convert Clerk return value to JSON'
            ret = json.dumps(RetVal(False, msg, None)._asdict())

        # Send the message via ZeroMQ.
        self.sock_cmd.send_multipart([addr, b'', ret.encode('utf8')])
        
    def pingClerk(self):
        """
        Return a 'pong'.

        :return: the string 'pong' to acknowledge the ping.
        :rtype: str
        :raises: None
        """
        return RetVal(True, None, 'pong clerk')

    # ----------------------------------------------------------------------
    # These methods service Client requests.
    # ----------------------------------------------------------------------
    @typecheck
    def addTemplates(self, templates: list):
        """
        Add all ``templates`` to Azrael so that they can be spawned.

{'fraction_non_alphanumeric': 0.08232559828594255, 'fraction_numerical': 0.0009441841885463195, 'mean_word_length': 1.9502892650524963, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 25, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5782331', 'n_tokens_mistral': 7317, 'n_tokens_neox': 6918, 'n_words': 2462}