<commit_before># -*- coding: utf-8 -*-

# Copyright (c) 2015 Ericsson AB
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import random
from calvin.actorstore.store import ActorStore
from calvin.utilities import dynops
from calvin.utilities.requirement_matching import ReqMatch
from calvin.runtime.south.plugins.async import async
from calvin.utilities.calvinlogger import get_logger
from calvin.utilities.calvin_callback import CalvinCB
import calvin.requests.calvinresponse as response
from calvin.utilities.security import Security, security_enabled
from calvin.actor.actor import ShadowActor
from calvin.runtime.north.plugins.port import DISCONNECT
from calvin.runtime.north.calvinsys import get_calvinsys, CalvinSys
from calvin.runtime.north.calvinlib import get_calvinlib, CalvinLib


_log = get_logger(__name__)


def log_callback(reply, **kwargs):
    if reply:
        _log.info("%s: %s" % (kwargs['prefix'], reply))


class ActorManager(object):

    """docstring for ActorManager"""

    def __init__(self, node):
        super(ActorManager, self).__init__()
        self.actors = {}
        self.node = node

    def _actor_not_found(self, actor_id):
        _log.exception("Actor '{}' not found".format(actor_id))
        raise Exception("Actor '{}' not found".format(actor_id))

    def new(self, actor_type, args, state=None, prev_connections=None, connection_list=None, callback=None,
            signature=None, actor_def=None, security=None, access_decision=None, shadow_actor=False,
            port_properties=None):
        """
        Instantiate an actor of type 'actor_type'. Parameters are passed in 'args',
        'name' is an optional parameter in 'args', specifying a human readable name.
        Returns actor id on success and raises an exception if anything goes wrong.
        Optionally applies a serialized state to the actor, the supplied args are ignored and args from state
        is used instead.
        Optionally reconnecting the ports, using either
          1) an unmodified connections structure obtained by the connections command supplied as
             prev_connections or,
          2) a mangled list of tuples with (in_node_id, in_port_id, out_node_id, out_port_id) supplied as
             connection_list
        """
        _log.debug("class: %s args: %s state: %s, signature: %s" % (actor_type, args, state, signature))
        _log.analyze(self.node.id, "+", {'actor_type': actor_type, 'state': state})

        try:
            if state:
                a = self._new_from_state(actor_type, state, actor_def, security, access_decision, shadow_actor)
            else:
                a = self._new(actor_type, args, actor_def, security, access_decision, shadow_actor, port_properties)
        except Exception as e:
            _log.exception("Actor creation failed")
            raise(e)

        # Store the actor signature to enable GlobalStore lookup
        a.signature_set(signature)

        self.actors[a.id] = a

        self.node.storage.add_actor(a, self.node.id)

        if prev_connections:
            # Convert prev_connections to connection_list format
            connection_list = self._prev_connections_to_connection_list(prev_connections)

        self.node.control.log_actor_new(a.id, a.name, actor_type, isinstance(a, ShadowActor))

        if connection_list:
            # Migrated actor
            self.connect(a.id, connection_list, callback=callback)
        else:
            # Nothing to connect then we are OK
            if callback:
                callback(status=response.CalvinResponse(True), actor_id=a.id)
            else:
                return a.id

    def _new_actor(self, actor_type, class_=None, actor_id=None, security=None, access_decision=None, shadow_actor=False):
        """Return a 'bare' actor of actor_type, raises an exception on failure."""
        if security_enabled() and not access_decision:
            _log.debug("Security policy check for actor failed")
            shadow_actor = True
        if shadow_actor:
            class_ = ShadowActor
        if class_ is None:
            try:
                class_, signer = self.lookup_and_verify(actor_type, security)
            except Exception:
                class_ = ShadowActor
        try:
            # Create a 'bare' instance of the actor
            a = class_(actor_type, actor_id=actor_id, security=security)
        except Exception as e:
            _log.error("The actor %s(%s) can't be instantiated." % (actor_type, class_.__init__))
            raise(e)
        a._calvinsys = self.node.calvinsys()
        if isinstance(access_decision, tuple):
            # Authorization checks needed if access_decision is a tuple.
            a.set_authorization_checks(access_decision[1])
        return a

    def _new(self, actor_type, args, actor_def=None, security=None, access_decision=None, shadow_actor=False,
             port_properties=None):
        """Return an initialized actor in PENDING state, raises an exception on failure."""
        try:
            a = self._new_actor(actor_type, actor_def, security=security,
                                access_decision=access_decision, shadow_actor=shadow_actor)
            # Now that required APIs are attached we can call init() which may use the APIs
            human_readable_name = args.pop('name', '')
            a.name = human_readable_name
            self.node.pm.add_ports_of_actor(a)
            self.node.pm.set_script_port_property(a.id, port_properties)
            a.init(**args)
            a.setup_complete()
        except Exception as e:
            _log.exception("_new")
            raise(e)
        return a

    def new_from_migration(self, actor_type, state, prev_connections=None, callback=None):
        """Instantiate an actor of type 'actor_type' and apply the 'state' to the actor."""
        try:
            _log.analyze(self.node.id, "+", state)
            subject_attributes = state.pop('_subject_attributes', None)
            migration_info = state.pop('_migration_info', None)
            try:
                state['_managed'].remove('_subject_attributes')
                state['_managed'].remove('_migration_info')
            except:
                pass
            if security_enabled():
                security = Security(self.node)
                security.set_subject_attributes(subject_attributes)
            else:
                security = None
            actor_def, signer = self.lookup_and_verify(actor_type, security)
            requirements = actor_def.requires if hasattr(actor_def, "requires") else []
            self.check_requirements_and_sec_policy(requirements, security, state['_id'],
                                                   signer, migration_info,
                                                   CalvinCB(self.new, actor_type, None,
                                                            state, prev_connections,
                                                            callback=callback,
                                                            actor_def=actor_def,
                                                            security=security))
        except Exception:
            # Still want to create shadow actor.
            self.new(actor_type, None, state, prev_connections, callback=callback, shadow_actor=True)

    def _new_from_state(self, actor_type, state, actor_def, security,
                             access_decision=None, shadow_actor=False):
        """Return a restored actor in PENDING state, raises an exception on failure."""
        try:
            a = self._new_actor(actor_type, actor_def, actor_id=state['_id'], security=security,
                                access_decision=access_decision, shadow_actor=shadow_actor)
            if '_shadow_args' in state:
                # We were a shadow, do a full init
                args = state.pop('_shadow_args')
                state['_managed'].remove('_shadow_args')
                a.init(**args)
                # If still shadow don't call did_migrate
                did_migrate = isinstance(a, ShadowActor)
            else:
                did_migrate = True
            # Always do a set_state for the port's state
            a._set_state(state)
            self.node.pm.add_ports_of_actor(a)
            if did_migrate:
                a.did_migrate()
            a.setup_complete()
        except Exception as e:
            _log.exception("Catched new from state %s %s" % (a, dir(a)))
            raise(e)
        return a

    def destroy(self, actor_id, temporary=False):
        """ Destroy an actor, temporary should be True when migrating """
        if actor_id not in self.actors:
            self._actor_not_found(actor_id)

        # @TOOD - check order here
        self.node.metering.remove_actor_info(actor_id)
        a = self.actors[actor_id]
        a.will_end()
        port_ids = self.node.pm.remove_ports_of_actor(a)
        # @TOOD - insert callback here
        if not temporary:
            self.node.storage.delete_actor(actor_id, cb=self._destroy_log_cb)
            for port_id in port_ids:
                self.node.storage.delete_port(port_id)
            self.node.control.log_actor_destroy(a.id)
        del self.actors[actor_id]

    def _destroy_log_cb(self, key, value):
        _log.debug("DESTROY CB %s %s %s" % (key, value, self.node.id))

    def destroy_with_disconnect(self, actor_id, terminate=DISCONNECT.TERMINATE, callback=None):
        if actor_id not in self.actors:
            self._actor_not_found(actor_id)
        if terminate == DISCONNECT.EXHAUST:
            actor = self.actors[actor_id]
            actor.exhaust(CalvinCB(self._destroy_with_disconnect_exhausted, actor_id=actor_id, terminate=terminate, callback=callback))
            # Exhaust first disconnects all inports then all outports after inports exhausted
            self.node.pm.disconnect(callback=CalvinCB(self._destroy_with_disconnect_in_cb, terminate=terminate,
                                                      callback=callback),
                                    actor_id=actor_id, port_dir="in", terminate=terminate)
        else:
            self.node.pm.disconnect(callback=CalvinCB(self._destroy_with_disconnect_cb,
                                                      callback=callback),
                                    actor_id=actor_id, terminate=terminate)

    def _destroy_with_disconnect_exhausted(self, status, actor_id, terminate, callback=None):
        # FIXME handled failed exhaust when we do return anything but OK
        _log.debug("Disconnected and exhausted all inports %s %s" % (actor_id, str(status)))
        self.node.pm.disconnect(callback=CalvinCB(self._destroy_with_disconnect_cb,
                                                  callback=callback),
                                actor_id=actor_id, port_dir="out", terminate=terminate)

    def _destroy_with_disconnect_in_cb(self, status, actor_id, terminate, callback=None, **kwargs):
        # FIXME handle failed disconnect
        _log.debug("Disconnected all inports %s %s" % (actor_id, str(status)))

    def _destroy_with_disconnect_cb(self, status, actor_id, callback=None, **kwargs):
        _log.debug("Disconnected all ports %s %s" % (actor_id, str(status)))
        self.destroy(actor_id)
        if callback:
            callback(status=status)

    # DEPRECATED: Enabling of an actor is dependent on whether it's connected or not
    def enable(self, actor_id):
        if actor_id not in self.actors:
            self._actor_not_found(actor_id)

        self.actors[actor_id].enable()

    # DEPRECATED: Disabling of an actor is dependent on whether it's connected or not
    def disable(self, actor_id):
        if actor_id not in self.actors:
            _log.info("!!!FAILED to disable %s", actor_id)
            self._actor_not_found(actor_id)

        self.actors[actor_id].disable()

    def lookup_and_verify(self, actor_type, security=None):
        """Lookup and verify actor in actor store."""
        found, is_primitive, actor_def, signer = ActorStore(security=security).lookup(actor_type)
        if not found or not is_primitive:
            raise Exception("Not known actor type: %s" % actor_type)
        return (actor_def, signer)


    def check_requirements_and_sec_policy(self, requirements, security=None, actor_id=None,
                                          signer=None, decision_from_migration=None, callback=None):
        """Check requirements and security policy for actor."""
        # Check if node has the capabilities required by the actor.
        try:
            return self._check_requirements_and_sec_policy(requirements=requirements, security=security, actor_id=actor_id,
                                              signer=signer, decision_from_migration=decision_from_migration, callback=callback)
        except Exception as e:
            _log.error("I don't think this should ever happen: {}".format(e))

    def _check_requirements_and_sec_policy(self, requirements, security=None, actor_id=None,
                                          signer=None, decision_from_migration=None, callback=None):
        for req in requirements:
            if not self.node.calvinsys().has_capability(req) and not get_calvinsys().has_capability(req) and not get_calvinlib().has_capability(req):
                _log.info("Missing requirement {}".format(req))
                raise Exception("Actor requires %s" % req)
        if security_enabled():
            # Check if access is permitted for the actor by the security policy.
            # Will continue directly with callback if authorization is not enabled.
            security.check_security_policy(callback, actor_id=actor_id, requires=['runtime'] + requirements,
                                            element_type="actor",
                                            element_value=signer,
                                            decision_from_migration=decision_from_migration)
        else:
            callback()

    def update_requirements(self, actor_id, requirements, extend=False, move=False,
                            authorization_check=False, callback=None):
        """ Update requirements and trigger a potential migration """
        if actor_id not in self.actors:
            # Can only migrate actors from our node
            _log.analyze(self.node.id, "+ NO ACTOR", {'actor_id': actor_id})
            if callback:
                callback(status=response.CalvinResponse(False))
            return
        if not isinstance(requirements, (list, tuple)):
            # Requirements need to be list
            _log.analyze(self.node.id, "+ NO REQ LIST", {'actor_id': actor_id})
            if callback:
                callback(status=response.CalvinResponse(response.BAD_REQUEST))
            return<commit_msg>ActorManager: Do not catch exceptions without checking if someone else is waiting for them<commit_after># -*- coding: utf-8 -*-

# Copyright (c) 2015 Ericsson AB
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import random
from calvin.actorstore.store import ActorStore
from calvin.utilities import dynops
from calvin.utilities.requirement_matching import ReqMatch
from calvin.runtime.south.plugins.async import async
from calvin.utilities.calvinlogger import get_logger
from calvin.utilities.calvin_callback import CalvinCB
import calvin.requests.calvinresponse as response
from calvin.utilities.security import Security, security_enabled
from calvin.actor.actor import ShadowActor
from calvin.runtime.north.plugins.port import DISCONNECT
from calvin.runtime.north.calvinsys import get_calvinsys
from calvin.runtime.north.calvinlib import get_calvinlib


_log = get_logger(__name__)


def log_callback(reply, **kwargs):
    if reply:
        _log.info("%s: %s" % (kwargs['prefix'], reply))


class ActorManager(object):

    """docstring for ActorManager"""

    def __init__(self, node):
        super(ActorManager, self).__init__()
        self.actors = {}
        self.node = node

    def _actor_not_found(self, actor_id):
        _log.exception("Actor '{}' not found".format(actor_id))
        raise Exception("Actor '{}' not found".format(actor_id))

    def new(self, actor_type, args, state=None, prev_connections=None, connection_list=None, callback=None,
            signature=None, actor_def=None, security=None, access_decision=None, shadow_actor=False,
            port_properties=None):
        """
        Instantiate an actor of type 'actor_type'. Parameters are passed in 'args',
        'name' is an optional parameter in 'args', specifying a human readable name.
        Returns actor id on success and raises an exception if anything goes wrong.
        Optionally applies a serialized state to the actor, the supplied args are ignored and args from state
        is used instead.
        Optionally reconnecting the ports, using either
          1) an unmodified connections structure obtained by the connections command supplied as
             prev_connections or,
          2) a mangled list of tuples with (in_node_id, in_port_id, out_node_id, out_port_id) supplied as
             connection_list
        """
        _log.debug("class: %s args: %s state: %s, signature: %s" % (actor_type, args, state, signature))
        _log.analyze(self.node.id, "+", {'actor_type': actor_type, 'state': state})

        try:
            if state:
                a = self._new_from_state(actor_type, state, actor_def, security, access_decision, shadow_actor)
            else:
                a = self._new(actor_type, args, actor_def, security, access_decision, shadow_actor, port_properties)
        except Exception as e:
            _log.exception("Actor creation failed")
            raise(e)

        # Store the actor signature to enable GlobalStore lookup
        a.signature_set(signature)

        self.actors[a.id] = a

        self.node.storage.add_actor(a, self.node.id)

        if prev_connections:
            # Convert prev_connections to connection_list format
            connection_list = self._prev_connections_to_connection_list(prev_connections)

        self.node.control.log_actor_new(a.id, a.name, actor_type, isinstance(a, ShadowActor))

        if connection_list:
            # Migrated actor
            self.connect(a.id, connection_list, callback=callback)
        else:
            # Nothing to connect then we are OK
            if callback:
                callback(status=response.CalvinResponse(True), actor_id=a.id)
            else:
                return a.id

    def _new_actor(self, actor_type, class_=None, actor_id=None, security=None, access_decision=None, shadow_actor=False):
        """Return a 'bare' actor of actor_type, raises an exception on failure."""
        if security_enabled() and not access_decision:
            _log.debug("Security policy check for actor failed")
            shadow_actor = True
        if shadow_actor:
            class_ = ShadowActor
        if class_ is None:
            try:
                class_, signer = self.lookup_and_verify(actor_type, security)
            except Exception:
                class_ = ShadowActor
        try:
            # Create a 'bare' instance of the actor
            a = class_(actor_type, actor_id=actor_id, security=security)
        except Exception as e:
            _log.error("The actor %s(%s) can't be instantiated." % (actor_type, class_.__init__))
            raise(e)
        a._calvinsys = self.node.calvinsys()
        if isinstance(access_decision, tuple):
            # Authorization checks needed if access_decision is a tuple.
            a.set_authorization_checks(access_decision[1])
        return a

    def _new(self, actor_type, args, actor_def=None, security=None, access_decision=None, shadow_actor=False,
             port_properties=None):
        """Return an initialized actor in PENDING state, raises an exception on failure."""
        try:
            a = self._new_actor(actor_type, actor_def, security=security,
                                access_decision=access_decision, shadow_actor=shadow_actor)
            # Now that required APIs are attached we can call init() which may use the APIs
            human_readable_name = args.pop('name', '')
            a.name = human_readable_name
            self.node.pm.add_ports_of_actor(a)
            self.node.pm.set_script_port_property(a.id, port_properties)
            a.init(**args)
            a.setup_complete()
        except Exception as e:
            _log.exception("_new")
            raise(e)
        return a

    def new_from_migration(self, actor_type, state, prev_connections=None, callback=None):
        """Instantiate an actor of type 'actor_type' and apply the 'state' to the actor."""
        try:
            _log.analyze(self.node.id, "+", state)
            subject_attributes = state.pop('_subject_attributes', None)
            migration_info = state.pop('_migration_info', None)
            try:
                state['_managed'].remove('_subject_attributes')
                state['_managed'].remove('_migration_info')
            except:
                pass
            if security_enabled():
                security = Security(self.node)
                security.set_subject_attributes(subject_attributes)
            else:
                security = None
            actor_def, signer = self.lookup_and_verify(actor_type, security)
            requirements = actor_def.requires if hasattr(actor_def, "requires") else []
            self.check_requirements_and_sec_policy(requirements, security, state['_id'],
                                                   signer, migration_info,
                                                   CalvinCB(self.new, actor_type, None,
                                                            state, prev_connections,
                                                            callback=callback,
                                                            actor_def=actor_def,
                                                            security=security))
        except Exception:
            # Still want to create shadow actor.
            self.new(actor_type, None, state, prev_connections, callback=callback, shadow_actor=True)

    def _new_from_state(self, actor_type, state, actor_def, security,
                             access_decision=None, shadow_actor=False):
        """Return a restored actor in PENDING state, raises an exception on failure."""
        try:
            a = self._new_actor(actor_type, actor_def, actor_id=state['_id'], security=security,
                                access_decision=access_decision, shadow_actor=shadow_actor)
            if '_shadow_args' in state:
                # We were a shadow, do a full init
                args = state.pop('_shadow_args')
                state['_managed'].remove('_shadow_args')
                a.init(**args)
                # If still shadow don't call did_migrate
                did_migrate = isinstance(a, ShadowActor)
            else:
                did_migrate = True
            # Always do a set_state for the port's state
            a._set_state(state)
            self.node.pm.add_ports_of_actor(a)
            if did_migrate:
                a.did_migrate()
            a.setup_complete()
        except Exception as e:
            _log.exception("Catched new from state %s %s" % (a, dir(a)))
            raise(e)
        return a

    def destroy(self, actor_id, temporary=False):
        """ Destroy an actor, temporary should be True when migrating """
        if actor_id not in self.actors:
            self._actor_not_found(actor_id)

        # @TOOD - check order here
        self.node.metering.remove_actor_info(actor_id)
        a = self.actors[actor_id]
        a.will_end()
        port_ids = self.node.pm.remove_ports_of_actor(a)
        # @TOOD - insert callback here
        if not temporary:
            self.node.storage.delete_actor(actor_id, cb=self._destroy_log_cb)
            for port_id in port_ids:
                self.node.storage.delete_port(port_id)
            self.node.control.log_actor_destroy(a.id)
        del self.actors[actor_id]

    def _destroy_log_cb(self, key, value):
        _log.debug("DESTROY CB %s %s %s" % (key, value, self.node.id))

    def destroy_with_disconnect(self, actor_id, terminate=DISCONNECT.TERMINATE, callback=None):
        if actor_id not in self.actors:
            self._actor_not_found(actor_id)
        if terminate == DISCONNECT.EXHAUST:
            actor = self.actors[actor_id]
            actor.exhaust(CalvinCB(self._destroy_with_disconnect_exhausted, actor_id=actor_id, terminate=terminate, callback=callback))
            # Exhaust first disconnects all inports then all outports after inports exhausted
            self.node.pm.disconnect(callback=CalvinCB(self._destroy_with_disconnect_in_cb, terminate=terminate,
                                                      callback=callback),
                                    actor_id=actor_id, port_dir="in", terminate=terminate)
        else:
            self.node.pm.disconnect(callback=CalvinCB(self._destroy_with_disconnect_cb,
                                                      callback=callback),
                                    actor_id=actor_id, terminate=terminate)

    def _destroy_with_disconnect_exhausted(self, status, actor_id, terminate, callback=None):
        # FIXME handled failed exhaust when we do return anything but OK
        _log.debug("Disconnected and exhausted all inports %s %s" % (actor_id, str(status)))
        self.node.pm.disconnect(callback=CalvinCB(self._destroy_with_disconnect_cb,
                                                  callback=callback),
                                actor_id=actor_id, port_dir="out", terminate=terminate)

    def _destroy_with_disconnect_in_cb(self, status, actor_id, terminate, callback=None, **kwargs):
        # FIXME handle failed disconnect
        _log.debug("Disconnected all inports %s %s" % (actor_id, str(status)))

    def _destroy_with_disconnect_cb(self, status, actor_id, callback=None, **kwargs):
        _log.debug("Disconnected all ports %s %s" % (actor_id, str(status)))
        self.destroy(actor_id)
        if callback:
            callback(status=status)

    # DEPRECATED: Enabling of an actor is dependent on whether it's connected or not
    def enable(self, actor_id):
        if actor_id not in self.actors:
            self._actor_not_found(actor_id)

        self.actors[actor_id].enable()

    # DEPRECATED: Disabling of an actor is dependent on whether it's connected or not
    def disable(self, actor_id):
        if actor_id not in self.actors:
            _log.info("!!!FAILED to disable %s", actor_id)
            self._actor_not_found(actor_id)

        self.actors[actor_id].disable()

    def lookup_and_verify(self, actor_type, security=None):
        """Lookup and verify actor in actor store."""
        found, is_primitive, actor_def, signer = ActorStore(security=security).lookup(actor_type)
        if not found or not is_primitive:
            raise Exception("Not known actor type: %s" % actor_type)
        return (actor_def, signer)

    def check_requirements_and_sec_policy(self, requirements, security=None, actor_id=None,
                                          signer=None, decision_from_migration=None, callback=None):
        for req in requirements:
            if not self.node.calvinsys().has_capability(req) and not get_calvinsys().has_capability(req) and not get_calvinlib().has_capability(req):
                raise Exception("Actor requires %s" % req)
        if security_enabled():
            # Check if access is permitted for the actor by the security policy.
            # Will continue directly with callback if authorization is not enabled.
            security.check_security_policy(callback, actor_id=actor_id, requires=['runtime'] + requirements,
                                            element_type="actor",
                                            element_value=signer,
                                            decision_from_migration=decision_from_migration)
        else:
            callback()

    def update_requirements(self, actor_id, requirements, extend=False, move=False,
                            authorization_check=False, callback=None):
        """ Update requirements and trigger a potential migration """
        if actor_id not in self.actors:
            # Can only migrate actors from our node
            _log.analyze(self.node.id, "+ NO ACTOR", {'actor_id': actor_id})
            if callback:
                callback(status=response.CalvinResponse(False))
            return
        if not isinstance(requirements, (list, tuple)):
            # Requirements need to be list
            _log.analyze(self.node.id, "+ NO REQ LIST", {'actor_id': actor_id})
            if callback:
                callback(status=response.CalvinResponse(response.BAD_REQUEST))
            return
{'fraction_non_alphanumeric': 0.08338320367045682, 'fraction_numerical': 0.0007979253939756633, 'mean_word_length': 2.2630722499457585, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 60, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1195329', 'n_tokens_mistral': 8091, 'n_tokens_neox': 7809, 'n_words': 2152}