<commit_before>import synapse.common as s_common
import synapse.telepath as s_telepath

import synapse.lib.tufo as s_tufo
import synapse.lib.storm as s_storm
import synapse.lib.service as s_service

from synapse.eventbus import EventBus
from synapse.lib.config import Configable

deftag = 'class.synapse.cores.common.Cortex'

class Runtime(s_storm.Runtime,EventBus):
    '''
    A STORM runtime capable of using a swarm cluster
    '''
    def __init__(self, svcbus, **opts):
        EventBus.__init__(self)
        s_storm.Runtime.__init__(self)

        self.addConfDef('svcbus:deftag', asloc='deftag', type='syn:tag', defval=deftag, doc='Default tag for cores')
        self.addConfDef('svcbus:timeout', asloc='svctime', type='int', doc='SvcBus Telepath Link Tufo')

        self.setConfOpts(opts)

        self.svcbus = svcbus
        self.svcprox = s_service.SvcProxy(svcbus, self.svctime)

    def _getTufosByFrom(self, by, prop, valu=None, limit=None, fromtag=None):

        if fromtag == None:
            fromtag = self.deftag

        ret = []

        limit = self.getLiftLimit(limit)

        dyntask = s_common.gentask('stormTufosBy', by, prop, valu=valu, limit=limit)

        for svcfo,retval in self.svcprox.callByTag(fromtag, dyntask, timeout=self.svctime):
            [ tufo[1].__setitem__('.from',svcfo[0]) for tufo in retval ]
            ret.extend(retval)

        return ret

    ####################################################################
    # We override the these methods from the base runtime
    # ( they are registered as operators in the base constructor )

    def _stormOperLift(self, query, oper):

        by = oper[1].get('cmp')
        prop = oper[1].get('prop')
        valu = oper[1].get('valu')
        limit = oper[1].get('limit')
        fromtag = oper[1].get('from')

        for tufo in self._getTufosByFrom(by, prop, valu, limit=limit, fromtag=fromtag):<commit_msg>added local core for type/prop normalization to swarm runtime<commit_after>import synapse.common as s_common
import synapse.cortex as s_cortex
import synapse.telepath as s_telepath

import synapse.lib.tufo as s_tufo
import synapse.lib.storm as s_storm
import synapse.lib.service as s_service

from synapse.eventbus import EventBus
from synapse.lib.config import Configable

deftag = 'class.synapse.cores.common.Cortex'

class Runtime(s_storm.Runtime,EventBus):
    '''
    A STORM runtime capable of using a swarm cluster
    '''
    def __init__(self, svcbus, **opts):
        EventBus.__init__(self)

        # a core we use for data model stuff..
        self.core = s_cortex.openurl('ram:///')
        self.onfini( self.core.fini )

        s_storm.Runtime.__init__(self)


        self.addConfDef('svcbus:deftag', asloc='deftag', type='syn:tag', defval=deftag, doc='Default tag for cores')
        self.addConfDef('svcbus:timeout', asloc='svctime', type='int', doc='SvcBus Telepath Link Tufo')

        self.setConfOpts(opts)

        self.svcbus = svcbus
        self.svcprox = s_service.SvcProxy(svcbus, self.svctime)

    def _getStormCore(self, name=None):
        return self.core

    def _getTufosByFrom(self, by, prop, valu=None, limit=None, fromtag=None):

        if fromtag == None:
            fromtag = self.deftag

        ret = []

        limit = self.getLiftLimit(limit)

        dyntask = s_common.gentask('stormTufosBy', by, prop, valu=valu, limit=limit)

        for svcfo,retval in self.svcprox.callByTag(fromtag, dyntask, timeout=self.svctime):
            [ tufo[1].__setitem__('.from',svcfo[0]) for tufo in retval ]
            ret.extend(retval)

        return ret

    ####################################################################
    # We override the these methods from the base runtime
    # ( they are registered as operators in the base constructor )

    def _stormOperLift(self, query, oper):

        by = oper[1].get('cmp')
        prop = oper[1].get('prop')
        valu = oper[1].get('valu')
        limit = oper[1].get('limit')
        fromtag = oper[1].get('from')

        for tufo in self._getTufosByFrom(by, prop, valu, limit=limit, fromtag=fromtag):
{'fraction_non_alphanumeric': 0.14469219583131362, 'fraction_numerical': 0.0033931168201648087, 'mean_word_length': 3.2026476578411405, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4977904', 'n_tokens_mistral': 1378, 'n_tokens_neox': 1338, 'n_words': 314}