<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Executes new jobs on behalf of the Django application."""

from uuid import uuid4
from twisted.internet.defer import inlineCallbacks, returnValue
from hiispider.components import *
from hiispider.metacomponents import *
import logging


LOGGER = logging.getLogger(__name__)


class Interface(Component):

    allow_clients = False
    requires = [MySQL, PageGetter, Cassandra, Logger]

    def __init__(self, server, config, server_mode, **kwargs):
        super(Interface, self).__init__(server, server_mode)
        self.mysql = self.server.mysql # For legacy plugins.

    def initialize(self):
        exposed = [x for x in self.server.functions.items() if x[1]["interval"] > 0]
        for f in self.server.functions.values():
            f["pass_kwargs_to_callback"] = True
        for function_name, func in exposed:
            self.server.add_callback(function_name, self._execute_callback)
            self.server.add_errback(function_name, self._execute_errback)
            LOGGER.debug("Added %s callback and errback." % function_name)

        # disable fast cache on the interface server
        self.server.config['enable_fast_cache'] = False

    @inlineCallbacks
    def _execute_callback(self, data, kwargs):
        uuid = uuid4().hex
        # FIXME: what should we do if there's no site_user_id?
        user_id = kwargs.get('site_user_id', '')
        if not user_id:
            raise Exception("site_user_id is a required keyword argument.")
        yield self.server.cassandra.setData(user_id, data, uuid)
        returnValue({uuid:data})

    def _execute_errback(self, error):
        return error
<commit_msg>do not raise an error if user_id is absent;  instead, just do not save to cassa<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Executes new jobs on behalf of the Django application."""

from uuid import uuid4
from twisted.internet.defer import inlineCallbacks, returnValue
from hiispider.components import *
from hiispider.metacomponents import *
import logging


LOGGER = logging.getLogger(__name__)


class Interface(Component):

    allow_clients = False
    requires = [MySQL, PageGetter, Cassandra, Logger]

    def __init__(self, server, config, server_mode, **kwargs):
        super(Interface, self).__init__(server, server_mode)
        self.mysql = self.server.mysql # For legacy plugins.

    def initialize(self):
        exposed = [x for x in self.server.functions.items() if x[1]["interval"] > 0]
        for f in self.server.functions.values():
            f["pass_kwargs_to_callback"] = True
        for function_name, func in exposed:
            self.server.add_callback(function_name, self._execute_callback)
            self.server.add_errback(function_name, self._execute_errback)
            LOGGER.debug("Added %s callback and errback." % function_name)

        # disable fast cache on the interface server
        self.server.config['enable_fast_cache'] = False

    @inlineCallbacks
    def _execute_callback(self, data, kwargs):
        uuid = uuid4().hex
        # FIXME: what should we do if there's no site_user_id?
        user_id = kwargs.get('site_user_id', '')
        if user_id:
            yield self.server.cassandra.setData(user_id, data, uuid)
        returnValue({uuid:data})

    def _execute_errback(self, error):
        return error

{'fraction_non_alphanumeric': 0.09982332155477032, 'fraction_numerical': 0.002944640753828033, 'mean_word_length': 3.173218673218673, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2349068', 'n_tokens_mistral': 986, 'n_tokens_neox': 980, 'n_words': 275}