<commit_before># coding: utf-8
from __future__ import unicode_literals
from __future__ import absolute_import

import arrow

# django
from django.conf import settings
from django.utils.log import getLogger

# trigger happy
from django_th.services import default_provider
from django_th.models import TriggerService

logger = getLogger('django_th.trigger_happy')

default_provider.load_services()


def update_trigger(service):
    """
        update the date when occurs the trigger
        :param service: service object to update
    """
    now = arrow.utcnow().to(settings.TIME_ZONE).format(
        'YYYY-MM-DD HH:mm:ssZZ')
    TriggerService.objects.filter(id=service.id).update(date_triggered=now)


def log_update(service, to_update, status, count):
    """
        lets log everything at the end
        :param service: service object
        :param to_update: boolean to check if we have to update
        :param status: is everything worked fine ?
        :param count: number of data to update
        :type service: service object
        :type to_update: boolean
        :type status: boolean
        :type count: interger
    """
    if to_update:
        if status:
            logger.info("{} - {} new data".format(service, count))
        else:
            logger.warn("{} AN ERROR OCCURS ".format(service))
    else:
        logger.debug("{} nothing new ".format(service))


def publishing(service):
    """
        the purpose of this tasks is to get the data from the cache
        then publish them
        :param service: service object where we will publish
        :type service: object
    """
    # flag to know if we have to update
    to_update = False
    # flag to get the status of a service
    status = False
    # provider - the service that offer data
    # check if the service has already been triggered
    # if date_triggered is None, then it's the first run
    if service.date_triggered is None:
        logger.debug("first run {}".format(service))
        to_update = True
        status = True
    # run run run
    service_provider = default_provider.get_service(
        str(service.provider.name.name))

    # 1) get the data from the provider service
    module_name = 'th_' + service.provider.name.name.split('Service')[1].lower()
    kw = {'trigger_id': str(service.id), 'cache_stack': module_name}
    data = getattr(service_provider, 'process_data')(**kw)
    count_new_data = len(data) if data else 0
    if count_new_data > 0:
        # consumer - the service which uses the data
        service_consumer = default_provider.get_service(
                    str(service.consumer.name.name))
        kwargs = {'user': service.user}
        getattr(service_consumer, '__init__')(service.consumer.token, **kwargs)
        consumer = getattr(service_consumer, 'save_data')

        # 2) for each one
        for d in data:
            d['userservice_id'] = service.consumer.id
            # the consumer will save the data and return if success or not
            status = consumer(service.id, **d)

            to_update = True
        # let's log
    log_update(service, to_update, status, count_new_data)
    # let's update
    if to_update and status:
        update_trigger(service)
<commit_msg>fix #145 - stop triggers that use a disabled service<commit_after># coding: utf-8
from __future__ import unicode_literals
from __future__ import absolute_import

import arrow

# django
from django.conf import settings
from django.utils.log import getLogger

# trigger happy
from django_th.services import default_provider
from django_th.models import TriggerService

logger = getLogger('django_th.trigger_happy')

default_provider.load_services()


def update_trigger(service):
    """
        update the date when occurs the trigger
        :param service: service object to update
    """
    now = arrow.utcnow().to(settings.TIME_ZONE).format(
        'YYYY-MM-DD HH:mm:ssZZ')
    TriggerService.objects.filter(id=service.id).update(date_triggered=now)


def log_update(service, to_update, status, count):
    """
        lets log everything at the end
        :param service: service object
        :param to_update: boolean to check if we have to update
        :param status: is everything worked fine ?
        :param count: number of data to update
        :type service: service object
        :type to_update: boolean
        :type status: boolean
        :type count: interger
    """
    if to_update:
        if status:
            logger.info("{} - {} new data".format(service, count))
        else:
            logger.warn("{} AN ERROR OCCURS ".format(service))
    else:
        logger.debug("{} nothing new ".format(service))


def publishing(service):
    """
        the purpose of this tasks is to get the data from the cache
        then publish them
        :param service: service object where we will publish
        :type service: object
    """
    if service.provider.name.status and service.consumer.name.status:
        # flag to know if we have to update
        to_update = False
        # flag to get the status of a service
        status = False
        # provider - the service that offer data
        # check if the service has already been triggered
        # if date_triggered is None, then it's the first run
        if service.date_triggered is None:
            logger.debug("first run {}".format(service))
            to_update = True
            status = True
        # run run run
        service_provider = default_provider.get_service(
            str(service.provider.name.name))

        # 1) get the data from the provider service
        module_name = 'th_' + \
                      service.provider.name.name.split('Service')[1].lower()
        kw = {'trigger_id': str(service.id), 'cache_stack': module_name}
        data = getattr(service_provider, 'process_data')(**kw)
        count_new_data = len(data) if data else 0
        if count_new_data > 0:
            # consumer - the service which uses the data
            service_consumer = default_provider.get_service(
                        str(service.consumer.name.name))
            kwargs = {'user': service.user}
            getattr(service_consumer, '__init__')(service.consumer.token,
                                                  **kwargs)
            consumer = getattr(service_consumer, 'save_data')

            # 2) for each one
            for d in data:
                d['userservice_id'] = service.consumer.id
                # the consumer will save the data and return if success or not
                status = consumer(service.id, **d)

                to_update = True
            # let's log
        log_update(service, to_update, status, count_new_data)
        # let's update
        if to_update and status:
            update_trigger(service)

{'fraction_non_alphanumeric': 0.08581051398447796, 'fraction_numerical': 0.002196514863083907, 'mean_word_length': 2.426994480682388, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7445715', 'n_tokens_mistral': 1875, 'n_tokens_neox': 1817, 'n_words': 614}