<commit_before>from __future__ import absolute_import

import os
import logging
from datetime import timedelta

from celery import Celery

from django.conf import settings
from dbaas import celeryconfig

from logging.handlers import SysLogHandler
from celery.log import redirect_stdouts_to_logger

from celery.signals import after_setup_task_logger, after_setup_logger

def setup_log(**args): 
    # redirect stdout and stderr to logger
    redirect_stdouts_to_logger(args['logger'])
    # logs to local syslog
    syslog = SysLogHandler(address=settings.SYSLOG_FILE, facility=logging.handlers.SysLogHandler.LOG_LOCAL3)
    # setting log level
    syslog.setLevel(args['loglevel'])
    # setting log format 
    formatter = logging.Formatter('dbaas: %(name)s %(message)s')
    syslog.setFormatter(formatter)
    # add new handler to logger
    args['logger'].addHandler(syslog)

after_setup_logger.connect(setup_log)
after_setup_task_logger.connect(setup_log)

LOG = logging.getLogger(__name__)

#set this variable to True to run celery tasks synchronously

# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'dbaas.settings')

app = Celery('dbaas')

app.config_from_object(celeryconfig)
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)<commit_msg>tag celery syslog messages<commit_after>from __future__ import absolute_import

import os
import logging
from datetime import timedelta

from celery import Celery

from django.conf import settings
from dbaas import celeryconfig

from logging.handlers import SysLogHandler
from celery.log import redirect_stdouts_to_logger

from celery.signals import after_setup_task_logger, after_setup_logger

def setup_log(**args): 
    # redirect stdout and stderr to logger
    redirect_stdouts_to_logger(args['logger'])
    # logs to local syslog
    syslog = SysLogHandler(address=settings.SYSLOG_FILE, facility=logging.handlers.SysLogHandler.LOG_LOCAL3)
    # setting log level
    syslog.setLevel(args['loglevel'])
    # setting log format 
    formatter = logging.Formatter('dbaas: #celery %(name)s %(message)s')
    syslog.setFormatter(formatter)
    # add new handler to logger
    args['logger'].addHandler(syslog)

after_setup_logger.connect(setup_log)
after_setup_task_logger.connect(setup_log)

LOG = logging.getLogger(__name__)

#set this variable to True to run celery tasks synchronously

# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'dbaas.settings')

app = Celery('dbaas')

app.config_from_object(celeryconfig)
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
{'fraction_non_alphanumeric': 0.07709665287702144, 'fraction_numerical': 0.000752162467092892, 'mean_word_length': 5.873385012919897, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '314191', 'n_tokens_mistral': 827, 'n_tokens_neox': 796, 'n_words': 207}