<commit_before># Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved.

# Python
import codecs
from collections import OrderedDict
import ConfigParser
import cStringIO
import json
import logging
import os
import signal
import pipes
import re
import shutil
import stat
import tempfile
import thread
import time
import traceback
import urlparse
import uuid
from distutils.version import LooseVersion as Version
import yaml
try:
    import psutil
except:
    psutil = None

# Pexpect
import pexpect

# Celery
from celery import Task, task
from celery.signals import celeryd_init, worker_ready
from celery import current_app

# Django
from django.conf import settings
from django.db import transaction, DatabaseError
from django.utils.timezone import now
from django.utils.encoding import smart_str
from django.core.mail import send_mail
from django.contrib.auth.models import User
from django.utils.translation import ugettext_lazy as _
from django.core.cache import cache

# AWX
from awx.main.constants import CLOUD_PROVIDERS
from awx.main.models import * # noqa
from awx.main.models import UnifiedJob
from awx.main.queue import CallbackQueueDispatcher
from awx.main.task_engine import TaskEnhancer
from awx.main.utils import (get_ansible_version, get_ssh_version, decrypt_field, update_scm_url,
                            check_proot_installed, build_proot_temp_dir, wrap_args_with_proot,
                            get_system_task_capacity, OutputEventFilter, parse_yaml_or_json)
from awx.main.consumers import emit_channel_notification

__all__ = ['RunJob', 'RunSystemJob', 'RunProjectUpdate', 'RunInventoryUpdate',
           'RunAdHocCommand', 'handle_work_error',
           'handle_work_success', 'update_inventory_computed_fields',
           'send_notifications', 'run_administrative_checks', 'purge_old_stdout_files']

HIDDEN_PASSWORD = '**********'

OPENSSH_KEY_ERROR = u'''\
It looks like you're trying to use a private key in OpenSSH format, which \
isn't supported by the installed version of OpenSSH on this Tower instance. \
Try upgrading OpenSSH or providing your private key in an different format. \
'''

logger = logging.getLogger('awx.main.tasks')


@celeryd_init.connect
def celery_startup(conf=None, **kwargs):
    # Re-init all schedules
    # NOTE: Rework this during the Rampart work
    startup_logger = logging.getLogger('awx.main.tasks')
    startup_logger.info("Syncing Tower Schedules")
    for sch in Schedule.objects.all():
        try:
            sch.update_computed_fields()
            sch.save()
        except Exception as e:
            logger.error("Failed to rebuild schedule {}: {}".format(sch, e))


def _setup_tower_logger():
    global logger
    from django.utils.log import configure_logging
    LOGGING_DICT = settings.LOGGING
    if settings.LOG_AGGREGATOR_ENABLED:
        LOGGING_DICT['handlers']['http_receiver']['class'] = 'awx.main.utils.handlers.HTTPSHandler'
        LOGGING_DICT['handlers']['http_receiver']['async'] = False
        if 'awx' in settings.LOG_AGGREGATOR_LOGGERS:
            if 'http_receiver' not in LOGGING_DICT['loggers']['awx']['handlers']:
                LOGGING_DICT['loggers']['awx']['handlers'] += ['http_receiver']
    configure_logging(settings.LOGGING_CONFIG, LOGGING_DICT)
    logger = logging.getLogger('awx.main.tasks')


@worker_ready.connect
def task_set_logger_pre_run(*args, **kwargs):
    cache.close()
    if settings.LOG_AGGREGATOR_ENABLED:
        _setup_tower_logger()
        logger.debug('Custom Tower logger configured for worker process.')


def _uwsgi_reload():
    # http://uwsgi-docs.readthedocs.io/en/latest/MasterFIFO.html#available-commands
    logger.warn('Initiating uWSGI chain reload of server')
    TRIGGER_CHAIN_RELOAD = 'c'
    with open('/var/lib/awx/awxfifo', 'w') as awxfifo:
        awxfifo.write(TRIGGER_CHAIN_RELOAD)


def _reset_celery_logging():
    # Worker logger reloaded, now send signal to restart pool
    app = current_app._get_current_object()
    app.control.broadcast('pool_restart', arguments={'reload': True},
                          destination=['celery@{}'.format(settings.CLUSTER_HOST_ID)], reply=False)


def _clear_cache_keys(set_of_keys):<commit_msg>have tasks.py deduce location of uwsgi fifo<commit_after># Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved.

# Python
import codecs
from collections import OrderedDict
import ConfigParser
import cStringIO
import json
import logging
import os
import signal
import pipes
import re
import shutil
import stat
import tempfile
import thread
import time
import traceback
import urlparse
import uuid
from distutils.version import LooseVersion as Version
import yaml
try:
    import psutil
except:
    psutil = None

# Pexpect
import pexpect

# Celery
from celery import Task, task
from celery.signals import celeryd_init, worker_ready
from celery import current_app

# Django
from django.conf import settings
from django.db import transaction, DatabaseError
from django.utils.timezone import now
from django.utils.encoding import smart_str
from django.core.mail import send_mail
from django.contrib.auth.models import User
from django.utils.translation import ugettext_lazy as _
from django.core.cache import cache

# AWX
from awx.main.constants import CLOUD_PROVIDERS
from awx.main.models import * # noqa
from awx.main.models import UnifiedJob
from awx.main.queue import CallbackQueueDispatcher
from awx.main.task_engine import TaskEnhancer
from awx.main.utils import (get_ansible_version, get_ssh_version, decrypt_field, update_scm_url,
                            check_proot_installed, build_proot_temp_dir, wrap_args_with_proot,
                            get_system_task_capacity, OutputEventFilter, parse_yaml_or_json)
from awx.main.consumers import emit_channel_notification

__all__ = ['RunJob', 'RunSystemJob', 'RunProjectUpdate', 'RunInventoryUpdate',
           'RunAdHocCommand', 'handle_work_error',
           'handle_work_success', 'update_inventory_computed_fields',
           'send_notifications', 'run_administrative_checks', 'purge_old_stdout_files']

HIDDEN_PASSWORD = '**********'

OPENSSH_KEY_ERROR = u'''\
It looks like you're trying to use a private key in OpenSSH format, which \
isn't supported by the installed version of OpenSSH on this Tower instance. \
Try upgrading OpenSSH or providing your private key in an different format. \
'''

logger = logging.getLogger('awx.main.tasks')


@celeryd_init.connect
def celery_startup(conf=None, **kwargs):
    # Re-init all schedules
    # NOTE: Rework this during the Rampart work
    startup_logger = logging.getLogger('awx.main.tasks')
    startup_logger.info("Syncing Tower Schedules")
    for sch in Schedule.objects.all():
        try:
            sch.update_computed_fields()
            sch.save()
        except Exception as e:
            logger.error("Failed to rebuild schedule {}: {}".format(sch, e))


def _setup_tower_logger():
    global logger
    from django.utils.log import configure_logging
    LOGGING_DICT = settings.LOGGING
    if settings.LOG_AGGREGATOR_ENABLED:
        LOGGING_DICT['handlers']['http_receiver']['class'] = 'awx.main.utils.handlers.HTTPSHandler'
        LOGGING_DICT['handlers']['http_receiver']['async'] = False
        if 'awx' in settings.LOG_AGGREGATOR_LOGGERS:
            if 'http_receiver' not in LOGGING_DICT['loggers']['awx']['handlers']:
                LOGGING_DICT['loggers']['awx']['handlers'] += ['http_receiver']
    configure_logging(settings.LOGGING_CONFIG, LOGGING_DICT)
    logger = logging.getLogger('awx.main.tasks')


@worker_ready.connect
def task_set_logger_pre_run(*args, **kwargs):
    cache.close()
    if settings.LOG_AGGREGATOR_ENABLED:
        _setup_tower_logger()
        logger.debug('Custom Tower logger configured for worker process.')


def _uwsgi_reload():
    # http://uwsgi-docs.readthedocs.io/en/latest/MasterFIFO.html#available-commands
    logger.warn('Initiating uWSGI chain reload of server')
    TRIGGER_CHAIN_RELOAD = 'c'
    if settings.DEBUG:
        uWSGI_FIFO_LOCATION = '/awxfifo'
    else:
        uWSGI_FIFO_LOCATION = '/var/lib/awx/awxfifo'
    with open(uWSGI_FIFO_LOCATION, 'w') as awxfifo:
        awxfifo.write(TRIGGER_CHAIN_RELOAD)


def _reset_celery_logging():
    # Worker logger reloaded, now send signal to restart pool
    app = current_app._get_current_object()
    app.control.broadcast('pool_restart', arguments={'reload': True},
                          destination=['celery@{}'.format(settings.CLUSTER_HOST_ID)], reply=False)


def _clear_cache_keys(set_of_keys):
{'fraction_non_alphanumeric': 0.09025989229688597, 'fraction_numerical': 0.0009365488176071178, 'mean_word_length': 4.180715585203154, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5854397', 'n_tokens_mistral': 2683, 'n_tokens_neox': 2625, 'n_words': 666}