<commit_before>import json
import os
import random
from PIL import Image
import socket
import StringIO
import time
import urllib2
from urlparse import urlparse

from django import http
from django.conf import settings
from django.core.cache import cache, parse_backend_uri
from django.http import HttpResponse, HttpResponseBadRequest
from django.views.decorators.cache import never_cache
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_POST

import caching.invalidation
import celery.task
import commonware.log
import jingo
import phpserialize as php

import amo
import mongoutils
from hera.contrib.django_utils import get_hera
from stats.models import Contribution, ContributionError, SubscriptionEvent

monitor_log = commonware.log.getLogger('z.monitor')
paypal_log = commonware.log.getLogger('z.paypal')
csp_log = commonware.log.getLogger('z.csp')


def check_redis():
    redis = caching.invalidation.get_redis_backend()
    try:
        return redis.info(), None
    except Exception, e:
        monitor_log.critical('Failed to chat with redis: (%s)' % e)
        return None, e


@never_cache
def monitor(request):

    # For each check, a boolean pass/fail status to show in the template
    status_summary = {}
    status = 200

    # Check all memcached servers
    scheme, servers, _ = parse_backend_uri(settings.CACHE_BACKEND)
    memcache_results = []
    status_summary['memcache'] = True
    if 'memcached' in scheme:
        hosts = servers.split(';')
        for host in hosts:
            ip, port = host.split(':')
            try:
                s = socket.socket()
                s.connect((ip, int(port)))
            except Exception, e:
                result = False
                status_summary['memcache'] = False
                monitor_log.critical('Failed to connect to memcached (%s): %s' %
                                                                    (host, e))
            else:
                result = True
            finally:
                s.close()

            memcache_results.append((ip, port, result))
        if len(memcache_results) < 2:
            status_summary['memcache'] = False
            monitor_log.warning('You should have 2+ memcache servers.  You have %s.' %
                                                        len(memcache_results))
    if not memcache_results:
        status_summary['memcache'] = False
        monitor_log.info('Memcache is not configured.')

    # Check Libraries and versions
    libraries_results = []
    status_summary['libraries'] = True
    try:
        Image.new('RGB', (16, 16)).save(StringIO.StringIO(), 'JPEG')
        libraries_results.append(('PIL+JPEG', True, 'Got it!'))
    except Exception, e:
        status_summary['libraries'] = False
        msg = "Failed to create a jpeg image: %s" % e
        libraries_results.append(('PIL+JPEG', False, msg))

    # Check file paths / permissions
    filepaths = (
        (settings.NETAPP_STORAGE, os.R_OK | os.W_OK, "We want read + write."),
        (settings.UPLOADS_PATH, os.R_OK | os.W_OK, "We want read + write."),
        (settings.ADDON_ICONS_PATH,
            os.R_OK | os.W_OK, "We want read + write."),
        (settings.COLLECTIONS_ICON_PATH,
            os.R_OK | os.W_OK, "We want read + write."),
        (settings.PREVIEWS_PATH, os.R_OK | os.W_OK, "We want read + write."),
        (settings.USERPICS_PATH, os.R_OK | os.W_OK, "We want read + write."),
    )
    filepath_results = []
    filepath_status = True

    for path, perms, notes in filepaths:
        path_exists = os.path.isdir(path)
        path_perms = os.access(path, perms)
        filepath_status = filepath_status and path_exists and path_perms
        filepath_results.append((path, path_exists, path_perms, notes))

    status_summary['filepaths'] = filepath_status

    # Check Redis
    redis_results = [None, 'REDIS_BACKEND is not set']
    if getattr(settings, 'REDIS_BACKEND', False):
        redis_results = check_redis()
    status_summary['redis'] = bool(redis_results[0])

    # Check Hera
    hera_results = []
    status_summary['hera'] = True
    for i in settings.HERA:
        r = {'location': urlparse(i['LOCATION'])[1],
             'result': bool(get_hera(i))}
        hera_results.append(r)
        if not hera_results[-1]['result']:
            status_summary['hera'] = False

    # Check Rabbit
    # start = time.time()
    # pong = celery.task.ping()<commit_msg>add in TMP_PATH (bug 614429)<commit_after>import json
import os
import random
from PIL import Image
import socket
import StringIO
import time
import urllib2
from urlparse import urlparse

from django import http
from django.conf import settings
from django.core.cache import cache, parse_backend_uri
from django.http import HttpResponse, HttpResponseBadRequest
from django.views.decorators.cache import never_cache
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_POST

import caching.invalidation
import celery.task
import commonware.log
import jingo
import phpserialize as php

import amo
import mongoutils
from hera.contrib.django_utils import get_hera
from stats.models import Contribution, ContributionError, SubscriptionEvent

monitor_log = commonware.log.getLogger('z.monitor')
paypal_log = commonware.log.getLogger('z.paypal')
csp_log = commonware.log.getLogger('z.csp')


def check_redis():
    redis = caching.invalidation.get_redis_backend()
    try:
        return redis.info(), None
    except Exception, e:
        monitor_log.critical('Failed to chat with redis: (%s)' % e)
        return None, e


@never_cache
def monitor(request):

    # For each check, a boolean pass/fail status to show in the template
    status_summary = {}
    status = 200

    # Check all memcached servers
    scheme, servers, _ = parse_backend_uri(settings.CACHE_BACKEND)
    memcache_results = []
    status_summary['memcache'] = True
    if 'memcached' in scheme:
        hosts = servers.split(';')
        for host in hosts:
            ip, port = host.split(':')
            try:
                s = socket.socket()
                s.connect((ip, int(port)))
            except Exception, e:
                result = False
                status_summary['memcache'] = False
                monitor_log.critical('Failed to connect to memcached (%s): %s' %
                                                                    (host, e))
            else:
                result = True
            finally:
                s.close()

            memcache_results.append((ip, port, result))
        if len(memcache_results) < 2:
            status_summary['memcache'] = False
            monitor_log.warning('You should have 2+ memcache servers.  You have %s.' %
                                                        len(memcache_results))
    if not memcache_results:
        status_summary['memcache'] = False
        monitor_log.info('Memcache is not configured.')

    # Check Libraries and versions
    libraries_results = []
    status_summary['libraries'] = True
    try:
        Image.new('RGB', (16, 16)).save(StringIO.StringIO(), 'JPEG')
        libraries_results.append(('PIL+JPEG', True, 'Got it!'))
    except Exception, e:
        status_summary['libraries'] = False
        msg = "Failed to create a jpeg image: %s" % e
        libraries_results.append(('PIL+JPEG', False, msg))

    # Check file paths / permissions
    filepaths = (
        (settings.TMP_PATH, os.R_OK | os.W_OK, "We want read + write."),
        (settings.NETAPP_STORAGE, os.R_OK | os.W_OK, "We want read + write."),
        (settings.UPLOADS_PATH, os.R_OK | os.W_OK, "We want read + write."),
        (settings.ADDON_ICONS_PATH,
            os.R_OK | os.W_OK, "We want read + write."),
        (settings.COLLECTIONS_ICON_PATH,
            os.R_OK | os.W_OK, "We want read + write."),
        (settings.PREVIEWS_PATH, os.R_OK | os.W_OK, "We want read + write."),
        (settings.USERPICS_PATH, os.R_OK | os.W_OK, "We want read + write."),
        (settings.SPHINX_CATALOG_PATH, os.R_OK | os.W_OK, "We want read + write."),
        (settings.SPHINX_LOG_PATH, os.R_OK | os.W_OK, "We want read + write."),
    )
    filepath_results = []
    filepath_status = True

    for path, perms, notes in filepaths:
        path_exists = os.path.isdir(path)
        path_perms = os.access(path, perms)
        filepath_status = filepath_status and path_exists and path_perms
        filepath_results.append((path, path_exists, path_perms, notes))

    status_summary['filepaths'] = filepath_status

    # Check Redis
    redis_results = [None, 'REDIS_BACKEND is not set']
    if getattr(settings, 'REDIS_BACKEND', False):
        redis_results = check_redis()
    status_summary['redis'] = bool(redis_results[0])

    # Check Hera
    hera_results = []
    status_summary['hera'] = True
    for i in settings.HERA:
        r = {'location': urlparse(i['LOCATION'])[1],
             'result': bool(get_hera(i))}
        hera_results.append(r)
        if not hera_results[-1]['result']:
            status_summary['hera'] = False

    # Check Rabbit
    # start = time.time()
    # pong = celery.task.ping()
{'fraction_non_alphanumeric': 0.10595522712453814, 'fraction_numerical': 0.003477504890241252, 'mean_word_length': 2.5560278207109737, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5957534', 'n_tokens_mistral': 2838, 'n_tokens_neox': 2770, 'n_words': 718}