<commit_before>from urlparse import urlparse
from datetime import datetime
import json
import os
import re
import uuid

from django.conf import settings
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.core import cache
from django.core.cache import InvalidCacheBackendError
from django.template.loader import render_to_string
from django.views.decorators.http import require_POST
from django.contrib.auth.forms import AdminPasswordChangeForm
from django.contrib.auth.models import User
from django.contrib.auth.views import login as django_login, redirect_to_login
from django.contrib.auth.views import logout as django_logout
from django.contrib.sites.models import Site
from django.http import HttpResponseRedirect, HttpResponse, Http404,\
    HttpResponseServerError, HttpResponseNotFound, HttpResponseBadRequest
from django.shortcuts import redirect, render
from django.views.generic import TemplateView
from couchdbkit import ResourceNotFound
from django.utils.translation import ugettext as _, ugettext_noop
from django.core.urlresolvers import reverse
from django.core.mail.message import EmailMessage
from django.template import loader
from django.template.context import RequestContext

from corehq.apps.announcements.models import Notification
from corehq.apps.app_manager.models import BUG_REPORTS_DOMAIN
from corehq.apps.app_manager.models import import_app
from corehq.apps.domain.decorators import require_superuser,\
    login_and_domain_required
from corehq.apps.domain.utils import normalize_domain_name, get_domain_from_url
from corehq.apps.hqwebapp.forms import EmailAuthenticationForm, CloudCareAuthenticationForm
from corehq.apps.receiverwrapper.models import Repeater
from corehq.apps.users.models import CouchUser
from corehq.apps.users.util import format_username
from corehq.apps.hqwebapp.doc_info import get_doc_info
from dimagi.utils.couch.database import get_db
from dimagi.utils.decorators.memoized import memoized
from dimagi.utils.logging import notify_exception
from dimagi.utils.web import get_url_base, json_response
from corehq.apps.domain.models import Domain
from couchforms.models import XFormInstance
from soil import heartbeat
from soil import views as soil_views


def pg_check():
    """check django db"""
    try:
        user_count = User.objects.count()
    except:
        user_count = None
    return user_count is not None


def couch_check():
    """check couch"""

    #in reality when things go wrong with couch and postgres (as of this
    # writing) - it's far from graceful, so this will # likely never be
    # reached because another exception will fire first - but for
    # completeness  sake, this check is done  here to verify our calls will
    # work, and if other error handling allows the request to get this far.

    try:
        xforms = XFormInstance.view('couchforms/by_user', limit=1).all()
    except:
        xforms = None
    return isinstance(xforms, list)


def hb_check():
    try:
        hb = heartbeat.is_alive()
    except:
        hb = False
    return hb


def redis_check():
    try:
        redis = cache.get_cache('redis')
        return redis.set('serverup_check_key', 'test')
    except (InvalidCacheBackendError, ValueError):
        return True  # redis not in use, ignore
    except:
        return False


def memcached_check():
    try:
        memcached = cache.get_cache('default')
        uuid_val = uuid.uuid1()
        memcached.set('serverup_check_key', uuid_val)
        return memcached.get('serverup_check_key') == uuid_val
    except:
        return False


def server_error(request, template_name='500.html'):
    """
    500 error handler.
    """

    domain = get_domain_from_url(request.path) or ''


    # hat tip: http://www.arthurkoziel.com/2009/01/15/passing-mediaurl-djangos-500-error-view/
    t = loader.get_template(template_name)
    return HttpResponseServerError(t.render(RequestContext(request,
        {'MEDIA_URL': settings.MEDIA_URL,
         'STATIC_URL': settings.STATIC_URL,
         'domain': domain
        })))


def not_found(request, template_name='404.html'):
    """
    404 error handler.
    """
    t = loader.get_template(template_name)
    return HttpResponseNotFound(t.render(RequestContext(request,
        {'MEDIA_URL': settings.MEDIA_URL,
         'STATIC_URL': settings.STATIC_URL
        })))


def redirect_to_default(req, domain=None):
    if not req.user.is_authenticated():
        if domain != None:
            url = reverse('domain_login', args=[domain])
        else:
            # this actually gets hijacked by the static site, but is necessary
            url = reverse('corehq.apps.hqwebapp.views.landing_page')
    else:
        if domain:
            domain = normalize_domain_name(domain)
            domains = [Domain.get_by_name(domain)]
        else:
            domains = Domain.active_for_user(req.user)
        if 0 == len(domains) and not req.user.is_superuser:
            return redirect('registration_domain')
        elif 1 == len(domains):
            if domains[0]:
                domain = domains[0].name
                if req.couch_user.is_commcare_user():
                    url = reverse("cloudcare_main", args=[domain, ""])
                elif req.couch_user.can_view_reports(domain) or req.couch_user.get_viewable_reports(domain):
                    url = reverse('corehq.apps.reports.views.default', args=[domain])
                else:
                    url = reverse('corehq.apps.app_manager.views.default', args=[domain])
            else:
                raise Http404
        else:
            url = settings.DOMAIN_SELECT_URL
    return HttpResponseRedirect(url)


def landing_page(req, template_name="home.html"):
    # this view, and the one below, is overridden because
    # we need to set the base template to use somewhere
    # somewhere that the login page can access it.
    if req.user.is_authenticated():
        return HttpResponseRedirect(reverse('homepage'))
    req.base_template = settings.BASE_TEMPLATE
    return django_login(req, template_name=template_name, authentication_form=EmailAuthenticationForm)


def yui_crossdomain(req):
    x_domain = """<?xml version="1.0"?>
<!DOCTYPE cross-domain-policy SYSTEM "http://www.macromedia.com/xml/dtds/cross-domain-policy.dtd">
<cross-domain-policy>
    <allow-access-from domain="yui.yahooapis.com"/>
    <allow-access-from domain="%s"/>
    <site-control permitted-cross-domain-policies="master-only"/>
</cross-domain-policy>""" % Site.objects.get(id=settings.SITE_ID).domain
    return HttpResponse(x_domain, mimetype="application/xml")


@login_required()
def password_change(req):
    user_to_edit = User.objects.get(id=req.user.id)
    if req.method == 'POST':
        password_form = AdminPasswordChangeForm(user_to_edit, req.POST)
        if password_form.is_valid():
            password_form.save()
            return HttpResponseRedirect('/')
    else:
        password_form = AdminPasswordChangeForm(user_to_edit)
    template_name = "password_change.html"
    return render(req, template_name, {"form": password_form})


def server_up(req):
    '''View that just returns "success", which can be hooked into server
       monitoring tools like: pingdom'''


    checkers = {
        "heartbeat": {
            "always_check": False,
            "message": "* celery heartbeat is down",
            "check_func": hb_check
        },
        "postgres": {
            "always_check": True,
            "message": "* postgres has issues",
            "check_func": pg_check
        },
        "couch": {
            "always_check": True,
            "message": "* couch has issues",
            "check_func": couch_check
        },
        "redis": {
            "always_check": True,
            "message": "* redis has issues",
            "check_func": redis_check
        },
        "memcached": {
            "always_check": True,
            "message": "* memcached has issues",
            "check_func": memcached_check
        }
    }

    failed = False
    message = ['Problems with HQ (%s):' % os.uname()[1]]
    for check, check_info in checkers.items():
        if check_info['always_check'] or req.GET.get(check, None) is not None:
            check_results = check_info['check_func']()
            if not check_results:
                failed = True
                message.append(check_info['message'])
    if failed:
        return HttpResponse('<br>'.join(message), status=500)
    else:
        return HttpResponse("success")

def no_permissions(request, redirect_to=None, template_name="no_permission.html"):
    next = redirect_to or request.GET.get('next', None)
    if request.GET.get('switch', None) == 'true':
        logout(request)
        return redirect_to_login(next or request.path)

    return render(request, template_name, {'next': next})
<commit_msg>use celery flower api to check all queues<commit_after>from urlparse import urlparse
from datetime import datetime
import json
import os
import re
import uuid

from django.conf import settings
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.core import cache
from django.core.cache import InvalidCacheBackendError
from django.template.loader import render_to_string
from django.views.decorators.http import require_POST
from django.contrib.auth.forms import AdminPasswordChangeForm
from django.contrib.auth.models import User
from django.contrib.auth.views import login as django_login, redirect_to_login
from django.contrib.auth.views import logout as django_logout
from django.contrib.sites.models import Site
from django.http import HttpResponseRedirect, HttpResponse, Http404,\
    HttpResponseServerError, HttpResponseNotFound, HttpResponseBadRequest
from django.shortcuts import redirect, render
from django.views.generic import TemplateView
from couchdbkit import ResourceNotFound
from django.utils.translation import ugettext as _, ugettext_noop
from django.core.urlresolvers import reverse
from django.core.mail.message import EmailMessage
from django.template import loader
from django.template.context import RequestContext
from restkit import Resource

from corehq.apps.announcements.models import Notification
from corehq.apps.app_manager.models import BUG_REPORTS_DOMAIN
from corehq.apps.app_manager.models import import_app
from corehq.apps.domain.decorators import require_superuser,\
    login_and_domain_required
from corehq.apps.domain.utils import normalize_domain_name, get_domain_from_url
from corehq.apps.hqwebapp.forms import EmailAuthenticationForm, CloudCareAuthenticationForm
from corehq.apps.receiverwrapper.models import Repeater
from corehq.apps.users.models import CouchUser
from corehq.apps.users.util import format_username
from corehq.apps.hqwebapp.doc_info import get_doc_info
from dimagi.utils.couch.database import get_db
from dimagi.utils.decorators.memoized import memoized
from dimagi.utils.logging import notify_exception
from dimagi.utils.web import get_url_base, json_response
from corehq.apps.domain.models import Domain
from couchforms.models import XFormInstance
from soil import heartbeat
from soil import views as soil_views


def pg_check():
    """check django db"""
    try:
        user_count = User.objects.count()
    except:
        user_count = None
    return user_count is not None


def couch_check():
    """check couch"""

    #in reality when things go wrong with couch and postgres (as of this
    # writing) - it's far from graceful, so this will # likely never be
    # reached because another exception will fire first - but for
    # completeness  sake, this check is done  here to verify our calls will
    # work, and if other error handling allows the request to get this far.

    try:
        xforms = XFormInstance.view('couchforms/by_user', limit=1).all()
    except:
        xforms = None
    return isinstance(xforms, list)


def hb_check():
    celery_monitoring = getattr(settings, 'CELERY_FLOWER_URL', None)
    if celery_monitoring:
        try:
            cresource = Resource(celery_monitoring, timeout=3)
            t = cresource.get("api/workers").body_string()
            all_workers = json.loads(t)
            bad_workers = []
            for hostname, w in all_workers.items():
                if not w['status']:
                    bad_workers.append('* {} celery worker down'.format(hostname))
            if bad_workers:
                return '\n'.join(bad_workers)
            else:
                hb = True
        except:
            hb = False
    else:
        try:
            hb = heartbeat.is_alive()
        except:
            hb = False
    return hb


def redis_check():
    try:
        redis = cache.get_cache('redis')
        return redis.set('serverup_check_key', 'test')
    except (InvalidCacheBackendError, ValueError):
        return True  # redis not in use, ignore
    except:
        return False


def memcached_check():
    try:
        memcached = cache.get_cache('default')
        uuid_val = uuid.uuid1()
        memcached.set('serverup_check_key', uuid_val)
        return memcached.get('serverup_check_key') == uuid_val
    except:
        return False


def server_error(request, template_name='500.html'):
    """
    500 error handler.
    """

    domain = get_domain_from_url(request.path) or ''


    # hat tip: http://www.arthurkoziel.com/2009/01/15/passing-mediaurl-djangos-500-error-view/
    t = loader.get_template(template_name)
    return HttpResponseServerError(t.render(RequestContext(request,
        {'MEDIA_URL': settings.MEDIA_URL,
         'STATIC_URL': settings.STATIC_URL,
         'domain': domain
        })))


def not_found(request, template_name='404.html'):
    """
    404 error handler.
    """
    t = loader.get_template(template_name)
    return HttpResponseNotFound(t.render(RequestContext(request,
        {'MEDIA_URL': settings.MEDIA_URL,
         'STATIC_URL': settings.STATIC_URL
        })))


def redirect_to_default(req, domain=None):
    if not req.user.is_authenticated():
        if domain != None:
            url = reverse('domain_login', args=[domain])
        else:
            # this actually gets hijacked by the static site, but is necessary
            url = reverse('corehq.apps.hqwebapp.views.landing_page')
    else:
        if domain:
            domain = normalize_domain_name(domain)
            domains = [Domain.get_by_name(domain)]
        else:
            domains = Domain.active_for_user(req.user)
        if 0 == len(domains) and not req.user.is_superuser:
            return redirect('registration_domain')
        elif 1 == len(domains):
            if domains[0]:
                domain = domains[0].name
                if req.couch_user.is_commcare_user():
                    url = reverse("cloudcare_main", args=[domain, ""])
                elif req.couch_user.can_view_reports(domain) or req.couch_user.get_viewable_reports(domain):
                    url = reverse('corehq.apps.reports.views.default', args=[domain])
                else:
                    url = reverse('corehq.apps.app_manager.views.default', args=[domain])
            else:
                raise Http404
        else:
            url = settings.DOMAIN_SELECT_URL
    return HttpResponseRedirect(url)


def landing_page(req, template_name="home.html"):
    # this view, and the one below, is overridden because
    # we need to set the base template to use somewhere
    # somewhere that the login page can access it.
    if req.user.is_authenticated():
        return HttpResponseRedirect(reverse('homepage'))
    req.base_template = settings.BASE_TEMPLATE
    return django_login(req, template_name=template_name, authentication_form=EmailAuthenticationForm)


def yui_crossdomain(req):
    x_domain = """<?xml version="1.0"?>
<!DOCTYPE cross-domain-policy SYSTEM "http://www.macromedia.com/xml/dtds/cross-domain-policy.dtd">
<cross-domain-policy>
    <allow-access-from domain="yui.yahooapis.com"/>
    <allow-access-from domain="%s"/>
    <site-control permitted-cross-domain-policies="master-only"/>
</cross-domain-policy>""" % Site.objects.get(id=settings.SITE_ID).domain
    return HttpResponse(x_domain, mimetype="application/xml")


@login_required()
def password_change(req):
    user_to_edit = User.objects.get(id=req.user.id)
    if req.method == 'POST':
        password_form = AdminPasswordChangeForm(user_to_edit, req.POST)
        if password_form.is_valid():
            password_form.save()
            return HttpResponseRedirect('/')
    else:
        password_form = AdminPasswordChangeForm(user_to_edit)
    template_name = "password_change.html"
    return render(req, template_name, {"form": password_form})


def server_up(req):
    '''View that just returns "success", which can be hooked into server
       monitoring tools like: pingdom'''


    checkers = {
        "heartbeat": {
            "always_check": False,
            "message": "* celery heartbeat is down",
            "check_func": hb_check
        },
        "postgres": {
            "always_check": True,
            "message": "* postgres has issues",
            "check_func": pg_check
        },
        "couch": {
            "always_check": True,
            "message": "* couch has issues",
            "check_func": couch_check
        },
        "redis": {
            "always_check": True,
            "message": "* redis has issues",
            "check_func": redis_check
        },
        "memcached": {
            "always_check": True,
            "message": "* memcached has issues",
            "check_func": memcached_check
        }
    }

    failed = False
    message = ['Problems with HQ (%s):' % os.uname()[1]]
    for check, check_info in checkers.items():
        if check_info['always_check'] or req.GET.get(check, None) is not None:
            check_results = check_info['check_func']()
            if not check_results:
                failed = True
                if isinstance(check_results, basestring):
                    message.append(check_results)
                else:
                    message.append(check_info['message'])
    if failed:
        return HttpResponse('<br>'.join(message), status=500)
    else:
        return HttpResponse("success")

def no_permissions(request, redirect_to=None, template_name="no_permission.html"):
    next = redirect_to or request.GET.get('next', None)
    if request.GET.get('switch', None) == 'true':
        logout(request)
        return redirect_to_login(next or request.path)

    return render(request, template_name, {'next': next})

{'fraction_non_alphanumeric': 0.09627961831242629, 'fraction_numerical': 0.004449447839605446, 'mean_word_length': 2.9997855917667238, 'pattern_counts': {'":': 42, '<': 19, '<?xml version=': 2, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '790477', 'n_tokens_mistral': 5431, 'n_tokens_neox': 5249, 'n_words': 1348}