<commit_before>from lxml import etree

import HTMLParser
import json
import socket
import csv
from datetime import timedelta, date, datetime
from collections import defaultdict, namedtuple, OrderedDict
from StringIO import StringIO

import dateutil
from django.utils.datastructures import SortedDict
from django.utils.safestring import mark_safe
from django.views.decorators.http import require_POST
from django.conf import settings
from django.contrib import messages
from django.contrib.auth.models import User
from django.contrib.auth import login
from django.core import management, cache
from django.shortcuts import render
from django.template.loader import render_to_string
from django.views.generic import FormView, TemplateView
from django.utils.decorators import method_decorator
from django.utils.translation import ugettext as _, ugettext_lazy
from django.http import (
    HttpResponseRedirect,
    HttpResponse,
    HttpResponseBadRequest,
    HttpResponseNotFound,
)
from restkit import Resource
from restkit.errors import Unauthorized
from couchdbkit import ResourceNotFound

from casexml.apps.case.models import CommCareCase
from casexml.apps.phone.xml import SYNC_XMLNS
from corehq.apps.callcenter.indicator_sets import CallCenterIndicators
from corehq.apps.callcenter.utils import CallCenterCase
from corehq.apps.hqwebapp.tasks import send_html_email_async
from corehq.apps.hqwebapp.views import BaseSectionPageView
from corehq.apps.style.decorators import use_datatables, use_jquery_ui, \
    use_nvd3_v3
from corehq.form_processor.exceptions import XFormNotFound, CaseNotFound
from corehq.form_processor.models import XFormInstanceSQL, CommCareCaseSQL
from corehq.form_processor.serializers import XFormInstanceSQLRawDocSerializer, \
    CommCareCaseSQLRawDocSerializer
from corehq.form_processor.interfaces.dbaccessors import CaseAccessors
from corehq.toggles import any_toggle_enabled, SUPPORT
from corehq.util import reverse
from corehq.util.couchdb_management import couch_config
from corehq.util.supervisord.api import (
    PillowtopSupervisorApi,
    SupervisorException,
    all_pillows_supervisor_status,
    pillow_supervisor_status
)
from corehq.apps.app_manager.models import ApplicationBase
from corehq.apps.app_manager.dbaccessors import get_app_ids_in_domain
from corehq.apps.data_analytics.models import MALTRow, GIRRow
from corehq.apps.data_analytics.const import GIR_FIELDS
from corehq.apps.data_analytics.admin import MALTRowAdmin
from corehq.apps.domain.decorators import require_superuser, require_superuser_or_developer
from corehq.apps.domain.models import Domain
from corehq.apps.ota.views import get_restore_response, get_restore_params
from corehq.apps.users.models import CommCareUser, WebUser, CouchUser
from corehq.apps.users.util import format_username
from corehq.elastic import parse_args_for_es, run_query, ES_META
from corehq.util.timer import TimingContext
from couchforms.models import XFormInstance
from dimagi.utils.couch.database import get_db, is_bigcouch
from dimagi.utils.dates import force_to_date
from dimagi.utils.django.management import export_as_csv_action
from dimagi.utils.decorators.datespan import datespan_in_request
from dimagi.utils.parsing import json_format_date
from dimagi.utils.web import json_response
from pillowtop.exceptions import PillowNotFoundError
from pillowtop.utils import get_all_pillows_json, get_pillow_json, get_pillow_config_by_name

from . import service_checks, escheck
from .forms import AuthenticateAsForm, BrokenBuildsForm
from .history import get_recent_changes, download_changes
from .models import HqDeploy, VCMMigration
from .reporting.reports import get_project_spaces, get_stats_data
from .utils import get_celery_stats
from corehq.apps.es.domains import DomainES


@require_superuser
def default(request):
    return HttpResponseRedirect(reverse('admin_report_dispatcher', args=('domains',)))

datespan_default = datespan_in_request(
    from_param="startdate",
    to_param="enddate",
    default_days=30,
)


def get_rabbitmq_management_url():
    if settings.BROKER_URL.startswith('amqp'):
        amqp_parts = settings.BROKER_URL.replace('amqp://','').split('/')
        mq_management_url = amqp_parts[0].replace('5672', '15672')
        return "http://%s" % mq_management_url.split('@')[-1]
    else:<commit_msg>added filters query<commit_after>from lxml import etree

import HTMLParser
import json
import socket
import csv
from datetime import timedelta, date, datetime
from collections import defaultdict, namedtuple, OrderedDict
from StringIO import StringIO

import dateutil
from django.utils.datastructures import SortedDict
from django.utils.safestring import mark_safe
from django.views.decorators.http import require_POST
from django.conf import settings
from django.contrib import messages
from django.contrib.auth.models import User
from django.contrib.auth import login
from django.core import management, cache
from django.shortcuts import render
from django.template.loader import render_to_string
from django.views.generic import FormView, TemplateView
from django.utils.decorators import method_decorator
from django.utils.translation import ugettext as _, ugettext_lazy
from django.http import (
    HttpResponseRedirect,
    HttpResponse,
    HttpResponseBadRequest,
    HttpResponseNotFound,
)
from restkit import Resource
from restkit.errors import Unauthorized
from couchdbkit import ResourceNotFound

from casexml.apps.case.models import CommCareCase
from casexml.apps.phone.xml import SYNC_XMLNS
from corehq.apps.callcenter.indicator_sets import CallCenterIndicators
from corehq.apps.callcenter.utils import CallCenterCase
from corehq.apps.hqwebapp.tasks import send_html_email_async
from corehq.apps.hqwebapp.views import BaseSectionPageView
from corehq.apps.style.decorators import use_datatables, use_jquery_ui, \
    use_nvd3_v3
from corehq.form_processor.exceptions import XFormNotFound, CaseNotFound
from corehq.form_processor.models import XFormInstanceSQL, CommCareCaseSQL
from corehq.form_processor.serializers import XFormInstanceSQLRawDocSerializer, \
    CommCareCaseSQLRawDocSerializer
from corehq.form_processor.interfaces.dbaccessors import CaseAccessors
from corehq.toggles import any_toggle_enabled, SUPPORT
from corehq.util import reverse
from corehq.util.couchdb_management import couch_config
from corehq.util.supervisord.api import (
    PillowtopSupervisorApi,
    SupervisorException,
    all_pillows_supervisor_status,
    pillow_supervisor_status
)
from corehq.apps.app_manager.models import ApplicationBase
from corehq.apps.app_manager.dbaccessors import get_app_ids_in_domain
from corehq.apps.data_analytics.models import MALTRow, GIRRow
from corehq.apps.data_analytics.const import GIR_FIELDS
from corehq.apps.data_analytics.admin import MALTRowAdmin
from corehq.apps.domain.decorators import require_superuser, require_superuser_or_developer
from corehq.apps.domain.models import Domain
from corehq.apps.ota.views import get_restore_response, get_restore_params
from corehq.apps.users.models import CommCareUser, WebUser, CouchUser
from corehq.apps.users.util import format_username
from corehq.elastic import parse_args_for_es, run_query, ES_META
from corehq.util.timer import TimingContext
from couchforms.models import XFormInstance
from dimagi.utils.couch.database import get_db, is_bigcouch
from dimagi.utils.dates import force_to_date
from dimagi.utils.django.management import export_as_csv_action
from dimagi.utils.decorators.datespan import datespan_in_request
from dimagi.utils.parsing import json_format_date
from dimagi.utils.web import json_response
from pillowtop.exceptions import PillowNotFoundError
from pillowtop.utils import get_all_pillows_json, get_pillow_json, get_pillow_config_by_name

from . import service_checks, escheck
from .forms import AuthenticateAsForm, BrokenBuildsForm
from .history import get_recent_changes, download_changes
from .models import HqDeploy, VCMMigration
from .reporting.reports import get_project_spaces, get_stats_data
from .utils import get_celery_stats
from corehq.apps.es.domains import DomainES
from corehq.apps.es import filters

@require_superuser
def default(request):
    return HttpResponseRedirect(reverse('admin_report_dispatcher', args=('domains',)))

datespan_default = datespan_in_request(
    from_param="startdate",
    to_param="enddate",
    default_days=30,
)


def get_rabbitmq_management_url():
    if settings.BROKER_URL.startswith('amqp'):
        amqp_parts = settings.BROKER_URL.replace('amqp://','').split('/')
        mq_management_url = amqp_parts[0].replace('5672', '15672')
        return "http://%s" % mq_management_url.split('@')[-1]
    else:
{'fraction_non_alphanumeric': 0.05890473999079614, 'fraction_numerical': 0.003451449608835711, 'mean_word_length': 6.546006944444445, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2946712', 'n_tokens_mistral': 2645, 'n_tokens_neox': 2581, 'n_words': 653}