<commit_before>from collections import defaultdict
import logging
import os
import simplejson
import StringIO
import datetime
from dateutil.relativedelta import relativedelta

from django.conf import settings
from django.core.exceptions import PermissionDenied
from django.core.urlresolvers import reverse
from django.db.models import Count
from django.http import (
    HttpResponse,
    HttpResponseBadRequest,
    HttpResponseForbidden,
    HttpResponseNotAllowed,
    )
from django.shortcuts import (
    get_object_or_404,
    redirect,
    render_to_response,
)
from django.template import RequestContext
from django.template import defaultfilters as filters
from django.utils.html import escape
from django.utils.safestring import mark_safe
from django.db import models

from django_tables2 import Attrs, Column

from lava.utils.data_tables.tables import DataTablesTable

from lava_server.views import index as lava_index
from lava_server.bread_crumbs import (
    BreadCrumb,
    BreadCrumbTrail,
)

from lava_scheduler_app.logfile_helper import (
    formatLogFile,
    getDispatcherErrors,
    getDispatcherLogMessages
    )
from lava_scheduler_app.models import (
    Tag,
    Device,
    DeviceType,
    DeviceStateTransition,
    TestJob,
    )


def post_only(func):
    def decorated(request, *args, **kwargs):
        if request.method != 'POST':
            return HttpResponseNotAllowed('Only POST here')
        return func(request, *args, **kwargs)
    return decorated


class DateColumn(Column):

    def __init__(self, **kw):
        self._format = kw.get('date_format', settings.DATETIME_FORMAT)
        super(DateColumn, self).__init__(**kw)

    def render(self, value):
        return filters.date(value, self._format)


def pklink(record):
    return mark_safe(
        '<a href="%s">%s</a>' % (
            record.get_absolute_url(),
            escape(record.pk)))


class IDLinkColumn(Column):

    def __init__(self, verbose_name="ID", **kw):
        kw['verbose_name'] = verbose_name
        super(IDLinkColumn, self).__init__(**kw)

    def render(self, record):
        return pklink(record)


class RestrictedIDLinkColumn(IDLinkColumn):

    def render(self, record, table):
        if record.is_accessible_by(table.context.get('request').user):
            return pklink(record)
        else:
            return record.pk


def all_jobs_with_device_sort():
    return TestJob.objects.select_related(
        "actual_device", "requested_device", "requested_device_type",
        "submitter", "user", "group").extra(
        select={
            'device_sort': 'coalesce(actual_device_id, requested_device_id, requested_device_type_id)'
            }).all()



class JobTable(DataTablesTable):

    def render_device(self, record):
        if record.actual_device:
            return pklink(record.actual_device)
        elif record.requested_device:
            return pklink(record.requested_device)
        else:
            return mark_safe(
                '<i>' + escape(record.requested_device_type.pk) + '</i>')

    def render_description(self, value):
        if value:
            return value
        else:
            return ''

    id = RestrictedIDLinkColumn()
    status = Column()
    device = Column(accessor='device_sort')
    description = Column(attrs=Attrs(width="30%"))
    submitter = Column()
    submit_time = DateColumn()
    end_time = DateColumn()

    datatable_opts = {
        'aaSorting': [[0, 'desc']],
        }
    searchable_columns=['description']


class IndexJobTable(JobTable):
    def get_queryset(self):
        return all_jobs_with_device_sort().filter(
            status__in=[TestJob.SUBMITTED, TestJob.RUNNING])

    class Meta:
        exclude = ('end_time',)


def index_active_jobs_json(request):
    return IndexJobTable.json(request)


class DeviceTable(DataTablesTable):

    def get_queryset(self):
        return Device.objects.select_related("device_type")

    hostname = IDLinkColumn("hostname")
    device_type = Column()
    status = Column()
    health_status = Column()

    searchable_columns=['hostname']


def index_devices_json(request):
    return DeviceTable.json(request)

def health_jobs_in_hr(hr=-24):
    return TestJob.objects.filter(health_check=True,
           start_time__gte=(datetime.datetime.now() + relativedelta(hours=hr)))

def _online_total():
    ''' returns a tuple of (num_online, num_not_retired) '''
    r = Device.objects.all().values('status').annotate(count=Count('status'))
    offline = total = 0
    for res in r:
        if res['status'] in [Device.OFFLINE, Device.OFFLINING]:
            offline += res['count']
        if res['status'] != Device.RETIRED:
            total += res['count']

    return (total-offline,total)

@BreadCrumb("Scheduler", parent=lava_index)
def index(request):
    return render_to_response(
        "lava_scheduler_app/index.html",
        {<commit_msg>Change health job total number statistics method<commit_after>from collections import defaultdict
import logging
import os
import simplejson
import StringIO
import datetime
from dateutil.relativedelta import relativedelta

from django.conf import settings
from django.core.exceptions import PermissionDenied
from django.core.urlresolvers import reverse
from django.db.models import Count
from django.http import (
    HttpResponse,
    HttpResponseBadRequest,
    HttpResponseForbidden,
    HttpResponseNotAllowed,
    )
from django.shortcuts import (
    get_object_or_404,
    redirect,
    render_to_response,
)
from django.template import RequestContext
from django.template import defaultfilters as filters
from django.utils.html import escape
from django.utils.safestring import mark_safe
from django.db import models

from django_tables2 import Attrs, Column

from lava.utils.data_tables.tables import DataTablesTable

from lava_server.views import index as lava_index
from lava_server.bread_crumbs import (
    BreadCrumb,
    BreadCrumbTrail,
)

from lava_scheduler_app.logfile_helper import (
    formatLogFile,
    getDispatcherErrors,
    getDispatcherLogMessages
    )
from lava_scheduler_app.models import (
    Tag,
    Device,
    DeviceType,
    DeviceStateTransition,
    TestJob,
    )


def post_only(func):
    def decorated(request, *args, **kwargs):
        if request.method != 'POST':
            return HttpResponseNotAllowed('Only POST here')
        return func(request, *args, **kwargs)
    return decorated


class DateColumn(Column):

    def __init__(self, **kw):
        self._format = kw.get('date_format', settings.DATETIME_FORMAT)
        super(DateColumn, self).__init__(**kw)

    def render(self, value):
        return filters.date(value, self._format)


def pklink(record):
    return mark_safe(
        '<a href="%s">%s</a>' % (
            record.get_absolute_url(),
            escape(record.pk)))


class IDLinkColumn(Column):

    def __init__(self, verbose_name="ID", **kw):
        kw['verbose_name'] = verbose_name
        super(IDLinkColumn, self).__init__(**kw)

    def render(self, record):
        return pklink(record)


class RestrictedIDLinkColumn(IDLinkColumn):

    def render(self, record, table):
        if record.is_accessible_by(table.context.get('request').user):
            return pklink(record)
        else:
            return record.pk


def all_jobs_with_device_sort():
    return TestJob.objects.select_related(
        "actual_device", "requested_device", "requested_device_type",
        "submitter", "user", "group").extra(
        select={
            'device_sort': 'coalesce(actual_device_id, requested_device_id, requested_device_type_id)'
            }).all()



class JobTable(DataTablesTable):

    def render_device(self, record):
        if record.actual_device:
            return pklink(record.actual_device)
        elif record.requested_device:
            return pklink(record.requested_device)
        else:
            return mark_safe(
                '<i>' + escape(record.requested_device_type.pk) + '</i>')

    def render_description(self, value):
        if value:
            return value
        else:
            return ''

    id = RestrictedIDLinkColumn()
    status = Column()
    device = Column(accessor='device_sort')
    description = Column(attrs=Attrs(width="30%"))
    submitter = Column()
    submit_time = DateColumn()
    end_time = DateColumn()

    datatable_opts = {
        'aaSorting': [[0, 'desc']],
        }
    searchable_columns=['description']


class IndexJobTable(JobTable):
    def get_queryset(self):
        return all_jobs_with_device_sort().filter(
            status__in=[TestJob.SUBMITTED, TestJob.RUNNING])

    class Meta:
        exclude = ('end_time',)


def index_active_jobs_json(request):
    return IndexJobTable.json(request)


class DeviceTable(DataTablesTable):

    def get_queryset(self):
        return Device.objects.select_related("device_type")

    hostname = IDLinkColumn("hostname")
    device_type = Column()
    status = Column()
    health_status = Column()

    searchable_columns=['hostname']


def index_devices_json(request):
    return DeviceTable.json(request)

def health_jobs_in_hr(hr=-24):
    return TestJob.objects.filter(health_check=True,
           start_time__gte=(datetime.datetime.now()
               + relativedelta(hours=hr))).exclude(status__in=[TestJob.SUBMITTED, TestJob.RUNNING])

def _online_total():
    ''' returns a tuple of (num_online, num_not_retired) '''
    r = Device.objects.all().values('status').annotate(count=Count('status'))
    offline = total = 0
    for res in r:
        if res['status'] in [Device.OFFLINE, Device.OFFLINING]:
            offline += res['count']
        if res['status'] != Device.RETIRED:
            total += res['count']

    return (total-offline,total)

@BreadCrumb("Scheduler", parent=lava_index)
def index(request):
    return render_to_response(
        "lava_scheduler_app/index.html",
        {
{'fraction_non_alphanumeric': 0.10296930045294414, 'fraction_numerical': 0.0020130850528434826, 'mean_word_length': 3.327526132404181, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3279690', 'n_tokens_mistral': 3018, 'n_tokens_neox': 2925, 'n_words': 639}