<commit_before>#
# ==============================
# Copyright 2011 Whamcloud, Inc.
# ==============================

import datetime

from chroma_core.models.log import Systemevents

from tastypie.resources import ModelResource
from tastypie.authorization import DjangoAuthorization
from chroma_api.authentication import AnonymousAuthentication


class LogResource(ModelResource):
    """
    syslog messages collected by Chroma server.  The fields in this table
    correspond to the default ``rsyslog`` MySQL output.

    You are probably mainly interested in ``fromhost``, ``devicereportedtime`` and
    ``message``.  Note that ``fromhost`` is a hostname rather than a reference to
    the ``host`` resource -- it is not guaranteed that a host mentioned in the
    syslog is configured as a host in Chroma server.
    """
    def dehydrate_message(self, bundle):
        return nid_finder(bundle.obj.message)

    class Meta:
        queryset = Systemevents.objects.all()
        filtering = {
                'severity': ['exact'],
                'fromhost': ['exact', 'startswith'],
                'devicereportedtime': ['gte', 'lte'],
                'message': ['icontains', 'startswith'],
                }
        authorization = DjangoAuthorization()
        authentication = AnonymousAuthentication()
        ordering = ['devicereportedtime', 'fromhost']
        list_allowed_methods = ['get']
        detail_allowed_methods = ['get']

    def build_filters(self, filters = None):
        # FIXME: get the UI to give us ISO8601 so that we
        # can let tastypie use it without mangling
        # TODO: document the expected time format in this call
        UI_TIME_FORMAT = "%m/%d/%Y %H:%M "
        start = filters.get('devicereportedtime_gte', None)
        if start:
            filters['devicereportedtime_gte'] = datetime.datetime.strptime(str(start), UI_TIME_FORMAT)
        end = filters.get('devicereportedtime_lte', None)
        if end:
            filters['devicereportedtime_lte'] = datetime.datetime.strptime(str(end), UI_TIME_FORMAT)

        # TODO: let the UI filter on nodename to avoid the need for this mangling
        host_id = filters.get('host_id', None)
        if host_id:
            del filters['host_id']
            from chroma_core.models import ManagedHost
            host = ManagedHost.objects.get(id=host_id)
            filters['fromhost__startswith'] = host.pretty_name()

        return super(LogResource, self).build_filters(filters)


def nid_finder(message):
    from chroma_api import api_log

    from chroma_core.models import ManagedHost, ManagedTarget
    from chroma_core.lib.lustre_audit import normalize_nid
    import re
    # TODO: detect IB/other(cray?) as well as tcp
    nid_regex = re.compile("(\d{1,3}\.){3}\d{1,3}@tcp(_\d+)?")
    target_regex = re.compile("\\b(\\w+-(MDT|OST)\\d\\d\\d\\d)\\b")
    for match in nid_regex.finditer(message):
        replace = match.group()
        replace = normalize_nid(replace)
        try:
            host = ManagedHost.get_by_nid(replace)
        except ManagedHost.DoesNotExist:
            api_log.warn("No host has NID %s" % replace)
            continue
        except ManagedHost.MultipleObjectsReturned:
            api_log.warn("Multiple hosts have NID %s" % replace)<commit_msg>Remove no-longer-needed mangling of datetimes in log filtering<commit_after>#
# ==============================
# Copyright 2011 Whamcloud, Inc.
# ==============================

from chroma_core.models.log import Systemevents

from tastypie.resources import ModelResource
from tastypie.authorization import DjangoAuthorization
from chroma_api.authentication import AnonymousAuthentication


class LogResource(ModelResource):
    """
    syslog messages collected by Chroma server.  The fields in this table
    correspond to the default ``rsyslog`` MySQL output.

    You are probably mainly interested in ``fromhost``, ``devicereportedtime`` and
    ``message``.  Note that ``fromhost`` is a hostname rather than a reference to
    the ``host`` resource -- it is not guaranteed that a host mentioned in the
    syslog is configured as a host in Chroma server.
    """
    def dehydrate_message(self, bundle):
        return nid_finder(bundle.obj.message)

    class Meta:
        queryset = Systemevents.objects.all()
        filtering = {
                'severity': ['exact'],
                'fromhost': ['exact', 'startswith'],
                'devicereportedtime': ['gte', 'lte'],
                'message': ['icontains', 'startswith'],
                }
        authorization = DjangoAuthorization()
        authentication = AnonymousAuthentication()
        ordering = ['devicereportedtime', 'fromhost']
        list_allowed_methods = ['get']
        detail_allowed_methods = ['get']

    def build_filters(self, filters = None):
        # TODO: let the UI filter on nodename to avoid the need for this mangling
        host_id = filters.get('host_id', None)
        if host_id:
            del filters['host_id']
            from chroma_core.models import ManagedHost
            host = ManagedHost.objects.get(id=host_id)
            filters['fromhost__startswith'] = host.pretty_name()

        return super(LogResource, self).build_filters(filters)


def nid_finder(message):
    from chroma_api import api_log

    from chroma_core.models import ManagedHost, ManagedTarget
    from chroma_core.lib.lustre_audit import normalize_nid
    import re
    # TODO: detect IB/other(cray?) as well as tcp
    nid_regex = re.compile("(\d{1,3}\.){3}\d{1,3}@tcp(_\d+)?")
    target_regex = re.compile("\\b(\\w+-(MDT|OST)\\d\\d\\d\\d)\\b")
    for match in nid_regex.finditer(message):
        replace = match.group()
        replace = normalize_nid(replace)
        try:
            host = ManagedHost.get_by_nid(replace)
        except ManagedHost.DoesNotExist:
            api_log.warn("No host has NID %s" % replace)
            continue
        except ManagedHost.MultipleObjectsReturned:
            api_log.warn("Multiple hosts have NID %s" % replace)
{'fraction_non_alphanumeric': 0.12054161162483487, 'fraction_numerical': 0.003632760898282695, 'mean_word_length': 2.905222437137331, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '63128', 'n_tokens_mistral': 1789, 'n_tokens_neox': 1690, 'n_words': 470}