<commit_before>"""CA Observer

Copyright (C) 2014 <NAME>

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

See LICENSE for details.
"""

from __future__ import print_function

#from django.shortcuts import redirect
from django.http import HttpResponseRedirect
from django.template.response import TemplateResponse
from django.core.cache import cache

from django.views.decorators.http import last_modified
from django.views.decorators.cache import cache_page

from datetime import datetime
from bson import Code
from bson.tz_util import utc as _utc

from . import forms, generic

# HTTP conditional decorator based on collection last modification time
def coll_lastmod(coll, sk, tk=None):
    tk = tk or sk
    def cond(req, *args, **kws):
        C = req.mongodb[coll]
        O = C.find_one({}, {tk:1}, sort=[(sk,-1)])
        if O is not None:
            return O[tk]
    return cond

cavermap = Code("function(){emit(this.ver,1);}")
caverred = Code("function(K,V){return Array.sum(V);}")

def collect_versions(coll):
    key = 'caver_'+coll.name
    val = cache.get(key)
    if val is None:
        val = coll.inline_map_reduce(cavermap, caverred)
        for ent in val:
            ent['id'], ent['value'] = int(ent['_id']), int(ent['value'])
        cache.set(key, val, 60)
    return val

@cache_page(5)
def home(req):
    last = list(req.mongodb.servers.find().sort([('seenLast',-1)]).limit(1))
    last = datetime.utcnow().replace(tzinfo=_utc)-last[0]['seenLast'] if last else None

    num_hosts = cache.get('num_hosts')
    if num_hosts is None:
        num_hosts = len(req.mongodb.servers.distinct('source.ipv4'))
        cache.set('num_hosts', num_hosts, 60)

    C = {
        'num_servers':req.mongodb.servers.count(),
        'num_hosts':num_hosts,
        'num_searches':req.mongodb.searches.count(),
        'last_beacon':last,
        'client_versions':collect_versions(req.mongodb.searches),
        'server_versions':collect_versions(req.mongodb.servers),
    }
    return TemplateResponse(req, 'home.html', C)

beaconlog = generic.MongoFindListView.as_view(
    collection_name='events',
    template_name='beaconevent_list.html',
    def_search_key = 'source.host',
    search_keys = {'host':'source.host','port':'source.port'},
    def_sort = [('$natural',-1)],     
    base_query = {'type':'beacon'},
    extra_context = [
        lambda V:V.context.update({'form':forms.SearchFilter(V.request.GET)}),
        lambda V:V.context.update({'now':datetime.utcnow().replace(tzinfo=_utc)}),
    ]
)
beaconlog = last_modified(coll_lastmod('events','$natural','time'))(beaconlog)

beacons = generic.MongoFindListView.as_view(
    collection_name='servers',
    template_name='beacon_list.html',
    def_search_key = 'source.host',
    search_keys = {'host':'source.host','port':'source.port'},
    def_sort = [('seenLast',-1)],                
    extra_context = [
        lambda V:V.context.update({'form':forms.SearchFilter(V.request.GET)}),
        lambda V:V.context.update({'now':datetime.utcnow().replace(tzinfo=_utc)}),
    ]
)
beacons = last_modified(coll_lastmod('servers','seenLast'))(beacons)

searches = generic.MongoFindListView.as_view(
    collection_name='searches',
    template_name='search_list.html',
    def_search_key = 'pv',
    search_keys = {'host':'source.host','port':'source.port','pv':'pv'},
    def_sort = [('seenLast',-1)],                
    extra_context = [
        lambda V:V.context.update({'form':forms.SearchFilter(V.request.GET)}),
        lambda V:V.context.update({'now':datetime.utcnow().replace(tzinfo=_utc)}),
    ]
)
searches = last_modified(coll_lastmod('searches','seenLast'))(searches)

@cache_page(5)
def servers(req):
    C = {'object_list':req.mongodb.servers.distinct('source.host'),
    }
    return TemplateResponse(req, 'host_list.html', C)

@cache_page(5)
def clients(req):
    C = {'object_list':req.mongodb.searches.distinct('source.host'),
    }
    return TemplateResponse(req, 'host_list.html', C)

@cache_page(5)
def host_detail(req, name):<commit_msg>never_cache<commit_after>"""CA Observer

Copyright (C) 2014 <NAME>

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

See LICENSE for details.
"""

from __future__ import print_function

#from django.shortcuts import redirect
from django.http import HttpResponseRedirect
from django.template.response import TemplateResponse
from django.core.cache import cache

from django.views.decorators.http import last_modified
from django.views.decorators.cache import cache_page, never_cache

from datetime import datetime
from bson import Code
from bson.tz_util import utc as _utc

from . import forms, generic

# HTTP conditional decorator based on collection last modification time
def coll_lastmod(coll, sk, tk=None):
    tk = tk or sk
    def cond(req, *args, **kws):
        C = req.mongodb[coll]
        O = C.find_one({}, {tk:1}, sort=[(sk,-1)])
        if O is not None:
            return O[tk]
    return cond

cavermap = Code("function(){emit(this.ver,1);}")
caverred = Code("function(K,V){return Array.sum(V);}")

def collect_versions(coll):
    key = 'caver_'+coll.name
    val = cache.get(key)
    if val is None:
        val = coll.inline_map_reduce(cavermap, caverred)
        for ent in val:
            ent['id'], ent['value'] = int(ent['_id']), int(ent['value'])
        cache.set(key, val, 60)
    return val

@cache_page(5)
def home(req):
    last = list(req.mongodb.servers.find().sort([('seenLast',-1)]).limit(1))
    last = datetime.utcnow().replace(tzinfo=_utc)-last[0]['seenLast'] if last else None

    num_hosts = cache.get('num_hosts')
    if num_hosts is None:
        num_hosts = len(req.mongodb.servers.distinct('source.ipv4'))
        cache.set('num_hosts', num_hosts, 60)

    C = {
        'num_servers':req.mongodb.servers.count(),
        'num_hosts':num_hosts,
        'num_searches':req.mongodb.searches.count(),
        'last_beacon':last,
        'client_versions':collect_versions(req.mongodb.searches),
        'server_versions':collect_versions(req.mongodb.servers),
    }
    return TemplateResponse(req, 'home.html', C)

beaconlog = generic.MongoFindListView.as_view(
    collection_name='events',
    template_name='beaconevent_list.html',
    def_search_key = 'source.host',
    search_keys = {'host':'source.host','port':'source.port'},
    def_sort = [('$natural',-1)],     
    base_query = {'type':'beacon'},
    extra_context = [
        lambda V:V.context.update({'form':forms.SearchFilter(V.request.GET)}),
        lambda V:V.context.update({'now':datetime.utcnow().replace(tzinfo=_utc)}),
    ]
)
beaconlog = never_cache(last_modified(coll_lastmod('events','$natural','time'))(beaconlog))

beacons = generic.MongoFindListView.as_view(
    collection_name='servers',
    template_name='beacon_list.html',
    def_search_key = 'source.host',
    search_keys = {'host':'source.host','port':'source.port'},
    def_sort = [('seenLast',-1)],                
    extra_context = [
        lambda V:V.context.update({'form':forms.SearchFilter(V.request.GET)}),
        lambda V:V.context.update({'now':datetime.utcnow().replace(tzinfo=_utc)}),
    ]
)
beacons = never_cache(last_modified(coll_lastmod('servers','seenLast'))(beacons))

searches = generic.MongoFindListView.as_view(
    collection_name='searches',
    template_name='search_list.html',
    def_search_key = 'pv',
    search_keys = {'host':'source.host','port':'source.port','pv':'pv'},
    def_sort = [('seenLast',-1)],                
    extra_context = [
        lambda V:V.context.update({'form':forms.SearchFilter(V.request.GET)}),
        lambda V:V.context.update({'now':datetime.utcnow().replace(tzinfo=_utc)}),
    ]
)
searches = never_cache(last_modified(coll_lastmod('searches','seenLast'))(searches))

@cache_page(5)
def servers(req):
    C = {'object_list':req.mongodb.servers.distinct('source.host'),
    }
    return TemplateResponse(req, 'host_list.html', C)

@cache_page(5)
def clients(req):
    C = {'object_list':req.mongodb.searches.distinct('source.host'),
    }
    return TemplateResponse(req, 'host_list.html', C)

@cache_page(5)
def host_detail(req, name):
{'fraction_non_alphanumeric': 0.151056338028169, 'fraction_numerical': 0.00539906103286385, 'mean_word_length': 4.059976247030879, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6849154', 'n_tokens_mistral': 2887, 'n_tokens_neox': 2824, 'n_words': 546}