<commit_before>import json
import math
import random
import re

from django.conf import settings
from django.contrib.humanize.templatetags import humanize
from django.core.cache import cache
from django.db.models import Q
from django.http import Http404, HttpResponse
from django.shortcuts import redirect, get_object_or_404
from django.utils import timezone
from django.utils.functional import cached_property
from django.utils.translation import ugettext as _
from django.views.generic import ListView, TemplateView, View
from django.views.decorators.cache import patch_cache_control
import pytz

from botbot.apps.bots.utils import reverse_channel
from botbot.apps.bots.views import ChannelMixin
from . import forms
from botbot.apps.logs.models import Log
from botbot.apps.kudos.models import KudosTotal



class Help(ChannelMixin, TemplateView):

    """
    Help page for a channel.
    """
    template_name = 'logs/help.html'


class PaginatorPageLinksMixin(object):

    def paginate_queryset(self, queryset, page_size):
        paginator, page, object_list, has_other_pages = super(
            PaginatorPageLinksMixin, self).paginate_queryset(
                queryset, page_size)

        self.next_page = self.get_next_page_link(page)
        self.prev_page = self.get_previous_page_link(page)
        self.current_page = self.get_current_page_link(page)

        return paginator, page, object_list, has_other_pages

    def get_next_page_link(self, page):
        url = self.request.path
        params = self.request.GET.copy()

        if not page.has_next():
            return ""
        else:
            params['page'] = page.next_page_number()

        return '{0}?{1}'.format(url, params.urlencode())

    def get_previous_page_link(self, page):
        url = self.request.path
        params = self.request.GET.copy()

        if not page.has_previous():
            return ""
        else:
            params['page'] = page.previous_page_number()

        return '{0}?{1}'.format(url, params.urlencode())

    def get_current_page_link(self, page):
        url = self.request.path
        params = self.request.GET.copy()
        params['page'] = page.number
        return '{0}?{1}'.format(url, params.urlencode())


class LogDateMixin(object):

    def _get_base_queryset(self):
        return self.channel.filtered_logs()

    def channel_date_url(self, date=None):
        if not date:
            date = self.date
        viewname = self.format == 'text' and 'log_day_text' or 'log_day'
        return reverse_channel(
            self.channel, viewname, kwargs=self._kwargs_with_date(date))

    def _kwargs_with_date(self, date):
        kwargs = {
            'year': date.year,
            'month': "%02d" % date.month,
            'day': "%02d" % date.day
        }
        return kwargs

    def _local_date_at_midnight(self, timestamp):
        # cast timestamp into local timezone
        localized = timestamp.astimezone(self.request_timezone)
        # create a new date object starting at midnight in that timezone
        return timezone.datetime(localized.year,
                                 localized.month,
                                 localized.day,
                                 tzinfo=localized.tzinfo)

    def _get_previous_date(self):
        """
        Find the previous day, that has content.
        """
        date = None
        try:
            ts = (self._get_base_queryset()
                      .filter(timestamp__lt=self.date)[0].timestamp)
            date = self._local_date_at_midnight(ts)
        except IndexError:
            pass
        return date

    def _get_next_date(self):
        """
        Find the next day, that has content.
        """
        date = None
        try:
            ts = (self._get_base_queryset()
                .filter(timestamp__gte=timezone.timedelta(days=1) + self.date)
                .order_by('timestamp')[0].timestamp)
            date = self._local_date_at_midnight(ts)
        except IndexError:
            pass
        return date

    def _date_query_set(self, date):
        qs = self._get_base_queryset()
        return qs.filter(timestamp__gte=date,
                         timestamp__lt=date + timezone.timedelta(days=1))

class LogStream(ChannelMixin, View):
    def get(self, request, channel_slug, bot_slug):
        response = HttpResponse()
        response['X-Accel-Redirect'] = '/internal-channel-stream/{}'.format(
            self.channel.pk)
        if 'HTTP_LAST_EVENT_ID' in request.META:
            response['Last-Event-ID'] = request.META['HTTP_LAST_EVENT_ID']
        return response

def _find_pk(pk, queryset):
    """Find a PK in a queryset in memory"""
    found = None
    try:
        pk = int(pk)
        found = next(obj for obj in queryset if obj.pk == pk)
    except (ValueError, StopIteration):
        pass
    return found

def _timeline_context(timeline):
    """
    Context (template) vars needed for timeline display.
    """

    if not timeline:
        return {}

    today = timezone.now().date()
    last_monday = today - timezone.timedelta(days=today.weekday())
    last_week = last_monday - timezone.timedelta(days=7)

    # the last month in the timeline needs special treatment so it
    # doesn't get ordered ahead of the last/current weeks
    last_month = timeline[timeline.keyOrder[-1]].pop()
    if last_month >= last_week:
        last_month_adjusted = (last_week -
                               timezone.timedelta(days=1))
    elif last_month >= last_monday:
        last_month_adjusted = (last_monday -
                               timezone.timedelta(days=1))
    else:
        last_month_adjusted = last_month

    result = {
        'timeline': timeline,
        'this_week': last_monday,
        'last_week': last_week,
        'last_month': {'real': last_month,
                       'adjusted': last_month_adjusted},
    }
    return result

class LogViewer(ChannelMixin, object):
    context_object_name = "message_list"
    newest_first = False
    show_first_header = False   # Display date header above first line
    paginate_by = 150
    format = ''

    def __init__(self, *args, **kwargs):
        super(LogViewer, self).__init__(*args, **kwargs)
        self.next_page = ""
        self.prev_page = ""
        self.current_page = ""

    def dispatch(self, request, *args, **kwargs):
        self._setup_response_format()
        return super(LogViewer, self).dispatch(request, *args, **kwargs)

    def _setup_response_format(self):
        if self.format == 'text':
            self.include_timeline = False
            self.template_name = 'logs/logs.txt'
            self.content_type = 'text/plain; charset=utf-8'
        elif self.request.is_ajax():
            self.format = 'ajax'
            self.include_timeline = False
            self.template_name = 'logs/log_display.html'
        # Default to HTML view
        else:
            self.format = 'html'
            self.include_timeline = True
            self.template_name = "logs/logs.html"


    def get_ordered_queryset(self, queryset):
        order = 'timestamp'
        if self.newest_first:
            order = '-timestamp'

        return queryset.order_by(order)



    def get_context_data(self, **kwargs):
        context = super(LogViewer, self).get_context_data(**kwargs)

        if self.include_timeline:
            context.update(
                _timeline_context(self.channel.get_months_active()))

        if self.format == 'html':
            context.update({
                'is_current': getattr(self, 'is_current', False),
                'search_form': forms.SearchForm(),
                'show_first_header': self.show_first_header,
                'newest_first': self.newest_first,
                'show_kudos': self.channel.user_can_access_kudos(
                    self.request.user),
            })

        size = self.channel.current_size()
        context.update({
            'size': size,
            'big': (size >= settings.BIG_CHANNEL),
            'prev_page': self.prev_page,
            'next_page': self.next_page,
            'current_page': self.current_page,
        })

        return context



    def render_to_response(self, context, **response_kwargs):
        response = super(LogViewer, self).render_to_response(
            context, **response_kwargs)

        has_next_page = False
        if self.format == 'html':
            # Official SEO header
            links = []
            if self.next_page:
                links.append('{0}; rel="next"'.format(self.next_page))
                has_next_page = True

            if self.prev_page:
                links.append('{0}; rel="prev"'.format(self.prev_page))
            response['Link'] = ','.join(links)
        else:
            # No HTML, pass page info in easily parseable headers
            if self.next_page:
                response['X-NextPage'] = self.next_page
                has_next_page = True

            if self.prev_page:
                response['X-PrevPage'] = self.prev_page

        if has_next_page and self.request.user.is_anonymous():
            patch_cache_control(
                response, public=True,
                max_age=settings.CACHE_MIDDLEWARE_SECONDS)
        else:
            patch_cache_control(response, private=True)
        return response

    def _pages_for_queryset(self, queryset):
        return int(math.ceil(queryset.count() / float(self.paginate_by)))


class DayLogViewer(PaginatorPageLinksMixin, LogDateMixin, LogViewer, ListView):
    show_first_header = False
    allow_empty = True

    def get(self, request, *args, **kwargs):
        self.date = self.set_view_date()
        self.object_list = self.get_queryset()

        # Redirect to nearby logs if this queryset is empty to avoid a 404
        if not self.get_allow_empty() and not self.object_list.exists():
            url = self._nearby_log_url()
            if url:
                return redirect(url)
            raise Http404(_("Empty list and '%(class_name)s.allow_empty' is False.")
                          % {'class_name': self.__class__.__name__})

        context = self.get_context_data()
        return self.render_to_response(context)

    def _nearby_log_url(self):
        """Find a date-based log URL that will not be empty"""
        # First check if there is anything in the past
        closet_qs = self.channel.filtered_logs().order_by(
            "-timestamp").filter(timestamp__lte=self.date)

        # If not go to the future
        if not closet_qs.exists():
            closet_qs = self.channel.filtered_logs().order_by(
                "timestamp").filter(
                timestamp__gte=self.date)

        # Return the URL where the first log line found will be
        try:
            return self.channel_date_url(closet_qs[0].timestamp)
        except IndexError:
            pass
        return None


    def get_context_data(self):
        context = super(DayLogViewer, self).get_context_data()
        try:
            context.update({
                'highlight': int(self.request.GET.get('msg')),
            })
        except ValueError:
            pass
        return context


    def get_queryset(self):
        qs = self.channel.filtered_logs()
        qs = self.get_ordered_queryset(qs)
        start = self.date
        end = start + timezone.timedelta(days=1)
        return qs.filter(timestamp__gte=start, timestamp__lt=end)<commit_msg>Check for both exceptions... woops!<commit_after>import json
import math
import random
import re

from django.conf import settings
from django.contrib.humanize.templatetags import humanize
from django.core.cache import cache
from django.db.models import Q
from django.http import Http404, HttpResponse
from django.shortcuts import redirect, get_object_or_404
from django.utils import timezone
from django.utils.functional import cached_property
from django.utils.translation import ugettext as _
from django.views.generic import ListView, TemplateView, View
from django.views.decorators.cache import patch_cache_control
import pytz

from botbot.apps.bots.utils import reverse_channel
from botbot.apps.bots.views import ChannelMixin
from . import forms
from botbot.apps.logs.models import Log
from botbot.apps.kudos.models import KudosTotal



class Help(ChannelMixin, TemplateView):

    """
    Help page for a channel.
    """
    template_name = 'logs/help.html'


class PaginatorPageLinksMixin(object):

    def paginate_queryset(self, queryset, page_size):
        paginator, page, object_list, has_other_pages = super(
            PaginatorPageLinksMixin, self).paginate_queryset(
                queryset, page_size)

        self.next_page = self.get_next_page_link(page)
        self.prev_page = self.get_previous_page_link(page)
        self.current_page = self.get_current_page_link(page)

        return paginator, page, object_list, has_other_pages

    def get_next_page_link(self, page):
        url = self.request.path
        params = self.request.GET.copy()

        if not page.has_next():
            return ""
        else:
            params['page'] = page.next_page_number()

        return '{0}?{1}'.format(url, params.urlencode())

    def get_previous_page_link(self, page):
        url = self.request.path
        params = self.request.GET.copy()

        if not page.has_previous():
            return ""
        else:
            params['page'] = page.previous_page_number()

        return '{0}?{1}'.format(url, params.urlencode())

    def get_current_page_link(self, page):
        url = self.request.path
        params = self.request.GET.copy()
        params['page'] = page.number
        return '{0}?{1}'.format(url, params.urlencode())


class LogDateMixin(object):

    def _get_base_queryset(self):
        return self.channel.filtered_logs()

    def channel_date_url(self, date=None):
        if not date:
            date = self.date
        viewname = self.format == 'text' and 'log_day_text' or 'log_day'
        return reverse_channel(
            self.channel, viewname, kwargs=self._kwargs_with_date(date))

    def _kwargs_with_date(self, date):
        kwargs = {
            'year': date.year,
            'month': "%02d" % date.month,
            'day': "%02d" % date.day
        }
        return kwargs

    def _local_date_at_midnight(self, timestamp):
        # cast timestamp into local timezone
        localized = timestamp.astimezone(self.request_timezone)
        # create a new date object starting at midnight in that timezone
        return timezone.datetime(localized.year,
                                 localized.month,
                                 localized.day,
                                 tzinfo=localized.tzinfo)

    def _get_previous_date(self):
        """
        Find the previous day, that has content.
        """
        date = None
        try:
            ts = (self._get_base_queryset()
                      .filter(timestamp__lt=self.date)[0].timestamp)
            date = self._local_date_at_midnight(ts)
        except IndexError:
            pass
        return date

    def _get_next_date(self):
        """
        Find the next day, that has content.
        """
        date = None
        try:
            ts = (self._get_base_queryset()
                .filter(timestamp__gte=timezone.timedelta(days=1) + self.date)
                .order_by('timestamp')[0].timestamp)
            date = self._local_date_at_midnight(ts)
        except IndexError:
            pass
        return date

    def _date_query_set(self, date):
        qs = self._get_base_queryset()
        return qs.filter(timestamp__gte=date,
                         timestamp__lt=date + timezone.timedelta(days=1))

class LogStream(ChannelMixin, View):
    def get(self, request, channel_slug, bot_slug):
        response = HttpResponse()
        response['X-Accel-Redirect'] = '/internal-channel-stream/{}'.format(
            self.channel.pk)
        if 'HTTP_LAST_EVENT_ID' in request.META:
            response['Last-Event-ID'] = request.META['HTTP_LAST_EVENT_ID']
        return response

def _find_pk(pk, queryset):
    """Find a PK in a queryset in memory"""
    found = None
    try:
        pk = int(pk)
        found = next(obj for obj in queryset if obj.pk == pk)
    except (ValueError, StopIteration):
        pass
    return found

def _timeline_context(timeline):
    """
    Context (template) vars needed for timeline display.
    """

    if not timeline:
        return {}

    today = timezone.now().date()
    last_monday = today - timezone.timedelta(days=today.weekday())
    last_week = last_monday - timezone.timedelta(days=7)

    # the last month in the timeline needs special treatment so it
    # doesn't get ordered ahead of the last/current weeks
    last_month = timeline[timeline.keyOrder[-1]].pop()
    if last_month >= last_week:
        last_month_adjusted = (last_week -
                               timezone.timedelta(days=1))
    elif last_month >= last_monday:
        last_month_adjusted = (last_monday -
                               timezone.timedelta(days=1))
    else:
        last_month_adjusted = last_month

    result = {
        'timeline': timeline,
        'this_week': last_monday,
        'last_week': last_week,
        'last_month': {'real': last_month,
                       'adjusted': last_month_adjusted},
    }
    return result

class LogViewer(ChannelMixin, object):
    context_object_name = "message_list"
    newest_first = False
    show_first_header = False   # Display date header above first line
    paginate_by = 150
    format = ''

    def __init__(self, *args, **kwargs):
        super(LogViewer, self).__init__(*args, **kwargs)
        self.next_page = ""
        self.prev_page = ""
        self.current_page = ""

    def dispatch(self, request, *args, **kwargs):
        self._setup_response_format()
        return super(LogViewer, self).dispatch(request, *args, **kwargs)

    def _setup_response_format(self):
        if self.format == 'text':
            self.include_timeline = False
            self.template_name = 'logs/logs.txt'
            self.content_type = 'text/plain; charset=utf-8'
        elif self.request.is_ajax():
            self.format = 'ajax'
            self.include_timeline = False
            self.template_name = 'logs/log_display.html'
        # Default to HTML view
        else:
            self.format = 'html'
            self.include_timeline = True
            self.template_name = "logs/logs.html"


    def get_ordered_queryset(self, queryset):
        order = 'timestamp'
        if self.newest_first:
            order = '-timestamp'

        return queryset.order_by(order)



    def get_context_data(self, **kwargs):
        context = super(LogViewer, self).get_context_data(**kwargs)

        if self.include_timeline:
            context.update(
                _timeline_context(self.channel.get_months_active()))

        if self.format == 'html':
            context.update({
                'is_current': getattr(self, 'is_current', False),
                'search_form': forms.SearchForm(),
                'show_first_header': self.show_first_header,
                'newest_first': self.newest_first,
                'show_kudos': self.channel.user_can_access_kudos(
                    self.request.user),
            })

        size = self.channel.current_size()
        context.update({
            'size': size,
            'big': (size >= settings.BIG_CHANNEL),
            'prev_page': self.prev_page,
            'next_page': self.next_page,
            'current_page': self.current_page,
        })

        return context



    def render_to_response(self, context, **response_kwargs):
        response = super(LogViewer, self).render_to_response(
            context, **response_kwargs)

        has_next_page = False
        if self.format == 'html':
            # Official SEO header
            links = []
            if self.next_page:
                links.append('{0}; rel="next"'.format(self.next_page))
                has_next_page = True

            if self.prev_page:
                links.append('{0}; rel="prev"'.format(self.prev_page))
            response['Link'] = ','.join(links)
        else:
            # No HTML, pass page info in easily parseable headers
            if self.next_page:
                response['X-NextPage'] = self.next_page
                has_next_page = True

            if self.prev_page:
                response['X-PrevPage'] = self.prev_page

        if has_next_page and self.request.user.is_anonymous():
            patch_cache_control(
                response, public=True,
                max_age=settings.CACHE_MIDDLEWARE_SECONDS)
        else:
            patch_cache_control(response, private=True)
        return response

    def _pages_for_queryset(self, queryset):
        return int(math.ceil(queryset.count() / float(self.paginate_by)))


class DayLogViewer(PaginatorPageLinksMixin, LogDateMixin, LogViewer, ListView):
    show_first_header = False
    allow_empty = True

    def get(self, request, *args, **kwargs):
        self.date = self.set_view_date()
        self.object_list = self.get_queryset()

        # Redirect to nearby logs if this queryset is empty to avoid a 404
        if not self.get_allow_empty() and not self.object_list.exists():
            url = self._nearby_log_url()
            if url:
                return redirect(url)
            raise Http404(_("Empty list and '%(class_name)s.allow_empty' is False.")
                          % {'class_name': self.__class__.__name__})

        context = self.get_context_data()
        return self.render_to_response(context)

    def _nearby_log_url(self):
        """Find a date-based log URL that will not be empty"""
        # First check if there is anything in the past
        closet_qs = self.channel.filtered_logs().order_by(
            "-timestamp").filter(timestamp__lte=self.date)

        # If not go to the future
        if not closet_qs.exists():
            closet_qs = self.channel.filtered_logs().order_by(
                "timestamp").filter(
                timestamp__gte=self.date)

        # Return the URL where the first log line found will be
        try:
            return self.channel_date_url(closet_qs[0].timestamp)
        except IndexError:
            pass
        return None


    def get_context_data(self):
        context = super(DayLogViewer, self).get_context_data()
        try:
            context.update({
                'highlight': int(self.request.GET.get('msg')),
            })
        except (TypeError, ValueError):
            pass
        return context


    def get_queryset(self):
        qs = self.channel.filtered_logs()
        qs = self.get_ordered_queryset(qs)
        start = self.date
        end = start + timezone.timedelta(days=1)
        return qs.filter(timestamp__gte=start, timestamp__lt=end)
{'fraction_non_alphanumeric': 0.09875201113188677, 'fraction_numerical': 0.0033047788842022873, 'mean_word_length': 2.1804729636288203, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3187333', 'n_tokens_mistral': 6802, 'n_tokens_neox': 6562, 'n_words': 1478}