<commit_before># -*- coding: utf-8 -*-

import mimetypes
import os

try:
    import json
except ImportError:
    import simplejson as json

import django.contrib.auth.views
from django.conf import settings
from django.contrib.auth import REDIRECT_FIELD_NAME, get_user_model
from django.core.exceptions import ImproperlyConfigured
from django.core.urlresolvers import reverse
from django.http import HttpResponse, StreamingHttpResponse, HttpResponseForbidden
from django.shortcuts import render_to_response, get_object_or_404
from django.template import RequestContext
from django.utils.translation import ugettext_lazy as _
from django.views.generic import RedirectView

from kobo.hub.models import Arch, Channel, Task
from kobo.hub.forms import TaskSearchForm
from kobo.django.views.generic import ExtraDetailView, SearchView

# max log size returned in HTML-embedded view
HTML_LOG_MAX_SIZE = getattr(settings, "HTML_LOG_MAX_SIZE", (1024 ** 2) * 2)

# max log size returned in a JSON request
JSON_LOG_MAX_SIZE = getattr(settings, "JSON_LOG_MAX_SIZE", (1024 ** 2) * 8)

# default LogWatcher JS poll interval
LOG_WATCHER_INTERVAL = getattr(settings, "LOG_WATCHER_INTERVAL", 5000)


class UserDetailView(ExtraDetailView):
    model = get_user_model()
    title = _("User detail")
    template_name = "user/detail.html"
    context_object_name = "usr"

    def get_context_data(self, **kwargs):
        context = super(UserDetailView, self).get_context_data(**kwargs)
        context['tasks'] = kwargs['object'].task_set.count()
        return context

class DetailViewWithWorkers(ExtraDetailView):
    model = Channel
    template_name = "channel/detail.html"
    context_object_name = "channel"
    title = _("Architecture detail")

    def get_context_data(self, **kwargs):
        context = super(DetailViewWithWorkers, self).get_context_data(**kwargs)
        context["worker_list"] = kwargs["object"].worker_set.order_by("name")
        return context

class ArchDetailView(ExtraDetailView):
    model = Arch
    template_name = "arch/detail.html"
    context_object_name = "arch"
    title = _("Architecture detail")

    def get_context_data(self, **kwargs):
        context = super(ArchDetailView, self).get_context_data(**kwargs)
        context["worker_list"] = kwargs["object"].worker_set.order_by("name")
        return context

class TaskListView(SearchView):
    # TODO: missing kwargs custom queries for backward compatibility
    title = _("All tasks")
    model = Task
    form_class = TaskSearchForm
    template_name = "task/list.html"
    context_object_name = "task_list"
    state = None
    order_by = ['-id']

    def get_form_kwargs(self):
        kwargs = super(TaskListView, self).get_form_kwargs()
        kwargs['state'] = self.state
        kwargs['order_by'] = self.order_by
        return kwargs


class TaskDetail(ExtraDetailView):
    queryset = Task.objects.select_related()
    context_object_name = "task"
    template_name = "task/detail.html"
    title = _("Task detail")

    def get_context_data(self, **kwargs):
        context = super(TaskDetail, self).get_context_data(**kwargs)
        logs = []
        for i in kwargs['object'].logs.list:
            if self.request.user.has_perm('hub.can_see_traceback'):
                logs.append(i)
                continue
            if not os.path.basename(i).startswith("traceback"):
                logs.append(i)
        logs.sort()
        context["logs"] = logs
        context['task_list'] = kwargs['object'].subtasks()
        return context


def _stream_file(file_path, offset=0):
    """Generator that returns 1M file chunks."""
    try:
        f = open(file_path, "r")
    except IOError:
        return

    f.seek(offset)
    while 1:
        data = f.read(1024 ** 2)
        if not data:
            break
        yield data
    f.close()


def _trim_log(text):
    # break at first line if possible
    nl = text.find('\n')
    if nl > 0:
        subtext = text[nl:]
    else:
        subtext = '\n' + text
    return '<...trimmed, download required for full log>' + subtext


def task_log(request, id, log_name):
    """
    IMPORTANT: reverse to 'task/log-json' *must* exist
    """
    if os.path.basename(log_name).startswith("traceback") and not request.user.has_perm('hub.can_see_traceback'):
        return HttpResponseForbidden("You don't have permission to see the traceback.")

    task = get_object_or_404(Task, id=id)

    file_path = task.logs._get_absolute_log_path(log_name)
    if not os.path.isfile(file_path) and not file_path.endswith(".gz"):
        file_path = task.logs._get_absolute_log_path(log_name + ".gz")

    mimetype = mimetypes.guess_type(file_path)[0] or 'application/octet-stream'
    offset = int(request.GET.get("offset", 0))
    try:
        content_len = os.path.getsize(file_path) - offset
    except OSError:
        content_len = 0

    if request.GET.get("format") == "raw":
        # use _stream_file() instad of passing file object in order to improve performance
        response = StreamingHttpResponse(_stream_file(file_path, offset), content_type=mimetype)

        response["Content-Length"] = content_len
        # set filename to be real filesystem name
        response['Content-Disposition'] = 'attachment; filename=%s' % os.path.basename(file_path)
        return response

    if log_name.endswith(".html") or log_name.endswith(".htm"):
        # use _stream_file() instad of passing file object in order to improve performance
        response = StreamingHttpResponse(_stream_file(file_path, offset), content_type=mimetype)
        response["Content-Length"] = content_len
        return response

    exts = getattr(settings, "VALID_TASK_LOG_EXTENSIONS", [".log"])
    found = False
    for ext in exts:
        if log_name.endswith(ext):
            found = True
    if not found:
        return HttpResponseForbidden("Can display only specific file types: %s" % ", ".join(exts))

    content = task.logs.get_chunk(log_name, offset, HTML_LOG_MAX_SIZE)
    needs_trim = content_len != 0 and len(content) < content_len
    content = content.decode("utf-8", "replace")
    if needs_trim:
        content = _trim_log(content)

    context = {
        "title": "Task log",
        "offset": offset + content_len + 1,
        "task_finished": task.is_finished() and 1 or 0,
        "next_poll": None if task.is_finished() else LOG_WATCHER_INTERVAL,
        "content": content,
        "log_name": log_name,
        "task": task,<commit_msg>Fix off-by-one error in view offset<commit_after># -*- coding: utf-8 -*-

import mimetypes
import os

try:
    import json
except ImportError:
    import simplejson as json

import django.contrib.auth.views
from django.conf import settings
from django.contrib.auth import REDIRECT_FIELD_NAME, get_user_model
from django.core.exceptions import ImproperlyConfigured
from django.core.urlresolvers import reverse
from django.http import HttpResponse, StreamingHttpResponse, HttpResponseForbidden
from django.shortcuts import render_to_response, get_object_or_404
from django.template import RequestContext
from django.utils.translation import ugettext_lazy as _
from django.views.generic import RedirectView

from kobo.hub.models import Arch, Channel, Task
from kobo.hub.forms import TaskSearchForm
from kobo.django.views.generic import ExtraDetailView, SearchView

# max log size returned in HTML-embedded view
HTML_LOG_MAX_SIZE = getattr(settings, "HTML_LOG_MAX_SIZE", (1024 ** 2) * 2)

# max log size returned in a JSON request
JSON_LOG_MAX_SIZE = getattr(settings, "JSON_LOG_MAX_SIZE", (1024 ** 2) * 8)

# default LogWatcher JS poll interval
LOG_WATCHER_INTERVAL = getattr(settings, "LOG_WATCHER_INTERVAL", 5000)


class UserDetailView(ExtraDetailView):
    model = get_user_model()
    title = _("User detail")
    template_name = "user/detail.html"
    context_object_name = "usr"

    def get_context_data(self, **kwargs):
        context = super(UserDetailView, self).get_context_data(**kwargs)
        context['tasks'] = kwargs['object'].task_set.count()
        return context

class DetailViewWithWorkers(ExtraDetailView):
    model = Channel
    template_name = "channel/detail.html"
    context_object_name = "channel"
    title = _("Architecture detail")

    def get_context_data(self, **kwargs):
        context = super(DetailViewWithWorkers, self).get_context_data(**kwargs)
        context["worker_list"] = kwargs["object"].worker_set.order_by("name")
        return context

class ArchDetailView(ExtraDetailView):
    model = Arch
    template_name = "arch/detail.html"
    context_object_name = "arch"
    title = _("Architecture detail")

    def get_context_data(self, **kwargs):
        context = super(ArchDetailView, self).get_context_data(**kwargs)
        context["worker_list"] = kwargs["object"].worker_set.order_by("name")
        return context

class TaskListView(SearchView):
    # TODO: missing kwargs custom queries for backward compatibility
    title = _("All tasks")
    model = Task
    form_class = TaskSearchForm
    template_name = "task/list.html"
    context_object_name = "task_list"
    state = None
    order_by = ['-id']

    def get_form_kwargs(self):
        kwargs = super(TaskListView, self).get_form_kwargs()
        kwargs['state'] = self.state
        kwargs['order_by'] = self.order_by
        return kwargs


class TaskDetail(ExtraDetailView):
    queryset = Task.objects.select_related()
    context_object_name = "task"
    template_name = "task/detail.html"
    title = _("Task detail")

    def get_context_data(self, **kwargs):
        context = super(TaskDetail, self).get_context_data(**kwargs)
        logs = []
        for i in kwargs['object'].logs.list:
            if self.request.user.has_perm('hub.can_see_traceback'):
                logs.append(i)
                continue
            if not os.path.basename(i).startswith("traceback"):
                logs.append(i)
        logs.sort()
        context["logs"] = logs
        context['task_list'] = kwargs['object'].subtasks()
        return context


def _stream_file(file_path, offset=0):
    """Generator that returns 1M file chunks."""
    try:
        f = open(file_path, "r")
    except IOError:
        return

    f.seek(offset)
    while 1:
        data = f.read(1024 ** 2)
        if not data:
            break
        yield data
    f.close()


def _trim_log(text):
    # break at first line if possible
    nl = text.find('\n')
    if nl > 0:
        subtext = text[nl:]
    else:
        subtext = '\n' + text
    return '<...trimmed, download required for full log>' + subtext


def task_log(request, id, log_name):
    """
    IMPORTANT: reverse to 'task/log-json' *must* exist
    """
    if os.path.basename(log_name).startswith("traceback") and not request.user.has_perm('hub.can_see_traceback'):
        return HttpResponseForbidden("You don't have permission to see the traceback.")

    task = get_object_or_404(Task, id=id)

    file_path = task.logs._get_absolute_log_path(log_name)
    if not os.path.isfile(file_path) and not file_path.endswith(".gz"):
        file_path = task.logs._get_absolute_log_path(log_name + ".gz")

    mimetype = mimetypes.guess_type(file_path)[0] or 'application/octet-stream'
    offset = int(request.GET.get("offset", 0))
    try:
        content_len = os.path.getsize(file_path) - offset
    except OSError:
        content_len = 0

    if request.GET.get("format") == "raw":
        # use _stream_file() instad of passing file object in order to improve performance
        response = StreamingHttpResponse(_stream_file(file_path, offset), content_type=mimetype)

        response["Content-Length"] = content_len
        # set filename to be real filesystem name
        response['Content-Disposition'] = 'attachment; filename=%s' % os.path.basename(file_path)
        return response

    if log_name.endswith(".html") or log_name.endswith(".htm"):
        # use _stream_file() instad of passing file object in order to improve performance
        response = StreamingHttpResponse(_stream_file(file_path, offset), content_type=mimetype)
        response["Content-Length"] = content_len
        return response

    exts = getattr(settings, "VALID_TASK_LOG_EXTENSIONS", [".log"])
    found = False
    for ext in exts:
        if log_name.endswith(ext):
            found = True
    if not found:
        return HttpResponseForbidden("Can display only specific file types: %s" % ", ".join(exts))

    content = task.logs.get_chunk(log_name, offset, HTML_LOG_MAX_SIZE)
    needs_trim = content_len != 0 and len(content) < content_len
    content = content.decode("utf-8", "replace")
    if needs_trim:
        content = _trim_log(content)

    context = {
        "title": "Task log",
        "offset": offset + content_len,
        "task_finished": task.is_finished() and 1 or 0,
        "next_poll": None if task.is_finished() else LOG_WATCHER_INTERVAL,
        "content": content,
        "log_name": log_name,
        "task": task,
{'fraction_non_alphanumeric': 0.10802706443180071, 'fraction_numerical': 0.006074119637090573, 'mean_word_length': 3.332778147901399, 'pattern_counts': {'":': 16, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3258845', 'n_tokens_mistral': 4037, 'n_tokens_neox': 3911, 'n_words': 967}