<commit_before>import os
from types import MethodType
import tempfile
import gzip
import tarfile
import StringIO

from celery import result as celeryresult
from celery.contrib.abortable import AbortableAsyncResult
from datetime import datetime
from django import forms
from django.contrib.auth.decorators import login_required
from django.contrib import messages
from django.core import serializers
from django.core.paginator import Paginator, InvalidPage, EmptyPage
from django.core.serializers.json import DjangoJSONEncoder
from django.core.servers.basehttp import FileWrapper
from django.conf import settings
from django.db import transaction
from django.db.models import Q, Count
from django.http import HttpResponse, HttpResponseRedirect, \
        HttpResponseServerError
from django.shortcuts import render_to_response, get_object_or_404
from django.template import RequestContext
from django.utils import simplejson
from django.utils.encoding import smart_str, smart_unicode
from ocradmin.batch import utils as batchutils
from ocradmin.ocr import tasks
from ocradmin.ocr import utils as ocrutils
from ocradmin.ocr.utils import saves_files
from ocradmin.ocrpresets.models import OcrPreset
from ocradmin.ocrtasks.models import OcrTask, OcrBatch, Transcript
from ocradmin.projects.tasks import IngestTask
from ocradmin.training.tasks import ComparisonTask

from ocradmin.projects.utils import project_required
from ocradmin.ocr.views import _get_best_params, _cleanup_params, \
        _handle_request, AppException
from ocradmin.ocr.tools.manager import PluginManager


PER_PAGE = 25


def batch_query(params):
    """
    Query the batch db.
    """
    order = [x for x in params.getlist("order_by") if x != ""] \
            or ["created_on"]

    query = Q()
    for key, val in params.items():
        if key.find("__") == -1 and \
                not key in OcrBatch._meta.get_all_field_names():
            continue
        if not val:
            continue
        query = query & Q(**{str(key): str(val)})
    query = OcrBatch.objects.select_related().filter(query)
    if order and order[0].replace("-", "", 1) == "task_count":
        query = query.annotate(task_count=Count("tasks"))
    return query.order_by(*order)


class OcrBatchForm(forms.ModelForm):
    """
        New project form.
    """
    def __init__(self, *args, **kwargs):
        super(forms.ModelForm, self).__init__(*args, **kwargs)

        # change a widget attribute:
        self.fields['description'].widget.attrs["rows"] = 2
        self.fields['description'].widget.attrs["cols"] = 40

    class Meta:
        model = OcrBatch
        exclude = ["user", "created_on", "project", "task_type"]


@login_required
@project_required
def new(request):
    """
    Present a new batch form.
    """
    template = "batch/new.html" if not request.is_ajax() \
        else "batch/includes/new_form.html"
    return render_to_response(template, _new_batch_context(request),
            context_instance=RequestContext(request))


@login_required
@project_required
def list(request):
    """
    List recent batches.
    """
    params = request.GET.copy()
    params["project"] = request.session["project"].pk
    context = dict(params=params)
    if not request.is_ajax():
        return render_to_response("batch/list.html", context,
                context_instance=RequestContext(request))

    paginator = Paginator(batch_query(params), PER_PAGE)
    try:
        page = int(request.GET.get('page', '1'))
    except ValueError:
        page = 1

    try:
        batches = paginator.page(page)
    except (EmptyPage, InvalidPage):
        batches = paginator.page(paginator.num_pages)

    pythonserializer = serializers.get_serializer("python")()
    serializedpage = {}
    serializedpage["num_pages"] = paginator.num_pages
    wanted = ("end_index", "has_next", "has_other_pages", "has_previous",
            "next_page_number", "number",
            "start_index", "previous_page_number")
    for attr in wanted:
        want = getattr(batches, attr)
        if isinstance(want, MethodType):
            serializedpage[attr] = want()
        elif isinstance(want, (str, int)):
            serializedpage[attr] = want
    # This gets rather gnarly, see:
    # http://code.google.com/p/wadofstuff/wiki/DjangoFullSerializers
    serializedpage["params"] = params
    serializedpage["object_list"] = pythonserializer.serialize(
        batches.object_list,
        extras=("username", "is_complete", "task_count",),
    )

    response = HttpResponse(mimetype="application/json")
    simplejson.dump(serializedpage, response,
            cls=DjangoJSONEncoder, ensure_ascii=False)
    return response


@login_required
@project_required
@saves_files
@transaction.commit_manually
def create(request):
    """
    Create a batch from pre-saved images convert them with Celery.
    """

    celerytask = tasks.ConvertPageTask

    # get the subject file paths from comma seperated POST data
    paths = _get_batch_file_paths(request)
    form = OcrBatchForm(request.POST)
    if not request.method == "POST" or not form.is_valid() or not paths:
        return render_to_response("batch/new.html",
            _new_batch_context(request),
            context_instance=RequestContext(request))

    # create a batch db job
    # TODO: catch potential integrity error for a duplicate
    # batch name within the given project
    batch = OcrBatch(
        user=request.user,
        name=form.cleaned_data["name"],
        description=form.cleaned_data["description"],
        tags=form.cleaned_data["tags"],
        task_type=celerytask.name,
        project=request.session["project"]
    )
    batch.save()

    # wrangle the params - this needs improving
    userparams = _get_best_params(
            _cleanup_params(request.POST.copy(),
                ("files", "name", "description", "tags")))
    userparams["intermediate_outdir"] = request.output_path.encode()

    asyncparams = []
    try:
        for path in paths:
            tid = ocrutils.get_new_task_id()
            args = (path.encode(), request.output_path.encode(), userparams)
            kwargs = dict(task_id=tid, loglevel=60, retries=2)
            ocrtask = OcrTask(
                task_id=tid,
                user=request.user,
                batch=batch,
                project=request.session["project"],
                page_name=os.path.basename(path),
                task_name=celerytask.name,
                status="INIT",
                args=args,
                kwargs=kwargs,
            )
            ocrtask.save()
            asyncparams.append((args, kwargs))

        publisher = celerytask.get_publisher(connect_timeout=5)
        try:
            for args, kwargs in asyncparams:
                celerytask.apply_async(args=args,<commit_msg>changed name of parameter indicating we're to save intermediate output paths<commit_after>import os
from types import MethodType
import tempfile
import gzip
import tarfile
import StringIO

from celery import result as celeryresult
from celery.contrib.abortable import AbortableAsyncResult
from datetime import datetime
from django import forms
from django.contrib.auth.decorators import login_required
from django.contrib import messages
from django.core import serializers
from django.core.paginator import Paginator, InvalidPage, EmptyPage
from django.core.serializers.json import DjangoJSONEncoder
from django.core.servers.basehttp import FileWrapper
from django.conf import settings
from django.db import transaction
from django.db.models import Q, Count
from django.http import HttpResponse, HttpResponseRedirect, \
        HttpResponseServerError
from django.shortcuts import render_to_response, get_object_or_404
from django.template import RequestContext
from django.utils import simplejson
from django.utils.encoding import smart_str, smart_unicode
from ocradmin.batch import utils as batchutils
from ocradmin.ocr import tasks
from ocradmin.ocr import utils as ocrutils
from ocradmin.ocr.utils import saves_files
from ocradmin.ocrpresets.models import OcrPreset
from ocradmin.ocrtasks.models import OcrTask, OcrBatch, Transcript
from ocradmin.projects.tasks import IngestTask
from ocradmin.training.tasks import ComparisonTask

from ocradmin.projects.utils import project_required
from ocradmin.ocr.views import _get_best_params, _cleanup_params, \
        _handle_request, AppException
from ocradmin.ocr.tools.manager import PluginManager


PER_PAGE = 25


def batch_query(params):
    """
    Query the batch db.
    """
    order = [x for x in params.getlist("order_by") if x != ""] \
            or ["created_on"]

    query = Q()
    for key, val in params.items():
        if key.find("__") == -1 and \
                not key in OcrBatch._meta.get_all_field_names():
            continue
        if not val:
            continue
        query = query & Q(**{str(key): str(val)})
    query = OcrBatch.objects.select_related().filter(query)
    if order and order[0].replace("-", "", 1) == "task_count":
        query = query.annotate(task_count=Count("tasks"))
    return query.order_by(*order)


class OcrBatchForm(forms.ModelForm):
    """
        New project form.
    """
    def __init__(self, *args, **kwargs):
        super(forms.ModelForm, self).__init__(*args, **kwargs)

        # change a widget attribute:
        self.fields['description'].widget.attrs["rows"] = 2
        self.fields['description'].widget.attrs["cols"] = 40

    class Meta:
        model = OcrBatch
        exclude = ["user", "created_on", "project", "task_type"]


@login_required
@project_required
def new(request):
    """
    Present a new batch form.
    """
    template = "batch/new.html" if not request.is_ajax() \
        else "batch/includes/new_form.html"
    return render_to_response(template, _new_batch_context(request),
            context_instance=RequestContext(request))


@login_required
@project_required
def list(request):
    """
    List recent batches.
    """
    params = request.GET.copy()
    params["project"] = request.session["project"].pk
    context = dict(params=params)
    if not request.is_ajax():
        return render_to_response("batch/list.html", context,
                context_instance=RequestContext(request))

    paginator = Paginator(batch_query(params), PER_PAGE)
    try:
        page = int(request.GET.get('page', '1'))
    except ValueError:
        page = 1

    try:
        batches = paginator.page(page)
    except (EmptyPage, InvalidPage):
        batches = paginator.page(paginator.num_pages)

    pythonserializer = serializers.get_serializer("python")()
    serializedpage = {}
    serializedpage["num_pages"] = paginator.num_pages
    wanted = ("end_index", "has_next", "has_other_pages", "has_previous",
            "next_page_number", "number",
            "start_index", "previous_page_number")
    for attr in wanted:
        want = getattr(batches, attr)
        if isinstance(want, MethodType):
            serializedpage[attr] = want()
        elif isinstance(want, (str, int)):
            serializedpage[attr] = want
    # This gets rather gnarly, see:
    # http://code.google.com/p/wadofstuff/wiki/DjangoFullSerializers
    serializedpage["params"] = params
    serializedpage["object_list"] = pythonserializer.serialize(
        batches.object_list,
        extras=("username", "is_complete", "task_count",),
    )

    response = HttpResponse(mimetype="application/json")
    simplejson.dump(serializedpage, response,
            cls=DjangoJSONEncoder, ensure_ascii=False)
    return response


@login_required
@project_required
@saves_files
@transaction.commit_manually
def create(request):
    """
    Create a batch from pre-saved images convert them with Celery.
    """

    celerytask = tasks.ConvertPageTask

    # get the subject file paths from comma seperated POST data
    paths = _get_batch_file_paths(request)
    form = OcrBatchForm(request.POST)
    if not request.method == "POST" or not form.is_valid() or not paths:
        return render_to_response("batch/new.html",
            _new_batch_context(request),
            context_instance=RequestContext(request))

    # create a batch db job
    # TODO: catch potential integrity error for a duplicate
    # batch name within the given project
    batch = OcrBatch(
        user=request.user,
        name=form.cleaned_data["name"],
        description=form.cleaned_data["description"],
        tags=form.cleaned_data["tags"],
        task_type=celerytask.name,
        project=request.session["project"]
    )
    batch.save()

    # wrangle the params - this needs improving
    userparams = _get_best_params(
            _cleanup_params(request.POST.copy(),
                ("files", "name", "description", "tags")))
    # preserve intermediate binary & segmentation results            
    userparams["write_intermediate_results"] = True
    asyncparams = []
    try:
        for path in paths:
            tid = ocrutils.get_new_task_id()
            args = (path.encode(), request.output_path.encode(), userparams)
            kwargs = dict(task_id=tid, loglevel=60, retries=2)
            ocrtask = OcrTask(
                task_id=tid,
                user=request.user,
                batch=batch,
                project=request.session["project"],
                page_name=os.path.basename(path),
                task_name=celerytask.name,
                status="INIT",
                args=args,
                kwargs=kwargs,
            )
            ocrtask.save()
            asyncparams.append((args, kwargs))

        publisher = celerytask.get_publisher(connect_timeout=5)
        try:
            for args, kwargs in asyncparams:
                celerytask.apply_async(args=args,
{'fraction_non_alphanumeric': 0.09942624736727432, 'fraction_numerical': 0.002469315128186506, 'mean_word_length': 3.04880917377242, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5251102', 'n_tokens_mistral': 3951, 'n_tokens_neox': 3871, 'n_words': 958}