<commit_before># -*- coding: utf-8 -*-
from contextlib import closing
import logging

from desktop.lib.django_util import PopupException, render, render_to_string
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.core.urlresolvers import reverse
from django.forms.util import ErrorList
from django.http import Http404
from django.shortcuts import get_object_or_404, redirect
import jobsub.views as jobsub
from jobsub.models import Submission
from jobsubd.ttypes import (BinHadoopStep, LocalizedFile, LocalizeFilesStep,
                            State, SubmissionPlan, SubmissionPlanStep)

from cosmos import conf, mongo, paths
from cosmos.jar import InvalidJarFile, JarFile
from cosmos.hdfs_util import CachedHDFSFile
from cosmos.models import JobRun
from cosmos.forms import DefineJobForm, ParameterizeJobForm

LOGGER = logging.getLogger(__name__)
TEMP_JAR = "tmp.jar"
DEFAULT_PAGE = 1


def index(request):
    """List job runs."""

    job_runs = JobRun.objects.filter(user=request.user).order_by('-start_date')
    for job_run in job_runs:
        job_run.refresh_state()
    return render('index.mako', request, dict(
        job_runs=job_runs
    ))


def submit(job):
    LOGGER.info("Submitting job %s (%d) for %s with JAR=%s on dataset %s" %
                (job.name, job.id, job.user.username, job.jar_path,
                 job.dataset_path))

    lf = LocalizedFile(target_name=TEMP_JAR, path_on_hdfs=job.jar_path)
    lfs = LocalizeFilesStep(localize_files=[lf])
    bhs = BinHadoopStep(arguments=job.hadoop_args(TEMP_JAR))
    plan = SubmissionPlan(name=job.name,
                          user=job.user.username,
                          groups=job.user.get_groups(),
                          steps=[SubmissionPlanStep(localize_files_step=lfs),
                                 SubmissionPlanStep(bin_hadoop_step=bhs)])
    submission = Submission(owner=job.user,
                            name=job.name,
                            last_seen_state=State.SUBMITTED,
                            submission_plan=plan)
    submission.save()
    job.submission = submission
    job.save()
    try:
        try:
            submission.submission_handle = jobsub.get_client().submit(plan)
        except Exception:
            import ipdb; ipdb.set_trace()
            submission.last_seen_state = State.ERROR
            raise
    finally:
        submission.save()


def job_wizard(request):
    return request.session.get('job_wizard', {})


def clear_job_wizard(request):
    request.session.pop('job_wizard', '')


def update_job_wizard(request, wizard):
    request.session['job_wizard'] = wizard


def define_job(request):
    """Defines a new job in user session."""

    wizard = job_wizard(request)
    job = wizard.get('job', {})

    if request.method != 'POST':
        form = DefineJobForm(initial=job)

    else:
        form = DefineJobForm(request.POST, request.FILES)
        if form.is_valid(request.fs):
            job.update(form.cleaned_data)
            wizard['job'] = job
            update_job_wizard(request, wizard)

            with closing(CachedHDFSFile(request.fs, job['jar_path'])) \
                    as cached_file:
                try:
                    jar = JarFile(cached_file.local_path())
                    try:
                        wizard['parameterized'] = jar.is_parameterized()
                        if wizard['parameterized']:
                            wizard['parameters'] = [{
                                'name': template.name,
                                'type': template.type,
                                'default_value': template.default_value
                            } for template in jar.parameters()]
                            next_action = 'configure_job'
                        else:
                            wizard['parameters'] = None
                            next_action = 'confirm_job'
                        return redirect(reverse(next_action))
                    finally:
                        jar.close()
                except InvalidJarFile as ex:
                    errors = form._errors.setdefault('jar_path', ErrorList())
                    errors.append('Invalid JAR: %s' % ex.message)

    return render('job_define.mako', request, dict(
        form=form
    ))


def configure_job(request):
    """Job parametrization."""

    wizard = job_wizard(request)
    job = wizard.get('job', {})
    parameters = wizard.get('parameters', None)

    if not parameters:
        return redirect(reverse('confirm_job'))

    if request.method != 'POST':
        form = ParameterizeJobForm(parameters)

    elif request.POST.has_key('back'):
        return redirect(reverse('define_job'))

    else:
        form = ParameterizeJobForm(parameters, request.POST)
        if form.is_valid():
            wizard['parameter_values'] = form.cleaned_data
            update_job_wizard(request, wizard)
            return redirect(reverse('confirm_job'))

    return render('job_parameterize.mako', request, dict(form=form))


def confirm_job(request):
    """Confirm and run."""

    wizard = job_wizard(request)
    job_data = wizard.get('job', {})
    job = JobRun(user=request.user)
    try:
        job.name = job_data['name']
        job.description = job_data['description']
        job.jar_path = job_data['jar_path']
        job.dataset_path = job_data['dataset_path']
        if wizard['parameterized']:
            job.parameters = wizard['parameter_values']
    except KeyError:
        return redirect(reverse('define_job'))

    if request.method != 'POST':
        return render('job_confirm.mako', request, wizard)

    elif request.POST.has_key('back'):
        if wizard['parameterized']:
            prev_action = 'configure_job'
        else:
            prev_action = 'define_job'
        return redirect(reverse(prev_action))

    else:
        job.save()
        submit(job)
        clear_job_wizard(request)
        return redirect(reverse('list_jobs'))


def cancel_job(request):
    """Cancels and restarts the new job wizard."""
    clear_job_wizard(request)
    return redirect(reverse('define_job'))


def ensure_dir(fs, path):
    """Makes sure a directory with a given path exists or it is created.
       Raises a popup if a file is in the way"""
    if fs.isdir(path):
        return
    if fs.exists(path):
        raise PopupException(('A file named "%s" is preventing the creation ' + 
                              'of the upload directory. Please, rename it to ' +
                              'proceed.') % path)
    else:
        fs.mkdir(path)


def upload_index(request):
    """Allows to upload datasets and jars."""
    
    datasets_base = paths.datasets_base(request.user)
    ensure_dir(request.fs, datasets_base)
    jars_base = paths.jars_base(request.user)
    ensure_dir(request.fs, jars_base)
    return render('upload_index.mako', request, dict(
        datasets_base=datasets_base,
        jars_base=jars_base
    ))


def show_results(request, job_id):
    job = get_object_or_404(JobRun, pk=job_id, user=request.user)
    if job.submission is None:
        raise Http404

    try:
        primary_key = request.GET.get('primary_key')
        paginator = mongo.retrieve_results(job.id, primary_key)

        try:
            page_num = request.GET.get('page', DEFAULT_PAGE)
        except ValueError:
            page_num = DEFAULT_PAGE

        try:
            page = paginator.page(page_num)
        except PageNotAnInteger:
            page = paginator.page(DEFAULT_PAGE)
        except EmptyPage:
            page = paginator.page(paginator.num_pages)

        prototype_result = page.object_list[0]
        if primary_key is None:
            primary_key = prototype_result.pk

        return render('job_results.mako', request, dict(
            title='Results of job %s' % job.id,
            page=page,
            hidden_keys=mongo.HIDDEN_KEYS,
            primary_key=primary_key
        ))

    except mongo.NoResultsError:
        return render('job_results.mako', request, dict(
            title='Results of job %s' % job.id,
            hidden_keys=mongo.HIDDEN_KEYS
        ))

    except mongo.NoConnectionError:
        raise PopupException('Database not available')
<commit_msg>Fix: restore parameter values<commit_after># -*- coding: utf-8 -*-
from contextlib import closing
import logging

from desktop.lib.django_util import PopupException, render, render_to_string
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.core.urlresolvers import reverse
from django.forms.util import ErrorList
from django.http import Http404
from django.shortcuts import get_object_or_404, redirect
import jobsub.views as jobsub
from jobsub.models import Submission
from jobsubd.ttypes import (BinHadoopStep, LocalizedFile, LocalizeFilesStep,
                            State, SubmissionPlan, SubmissionPlanStep)

from cosmos import conf, mongo, paths
from cosmos.jar import InvalidJarFile, JarFile
from cosmos.hdfs_util import CachedHDFSFile
from cosmos.models import JobRun
from cosmos.forms import DefineJobForm, ParameterizeJobForm

LOGGER = logging.getLogger(__name__)
TEMP_JAR = "tmp.jar"
DEFAULT_PAGE = 1


def index(request):
    """List job runs."""

    job_runs = JobRun.objects.filter(user=request.user).order_by('-start_date')
    for job_run in job_runs:
        job_run.refresh_state()
    return render('index.mako', request, dict(
        job_runs=job_runs
    ))


def submit(job):
    LOGGER.info("Submitting job %s (%d) for %s with JAR=%s on dataset %s" %
                (job.name, job.id, job.user.username, job.jar_path,
                 job.dataset_path))

    lf = LocalizedFile(target_name=TEMP_JAR, path_on_hdfs=job.jar_path)
    lfs = LocalizeFilesStep(localize_files=[lf])
    bhs = BinHadoopStep(arguments=job.hadoop_args(TEMP_JAR))
    plan = SubmissionPlan(name=job.name,
                          user=job.user.username,
                          groups=job.user.get_groups(),
                          steps=[SubmissionPlanStep(localize_files_step=lfs),
                                 SubmissionPlanStep(bin_hadoop_step=bhs)])
    submission = Submission(owner=job.user,
                            name=job.name,
                            last_seen_state=State.SUBMITTED,
                            submission_plan=plan)
    submission.save()
    job.submission = submission
    job.save()
    try:
        try:
            submission.submission_handle = jobsub.get_client().submit(plan)
        except Exception:
            import ipdb; ipdb.set_trace()
            submission.last_seen_state = State.ERROR
            raise
    finally:
        submission.save()


def job_wizard(request):
    return request.session.get('job_wizard', {})


def clear_job_wizard(request):
    request.session.pop('job_wizard', '')


def update_job_wizard(request, wizard):
    request.session['job_wizard'] = wizard


def define_job(request):
    """Defines a new job in user session."""

    wizard = job_wizard(request)
    job = wizard.get('job', {})

    if request.method != 'POST':
        form = DefineJobForm(initial=job)

    else:
        form = DefineJobForm(request.POST, request.FILES)
        if form.is_valid(request.fs):
            job.update(form.cleaned_data)
            wizard['job'] = job
            update_job_wizard(request, wizard)

            with closing(CachedHDFSFile(request.fs, job['jar_path'])) \
                    as cached_file:
                try:
                    jar = JarFile(cached_file.local_path())
                    try:
                        wizard['parameterized'] = jar.is_parameterized()
                        if wizard['parameterized']:
                            wizard['parameters'] = [{
                                'name': template.name,
                                'type': template.type,
                                'default_value': template.default_value
                            } for template in jar.parameters()]
                            next_action = 'configure_job'
                        else:
                            wizard['parameters'] = None
                            next_action = 'confirm_job'
                        return redirect(reverse(next_action))
                    finally:
                        jar.close()
                except InvalidJarFile as ex:
                    errors = form._errors.setdefault('jar_path', ErrorList())
                    errors.append('Invalid JAR: %s' % ex.message)

    return render('job_define.mako', request, dict(
        form=form
    ))


def configure_job(request):
    """Job parametrization."""

    wizard = job_wizard(request)
    job = wizard.get('job', {})
    parameters = wizard.get('parameters', None)

    if not parameters:
        return redirect(reverse('confirm_job'))

    if request.method != 'POST':
        form = ParameterizeJobForm(parameters, wizard.get('parameter_values',
                                                          None))

    elif request.POST.has_key('back'):
        return redirect(reverse('define_job'))

    else:
        form = ParameterizeJobForm(parameters, request.POST)
        if form.is_valid():
            wizard['parameter_values'] = form.cleaned_data
            update_job_wizard(request, wizard)
            return redirect(reverse('confirm_job'))

    return render('job_parameterize.mako', request, dict(form=form))


def confirm_job(request):
    """Confirm and run."""

    wizard = job_wizard(request)
    job_data = wizard.get('job', {})
    job = JobRun(user=request.user)
    try:
        job.name = job_data['name']
        job.description = job_data['description']
        job.jar_path = job_data['jar_path']
        job.dataset_path = job_data['dataset_path']
        if wizard['parameterized']:
            job.parameters = wizard['parameter_values']
    except KeyError:
        return redirect(reverse('define_job'))

    if request.method != 'POST':
        return render('job_confirm.mako', request, wizard)

    elif request.POST.has_key('back'):
        if wizard['parameterized']:
            prev_action = 'configure_job'
        else:
            prev_action = 'define_job'
        return redirect(reverse(prev_action))

    else:
        job.save()
        submit(job)
        clear_job_wizard(request)
        return redirect(reverse('list_jobs'))


def cancel_job(request):
    """Cancels and restarts the new job wizard."""
    clear_job_wizard(request)
    return redirect(reverse('define_job'))


def ensure_dir(fs, path):
    """Makes sure a directory with a given path exists or it is created.
       Raises a popup if a file is in the way"""
    if fs.isdir(path):
        return
    if fs.exists(path):
        raise PopupException(('A file named "%s" is preventing the creation ' + 
                              'of the upload directory. Please, rename it to ' +
                              'proceed.') % path)
    else:
        fs.mkdir(path)


def upload_index(request):
    """Allows to upload datasets and jars."""
    
    datasets_base = paths.datasets_base(request.user)
    ensure_dir(request.fs, datasets_base)
    jars_base = paths.jars_base(request.user)
    ensure_dir(request.fs, jars_base)
    return render('upload_index.mako', request, dict(
        datasets_base=datasets_base,
        jars_base=jars_base
    ))


def show_results(request, job_id):
    job = get_object_or_404(JobRun, pk=job_id, user=request.user)
    if job.submission is None:
        raise Http404

    try:
        primary_key = request.GET.get('primary_key')
        paginator = mongo.retrieve_results(job.id, primary_key)

        try:
            page_num = request.GET.get('page', DEFAULT_PAGE)
        except ValueError:
            page_num = DEFAULT_PAGE

        try:
            page = paginator.page(page_num)
        except PageNotAnInteger:
            page = paginator.page(DEFAULT_PAGE)
        except EmptyPage:
            page = paginator.page(paginator.num_pages)

        prototype_result = page.object_list[0]
        if primary_key is None:
            primary_key = prototype_result.pk

        return render('job_results.mako', request, dict(
            title='Results of job %s' % job.id,
            page=page,
            hidden_keys=mongo.HIDDEN_KEYS,
            primary_key=primary_key
        ))

    except mongo.NoResultsError:
        return render('job_results.mako', request, dict(
            title='Results of job %s' % job.id,
            hidden_keys=mongo.HIDDEN_KEYS
        ))

    except mongo.NoConnectionError:
        raise PopupException('Database not available')

{'fraction_non_alphanumeric': 0.09990428332136875, 'fraction_numerical': 0.0017946877243359654, 'mean_word_length': 2.2166634596882817, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '826383', 'n_tokens_mistral': 4865, 'n_tokens_neox': 4705, 'n_words': 1011}