<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division

import csv
import json
import io
from copy import copy

from django.db import transaction
from django.shortcuts import get_object_or_404, render_to_response
from django.template import RequestContext
from django.core.paginator import Paginator, Page
from django.core.urlresolvers import reverse
from django.http import HttpResponse
from django.utils.translation import ugettext as trans

from django.contrib.auth.decorators import login_required

from django_tinsel.utils import decorate as do
from django_tinsel.decorators import render_template

from treemap.models import Species, Tree, User
from treemap.decorators import (admin_instance_request, require_http_method,
                                requires_feature)
from treemap.units import get_conversion_factor

from importer.models import GenericImportEvent, GenericImportRow
from importer.trees import TreeImportEvent, TreeImportRow
from importer.species import SpeciesImportEvent, SpeciesImportRow
from importer.tasks import (run_import_event_validation, commit_import_event,
                            get_import_event_model, get_import_row_model)
from importer import errors, fields
from importer.util import lowerkeys


def find_similar_species(request, instance):
    target = request.REQUEST['target']

    species = Species.objects\
                     .filter(instance=instance)\
                     .extra(
                         select={
                             'l': ("levenshtein(genus || ' ' || species || "
                                   "' ' || cultivar_name || ' ' || "
                                   "other_part_of_name, %s)")
                         },
                         select_params=(target,))\
                     .order_by('l')[0:2]  # Take top 2

    output = [{fields.trees.GENUS: s.genus,
               fields.trees.SPECIES: s.species,
               fields.trees.CULTIVAR: s.cultivar_name,
               fields.trees.OTHER_PART_OF_NAME: s.other_part_of_name,
               'pk': s.pk} for s in species]

    return HttpResponse(json.dumps(output), content_type='application/json')


def counts(request, instance):
    active_trees = TreeImportEvent\
        .objects\
        .filter(instance=instance)\
        .order_by('id')\
        .exclude(status=GenericImportEvent.FINISHED_CREATING)\
        .exclude(status=GenericImportEvent.FINISHED_VERIFICATION)\
        .exclude(status=GenericImportEvent.FAILED_FILE_VERIFICATION)

    active_species = SpeciesImportEvent\
        .objects\
        .filter(instance=instance)\
        .order_by('id')\
        .exclude(status=GenericImportEvent.FINISHED_CREATING)\
        .exclude(status=GenericImportEvent.FINISHED_VERIFICATION)\
        .exclude(status=GenericImportEvent.FAILED_FILE_VERIFICATION)

    output = {}
    output['trees'] = {t.pk: t.row_counts_by_status() for t in active_trees}
    output['species'] = {s.pk: s.row_counts_by_status()
                         for s in active_species}

    return HttpResponse(json.dumps(output), content_type='application/json')


def start_import(request, instance):
    import_type = request.REQUEST['type']
    if import_type == TreeImportEvent.import_type:
        kwargs = {
            'plot_length_conversion_factor':
            float(request.REQUEST.get('unit_plot_length', 1.0)),

            'plot_width_conversion_factor':
            float(request.REQUEST.get('unit_plot_width', 1.0)),

            'diameter_conversion_factor':
            float(request.REQUEST.get('unit_diameter', 1.0)),

            'tree_height_conversion_factor':
            float(request.REQUEST.get('unit_tree_height', 1.0)),

            'canopy_height_conversion_factor':
            float(request.REQUEST.get('unit_canopy_height', 1.0))
        }
    else:
        kwargs = {
            'max_diameter_conversion_factor':
            get_conversion_factor(instance, 'tree', 'diameter'),
            'max_tree_height_conversion_factor':
            get_conversion_factor(instance, 'tree', 'height')
        }
    process_csv(request, instance, import_type, **kwargs)

    return list_imports(request, instance)


def list_imports(request, instance):
    finished = GenericImportEvent.FINISHED_CREATING

    trees = TreeImportEvent.objects.filter(instance=instance).order_by('id')
    active_trees = trees.exclude(status=finished)
    finished_trees = trees.filter(status=finished)

    species = SpeciesImportEvent.objects\
        .filter(instance=instance)\
        .order_by('id')
    active_species = species.exclude(status=finished)
    finished_species = species.filter(status=finished)

    active_events = list(active_trees) + list(active_species)
    imports_finished = all(ie.is_finished() for ie in active_events)

    return {'active_trees': active_trees,
            'finished_trees': finished_trees,<commit_msg>Make unit retrieval view DRY<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division

import csv
import json
import io
from copy import copy

from django.db import transaction
from django.shortcuts import get_object_or_404, render_to_response
from django.template import RequestContext
from django.core.paginator import Paginator, Page
from django.core.urlresolvers import reverse
from django.http import HttpResponse
from django.utils.translation import ugettext as trans

from django.contrib.auth.decorators import login_required

from django_tinsel.utils import decorate as do
from django_tinsel.decorators import render_template

from treemap.models import Species, Tree, User
from treemap.decorators import (admin_instance_request, require_http_method,
                                requires_feature)
from treemap.units import get_conversion_factor

from importer.models import GenericImportEvent, GenericImportRow
from importer.trees import TreeImportEvent, TreeImportRow
from importer.species import SpeciesImportEvent, SpeciesImportRow
from importer.tasks import (run_import_event_validation, commit_import_event,
                            get_import_event_model, get_import_row_model)
from importer import errors, fields
from importer.util import lowerkeys


def find_similar_species(request, instance):
    target = request.REQUEST['target']

    species = Species.objects\
                     .filter(instance=instance)\
                     .extra(
                         select={
                             'l': ("levenshtein(genus || ' ' || species || "
                                   "' ' || cultivar_name || ' ' || "
                                   "other_part_of_name, %s)")
                         },
                         select_params=(target,))\
                     .order_by('l')[0:2]  # Take top 2

    output = [{fields.trees.GENUS: s.genus,
               fields.trees.SPECIES: s.species,
               fields.trees.CULTIVAR: s.cultivar_name,
               fields.trees.OTHER_PART_OF_NAME: s.other_part_of_name,
               'pk': s.pk} for s in species]

    return HttpResponse(json.dumps(output), content_type='application/json')


def counts(request, instance):
    active_trees = TreeImportEvent\
        .objects\
        .filter(instance=instance)\
        .order_by('id')\
        .exclude(status=GenericImportEvent.FINISHED_CREATING)\
        .exclude(status=GenericImportEvent.FINISHED_VERIFICATION)\
        .exclude(status=GenericImportEvent.FAILED_FILE_VERIFICATION)

    active_species = SpeciesImportEvent\
        .objects\
        .filter(instance=instance)\
        .order_by('id')\
        .exclude(status=GenericImportEvent.FINISHED_CREATING)\
        .exclude(status=GenericImportEvent.FINISHED_VERIFICATION)\
        .exclude(status=GenericImportEvent.FAILED_FILE_VERIFICATION)

    output = {}
    output['trees'] = {t.pk: t.row_counts_by_status() for t in active_trees}
    output['species'] = {s.pk: s.row_counts_by_status()
                         for s in active_species}

    return HttpResponse(json.dumps(output), content_type='application/json')


def start_import(request, instance):
    import_type = request.REQUEST['type']
    if import_type == TreeImportEvent.import_type:
        factors = {'plot_length_conversion_factor': 'unit_plot_length',
                   'plot_width_conversion_factor': 'unit_plot_width',
                   'diameter_conversion_factor': 'unit_diameter',
                   'tree_height_conversion_factor': 'unit_tree_height',
                   'canopy_height_conversion_factor': 'unit_canopy_height'}
        kwargs = {k: float(request.REQUEST.get(v, 1.0))
                  for (k, v) in factors.items()}
    else:
        kwargs = {
            'max_diameter_conversion_factor':
            get_conversion_factor(instance, 'tree', 'diameter'),
            'max_tree_height_conversion_factor':
            get_conversion_factor(instance, 'tree', 'height')
        }
    process_csv(request, instance, import_type, **kwargs)

    return list_imports(request, instance)


def list_imports(request, instance):
    finished = GenericImportEvent.FINISHED_CREATING

    trees = TreeImportEvent.objects.filter(instance=instance).order_by('id')
    active_trees = trees.exclude(status=finished)
    finished_trees = trees.filter(status=finished)

    species = SpeciesImportEvent.objects\
        .filter(instance=instance)\
        .order_by('id')
    active_species = species.exclude(status=finished)
    finished_species = species.filter(status=finished)

    active_events = list(active_trees) + list(active_species)
    imports_finished = all(ie.is_finished() for ie in active_events)

    return {'active_trees': active_trees,
            'finished_trees': finished_trees,
{'fraction_non_alphanumeric': 0.09151785714285714, 'fraction_numerical': 0.002637987012987013, 'mean_word_length': 2.9443777511004403, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7365168', 'n_tokens_mistral': 2875, 'n_tokens_neox': 2667, 'n_words': 543}