<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division

import csv

from contextlib import contextmanager
from functools import wraps
from collections import OrderedDict
from celery import task
from tempfile import TemporaryFile

from django.core.files import File
from treemap.lib.object_caches import permissions

from treemap.search import Filter
from treemap.models import Species, Plot
from treemap.util import safe_get_model_class
from treemap.audit import model_hasattr, FieldPermission
from treemap.udf import UserDefinedCollectionValue

from treemap.lib.object_caches import udf_defs

from djqscsv import write_csv, generate_filename
from exporter.models import ExportJob

from exporter.user import write_users
from exporter.util import sanitize_unicode_record


@contextmanager
def _job_transaction_manager(job_pk):
    job = ExportJob.objects.get(pk=job_pk)
    try:
        yield job
    except:
        job.fail()
        job.save()
        raise


def _job_transaction(fn):
    @wraps(fn)
    def wrapper(job_pk, *args, **kwargs):
        with _job_transaction_manager(job_pk) as job:
            return fn(job, *args, **kwargs)
    return wrapper


def values_for_model(
        instance, job, table, model,
        select, select_params, prefix=None):
    if prefix:
        prefix += '__'
    else:
        prefix = ''

    perms = permissions(job.user, instance, model)

    prefixed_names = []
    model_class = safe_get_model_class(model)
    dummy_instance = model_class()

    for perm in (perm for perm in perms
                 if perm.permission_level >= FieldPermission.READ_ONLY):
        field_name = perm.field_name
        prefixed_name = prefix + field_name

        if field_name in model_class.collection_udf_settings.items():

            field_definition_id = None
            for udfd in udf_defs(instance, model):
                if udfd.iscollection and udfd.name == field_name[4:]:
                    field_definition_id = udfd.id

            if field_definition_id is None:
                continue

            select[prefixed_name] = (
                """
                WITH formatted_data AS (
                    SELECT concat('(', data, ')') as fdata
                    FROM %s
                    WHERE field_definition_id = %s and model_id = %s.id
                )

                SELECT array_to_string(array_agg(fdata), ', ', '*')
                FROM formatted_data
                """
                % (UserDefinedCollectionValue._meta.db_table,
                   field_definition_id, table))
        elif field_name.startswith('udf:'):
            name = field_name[4:]
            select[prefixed_name] = "{0}.udfs->%s".format(table)
            select_params.append(name)
        else:
            if not model_hasattr(dummy_instance, field_name):
                # Exception will be raised downstream if you look for
                # a field on a model that no longer exists but still
                # has a stale permission record. Here we check for that
                # case and don't include the field if it does not exist.
                continue

        prefixed_names.append(prefixed_name)

    return prefixed_names


@task
@_job_transaction
def async_users_export(job, data_format):
    instance = job.instance

    if data_format == 'csv':
        filename = 'users.csv'
    else:
        filename = 'users.json'
<commit_msg>Fix search exporting of collection UDFs.<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division

import csv

from contextlib import contextmanager
from functools import wraps
from collections import OrderedDict
from celery import task
from tempfile import TemporaryFile

from django.core.files import File
from treemap.lib.object_caches import permissions

from treemap.search import Filter
from treemap.models import Species, Plot
from treemap.util import safe_get_model_class
from treemap.audit import model_hasattr, FieldPermission
from treemap.udf import UserDefinedCollectionValue

from treemap.lib.object_caches import udf_defs

from djqscsv import write_csv, generate_filename
from exporter.models import ExportJob

from exporter.user import write_users
from exporter.util import sanitize_unicode_record


@contextmanager
def _job_transaction_manager(job_pk):
    job = ExportJob.objects.get(pk=job_pk)
    try:
        yield job
    except:
        job.fail()
        job.save()
        raise


def _job_transaction(fn):
    @wraps(fn)
    def wrapper(job_pk, *args, **kwargs):
        with _job_transaction_manager(job_pk) as job:
            return fn(job, *args, **kwargs)
    return wrapper


def values_for_model(
        instance, job, table, model,
        select, select_params, prefix=None):
    if prefix:
        prefix += '__'
    else:
        prefix = ''

    perms = permissions(job.user, instance, model)

    prefixed_names = []
    model_class = safe_get_model_class(model)
    dummy_instance = model_class()

    for perm in (perm for perm in perms
                 if perm.permission_level >= FieldPermission.READ_ONLY):
        field_name = perm.field_name
        prefixed_name = prefix + field_name

        if field_name.startswith('udf:'):
            name = field_name[4:]
            if name in model_class.collection_udf_settings.keys():
                field_definition_id = None
                for udfd in udf_defs(instance, model):
                    if udfd.iscollection and udfd.name == name:
                        field_definition_id = udfd.id

                if field_definition_id is None:
                    continue

                select[prefixed_name] = (
                    """
                    WITH formatted_data AS (
                        SELECT concat('(', data, ')') as fdata
                        FROM %s
                        WHERE field_definition_id = %s and model_id = %s.id
                    )

                    SELECT array_to_string(array_agg(fdata), ', ', '*')
                    FROM formatted_data
                    """
                    % (UserDefinedCollectionValue._meta.db_table,
                       field_definition_id, table))
            else:
                select[prefixed_name] = "{0}.udfs->%s".format(table)
                select_params.append(name)
        else:
            if not model_hasattr(dummy_instance, field_name):
                # Exception will be raised downstream if you look for
                # a field on a model that no longer exists but still
                # has a stale permission record. Here we check for that
                # case and don't include the field if it does not exist.
                continue

        prefixed_names.append(prefixed_name)

    return prefixed_names


@task
@_job_transaction
def async_users_export(job, data_format):
    instance = job.instance

    if data_format == 'csv':
        filename = 'users.csv'
    else:
        filename = 'users.json'

{'fraction_non_alphanumeric': 0.07023223082336383, 'fraction_numerical': 0.0009852216748768472, 'mean_word_length': 2.252173913043478, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '900205', 'n_tokens_mistral': 2031, 'n_tokens_neox': 1915, 'n_words': 556}