<commit_before>from arches.app.datatypes.base import BaseDataType
from arches.app.models import models
from arches.app.views.concept import get_preflabel_from_valueid
import uuid

class BaseConceptDataType(BaseDataType):
    def get_concept_export_value(self, value, concept_export_value_type):
        if concept_export_value_type != None:
            if concept_export_value_type == "label" or concept_export_value_type == "both":
                if concept_export_value_type == "label":
                    value = Value.objects.get(valueid=value).value
                elif concept_export_value_type == "both":
                    value = value + '|' + Value.objects.get(valueid=value).value
        return value

    def append_to_document(self, document, nodevalue):
        try:
            assert isinstance(nodevalue, (list, tuple)) #assert nodevalue is an array
        except AssertionError:
            nodevalue = [nodevalue]
        for concept_valueid in nodevalue:
            value = models.Value.objects.get(pk=concept_valueid)
            document['domains'].append({'label': value.value, 'conceptid': value.concept_id, 'valueid': concept_valueid})

class ConceptDataType(BaseConceptDataType):
    def transform_import_values(self, value):
        return value.strip()

    def transform_export_values(self, value):
        return self.get_concept_export_value(value, concept_export_value_type)

    def get_pref_label(self, nodevalue, lang='en-US'):
        return get_preflabel_from_valueid(nodevalue, lang)['value']

    def get_display_value(self, tile, node):
        value = None
        concept_values = models.Value.objects.filter(valueid=uuid.UUID(tile.data[str(node.nodeid)]))
        if len(concept_values) > 0:
            value = concept_values[0].value
        return value

class ConceptListDataType(BaseConceptDataType):
    def transform_import_values(self, value):
        return [v.strip() for v in value.split(',')]

    def transform_export_values(self, value):
        new_values = []
        for val in value:
            new_val = self.get_concept_export_value(val, concept_export_value_type)
            new_values.append(new_val)
        return ','.join(new_values)<commit_msg>cache value lookups in the datatype factory, re #1544<commit_after>from arches.app.datatypes.base import BaseDataType
from arches.app.models import models
from arches.app.views.concept import get_preflabel_from_valueid
import uuid

class BaseConceptDataType(BaseDataType):
    def __init__(self, model=None):
        self.value_lookup = {}

    def get_value(self, valueid):
        try:
            return self.value_lookup[valueid]
        except:
            self.value_lookup[valueid] = models.Value.objects.get(pk=valueid)
            return self.value_lookup[valueid]

    def get_concept_export_value(self, valueid, concept_export_value_type):
        ret = ''
        if concept_export_value_type != None:
            if concept_export_value_type == "label" or concept_export_value_type == "both":
                if concept_export_value_type == "label":
                    ret = self.get_value(valueid)
                elif concept_export_value_type == "both":
                    ret = valueid + '|' + self.get_value(valueid)
        return ret

    def append_to_document(self, document, nodevalue):
        try:
            assert isinstance(nodevalue, (list, tuple)) #assert nodevalue is an array
        except AssertionError:
            nodevalue = [nodevalue]
        for valueid in nodevalue:
            value = self.get_value(valueid)
            document['domains'].append({'label': value.value, 'conceptid': value.concept_id, 'valueid': valueid})


class ConceptDataType(BaseConceptDataType):
    def transform_import_values(self, value):
        return value.strip()

    def transform_export_values(self, value):
        return self.get_concept_export_value(value, concept_export_value_type)

    def get_pref_label(self, nodevalue, lang='en-US'):
        return get_preflabel_from_valueid(nodevalue, lang)['value']

    def get_display_value(self, tile, node):
        value = None
        return self.get_value(uuid.UUID(tile.data[str(node.nodeid)]))


class ConceptListDataType(BaseConceptDataType):
    def transform_import_values(self, value):
        return [v.strip() for v in value.split(',')]

    def transform_export_values(self, value):
        new_values = []
        for val in value:
            new_val = self.get_concept_export_value(val, concept_export_value_type)
            new_values.append(new_val)
        return ','.join(new_values)
{'fraction_non_alphanumeric': 0.09491746307558645, 'fraction_numerical': 0.0013032145960034753, 'mean_word_length': 3.1411870503597124, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2748410', 'n_tokens_mistral': 1357, 'n_tokens_neox': 1324, 'n_words': 280}