<commit_before>'''
ARCHES - a program developed to inventory and manage immovable cultural heritage.
Copyright (C) 2013 <NAME> and World Monuments Fund

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
'''

import uuid
import copy
from operator import itemgetter
from operator import methodcaller
from django.db import transaction, connection
from django.db.models import Q
from django.conf import settings
from arches.app.models import models
from arches.app.search.search_engine_factory import SearchEngineFactory
from arches.app.search.elasticsearch_dsl_builder import Match, Query
from arches.app.utils.betterJSONSerializer import JSONSerializer, JSONDeserializer
from django.utils.translation import ugettext as _
import logging


logger = logging.getLogger(__name__)

CORE_CONCEPTS = (
    '00000000-0000-0000-0000-000000000001',
    '00000000-0000-0000-0000-000000000002',
    '00000000-0000-0000-0000-000000000003',
    '00000000-0000-0000-0000-000000000004',
    '00000000-0000-0000-0000-000000000006'
)

class Concept(object):
    def __init__(self, *args, **kwargs):
        self.id = ''
        self.nodetype = ''
        self.legacyoid = ''
        self.relationshiptype = ''
        self.values = []
        self.subconcepts = []
        self.parentconcepts = []
        self.relatedconcepts = []
        self.hassubconcepts = False

        if len(args) != 0:
            if isinstance(args[0], basestring):
                try:
                    uuid.UUID(args[0])
                    self.get(args[0])
                except(ValueError):
                    self.load(JSONDeserializer().deserialize(args[0]))  
            elif isinstance(args[0], dict):
                self.load(args[0])  
            elif isinstance(args[0], object):
                self.load(args[0])  

    def __unicode__(self):
        return ('%s - %s') % (self.get_preflabel().value, self.id)
    
    def load(self, value):
        if isinstance(value, dict):
            self.id = value['id'] if 'id' in value else ''
            self.nodetype = value['nodetype'] if 'nodetype' in value else ''
            self.legacyoid = value['legacyoid'] if 'legacyoid' in value else ''
            self.relationshiptype = value['relationshiptype'] if 'relationshiptype' in value else ''            
            if 'values' in value:
                for val in value['values']:
                    self.addvalue(val)
            if 'subconcepts' in value:
                for subconcept in value['subconcepts']:
                    self.addsubconcept(subconcept)
            if 'parentconcepts' in value:
                for parentconcept in value['parentconcepts']:
                    self.addparent(parentconcept)
            if 'relatedconcepts' in value:
                for relatedconcept in value['relatedconcepts']:
                    self.addrelatedconcept(relatedconcept)

        if isinstance(value, models.Concepts):
            self.id = value.pk
            self.nodetype = value.nodetype_id
            self.legacyoid = value.legacyoid

    def get(self, id='', legacyoid='', include_subconcepts=False, include_parentconcepts=False, include_relatedconcepts=False, exclude=[], include=[], depth_limit=None, up_depth_limit=None, lang=settings.LANGUAGE_CODE, semantic=True, **kwargs):
        if id != '':
            self.load(models.Concepts.objects.get(pk=id))
        elif legacyoid != '':
            self.load(models.Concepts.objects.get(legacyoid=legacyoid))

        if semantic == True:
            pathway_filter = Q(relationtype__category = 'Semantic Relations')
        else:
            pathway_filter = Q(relationtype = 'member')

        if self.id != '':
            nodetype = kwargs.pop('nodetype', self.nodetype)
            uplevel = kwargs.pop('uplevel', 0)
            downlevel = kwargs.pop('downlevel', 0)
            depth_limit = depth_limit if depth_limit == None else int(depth_limit)
            up_depth_limit = up_depth_limit if up_depth_limit == None else int(up_depth_limit)

            if include != None:
                if len(include) > 0 and len(exclude) > 0:
                    raise Exception('Only include values for include or exclude, but not both')
                include = include if len(include) != 0 else models.ValueTypes.objects.distinct('category').values_list('category', flat=True)
                include = set(include).difference(exclude)
                exclude = []

                if len(include) > 0:
                    values = models.Values.objects.filter(conceptid = self.id)
                    for value in values:
                        if value.valuetype.category in include:
                            self.values.append(ConceptValue(value))

            hassubconcepts = models.ConceptRelations.objects.filter(Q(conceptidfrom = self.id), pathway_filter | Q(relationtype__category = 'Properties'), ~Q(relationtype = 'related'))[0:1]
            if len(hassubconcepts) > 0:
                self.hassubconcepts = True

            if include_subconcepts:
                conceptrealations = models.ConceptRelations.objects.filter(Q(conceptidfrom = self.id), pathway_filter | Q(relationtype__category = 'Properties'), ~Q(relationtype = 'related'))
                if depth_limit == None or downlevel < depth_limit:
                    if depth_limit != None:
                        downlevel = downlevel + 1                
                    for relation in conceptrealations:
                        subconcept = Concept().get(id=relation.conceptidto_id, include_subconcepts=include_subconcepts, 
                            include_parentconcepts=False, include_relatedconcepts=include_relatedconcepts, exclude=exclude, include=include, depth_limit=depth_limit, 
                            up_depth_limit=up_depth_limit, downlevel=downlevel, uplevel=uplevel, nodetype=nodetype, semantic=semantic)
                        subconcept.relationshiptype = relation.relationtype.pk
                        self.subconcepts.append(subconcept)

                    self.subconcepts = sorted(self.subconcepts, key=methodcaller('get_sortkey', lang=lang), reverse=False) 

            if include_parentconcepts:
                conceptrealations = models.ConceptRelations.objects.filter(Q(conceptidto = self.id), pathway_filter | Q(relationtype__category = 'Properties'), ~Q(relationtype = 'related'))
                if up_depth_limit == None or uplevel < up_depth_limit:
                    if up_depth_limit != None:
                        uplevel = uplevel + 1          
                    for relation in conceptrealations:
                        parentconcept = Concept().get(id=relation.conceptidfrom_id, include_subconcepts=False, 
                            include_parentconcepts=include_parentconcepts, include_relatedconcepts=include_relatedconcepts, 
                            exclude=exclude, include=include, depth_limit=depth_limit, 
                            up_depth_limit=up_depth_limit, downlevel=downlevel, uplevel=uplevel, nodetype=nodetype, semantic=semantic)
                        parentconcept.relationshiptype = relation.relationtype.pk
                        self.parentconcepts.append(parentconcept)

            if include_relatedconcepts:
                conceptrealations = models.ConceptRelations.objects.filter(Q(relationtype = 'related') | Q(relationtype__category = 'Mapping Properties'), Q(conceptidto = self.id) | Q(conceptidfrom = self.id))
                for relation in conceptrealations:
                    if relation.conceptidto_id != self.id:
                        relatedconcept = Concept().get(relation.conceptidto_id, include=['label'])
                        relatedconcept.relationshiptype = relation.relationtype.pk
                        self.relatedconcepts.append(relatedconcept)
                    if relation.conceptidfrom_id != self.id:
                        relatedconcept = Concept().get(relation.conceptidfrom_id, include=['label'])
                        relatedconcept.relationshiptype = relation.relationtype.pk
                        self.relatedconcepts.append(relatedconcept)

        return self
            
    def save(self):
        self.id = self.id if (self.id != '' and self.id != None) else str(uuid.uuid4())
        concept, created = models.Concepts.objects.get_or_create(pk=self.id, defaults={'legacyoid': self.legacyoid if self.legacyoid != '' else self.id, 'nodetype_id': self.nodetype})

        for value in self.values:
            if not isinstance(value, ConceptValue): 
                value = ConceptValue(value)
            value.conceptid = self.id
            value.save()        

        for parentconcept in self.parentconcepts:
            parentconcept.save()
            parentconcept.add_relation(self, parentconcept.relationshiptype)

        for subconcept in self.subconcepts:
            subconcept.save()
            self.add_relation(subconcept, subconcept.relationshiptype)

        for relatedconcept in self.relatedconcepts:
            self.add_relation(relatedconcept, relatedconcept.relationshiptype)

        return concept

    def delete(self, delete_self=False):
        """
        Deletes any subconcepts associated with this concept and additionally this concept if 'delete_self' is True
        If any parentconcepts or relatedconcepts are included then it will only delete the relationship to those concepts but not the concepts themselves
        If any values are passed, then those values as well as the relationship to those values will be deleted

        Note, django will automatically take care of deleting any db models that have a foreign key relationship to the model being deleted 
        (eg: deleting a concept model will also delete all values and relationships), but because we need to manage deleting 
        parent concepts and related concepts and values we have to do that here too

        """

        for subconcept in self.subconcepts:
            concepts_to_delete = Concept.gather_concepts_to_delete(subconcept)
            for key, concept in concepts_to_delete.iteritems():
                models.Concepts.objects.get(pk=key).delete()
<commit_msg>fixes #256 adding a concept to a dropdown now adds all the children concepts as well<commit_after>'''
ARCHES - a program developed to inventory and manage immovable cultural heritage.
Copyright (C) 2013 <NAME> and World Monuments Fund

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
'''

import uuid
import copy
from operator import itemgetter
from operator import methodcaller
from django.db import transaction, connection
from django.db.models import Q
from django.conf import settings
from arches.app.models import models
from arches.app.search.search_engine_factory import SearchEngineFactory
from arches.app.search.elasticsearch_dsl_builder import Match, Query
from arches.app.utils.betterJSONSerializer import JSONSerializer, JSONDeserializer
from django.utils.translation import ugettext as _
import logging


logger = logging.getLogger(__name__)

CORE_CONCEPTS = (
    '00000000-0000-0000-0000-000000000001',
    '00000000-0000-0000-0000-000000000002',
    '00000000-0000-0000-0000-000000000003',
    '00000000-0000-0000-0000-000000000004',
    '00000000-0000-0000-0000-000000000006'
)

class Concept(object):
    def __init__(self, *args, **kwargs):
        self.id = ''
        self.nodetype = ''
        self.legacyoid = ''
        self.relationshiptype = ''
        self.values = []
        self.subconcepts = []
        self.parentconcepts = []
        self.relatedconcepts = []
        self.hassubconcepts = False

        if len(args) != 0:
            if isinstance(args[0], basestring):
                try:
                    uuid.UUID(args[0])
                    self.get(args[0])
                except(ValueError):
                    self.load(JSONDeserializer().deserialize(args[0]))  
            elif isinstance(args[0], dict):
                self.load(args[0])  
            elif isinstance(args[0], object):
                self.load(args[0])  

    def __unicode__(self):
        return ('%s - %s') % (self.get_preflabel().value, self.id)
    
    def load(self, value):
        if isinstance(value, dict):
            self.id = value['id'] if 'id' in value else ''
            self.nodetype = value['nodetype'] if 'nodetype' in value else ''
            self.legacyoid = value['legacyoid'] if 'legacyoid' in value else ''
            self.relationshiptype = value['relationshiptype'] if 'relationshiptype' in value else ''            
            if 'values' in value:
                for val in value['values']:
                    self.addvalue(val)
            if 'subconcepts' in value:
                for subconcept in value['subconcepts']:
                    self.addsubconcept(subconcept)
            if 'parentconcepts' in value:
                for parentconcept in value['parentconcepts']:
                    self.addparent(parentconcept)
            if 'relatedconcepts' in value:
                for relatedconcept in value['relatedconcepts']:
                    self.addrelatedconcept(relatedconcept)

        if isinstance(value, models.Concepts):
            self.id = value.pk
            self.nodetype = value.nodetype_id
            self.legacyoid = value.legacyoid

    def get(self, id='', legacyoid='', include_subconcepts=False, include_parentconcepts=False, include_relatedconcepts=False, exclude=[], include=[], depth_limit=None, up_depth_limit=None, lang=settings.LANGUAGE_CODE, semantic=True, **kwargs):
        if id != '':
            self.load(models.Concepts.objects.get(pk=id))
        elif legacyoid != '':
            self.load(models.Concepts.objects.get(legacyoid=legacyoid))

        if semantic == True:
            pathway_filter = Q(relationtype__category = 'Semantic Relations') | Q(relationtype__category = 'Properties')
        else:
            pathway_filter = Q(relationtype = 'member')

        if self.id != '':
            nodetype = kwargs.pop('nodetype', self.nodetype)
            uplevel = kwargs.pop('uplevel', 0)
            downlevel = kwargs.pop('downlevel', 0)
            depth_limit = depth_limit if depth_limit == None else int(depth_limit)
            up_depth_limit = up_depth_limit if up_depth_limit == None else int(up_depth_limit)

            if include != None:
                if len(include) > 0 and len(exclude) > 0:
                    raise Exception('Only include values for include or exclude, but not both')
                include = include if len(include) != 0 else models.ValueTypes.objects.distinct('category').values_list('category', flat=True)
                include = set(include).difference(exclude)
                exclude = []

                if len(include) > 0:
                    values = models.Values.objects.filter(conceptid = self.id)
                    for value in values:
                        if value.valuetype.category in include:
                            self.values.append(ConceptValue(value))

            hassubconcepts = models.ConceptRelations.objects.filter(Q(conceptidfrom = self.id), pathway_filter, ~Q(relationtype = 'related'))[0:1]
            if len(hassubconcepts) > 0:
                self.hassubconcepts = True

            if include_subconcepts:
                conceptrealations = models.ConceptRelations.objects.filter(Q(conceptidfrom = self.id), pathway_filter, ~Q(relationtype = 'related'))
                if depth_limit == None or downlevel < depth_limit:
                    if depth_limit != None:
                        downlevel = downlevel + 1                
                    for relation in conceptrealations:
                        subconcept = Concept().get(id=relation.conceptidto_id, include_subconcepts=include_subconcepts, 
                            include_parentconcepts=False, include_relatedconcepts=include_relatedconcepts, exclude=exclude, include=include, depth_limit=depth_limit, 
                            up_depth_limit=up_depth_limit, downlevel=downlevel, uplevel=uplevel, nodetype=nodetype, semantic=semantic)
                        subconcept.relationshiptype = relation.relationtype.pk
                        self.subconcepts.append(subconcept)

                    self.subconcepts = sorted(self.subconcepts, key=methodcaller('get_sortkey', lang=lang), reverse=False) 

            if include_parentconcepts:
                conceptrealations = models.ConceptRelations.objects.filter(Q(conceptidto = self.id), pathway_filter, ~Q(relationtype = 'related'))
                if up_depth_limit == None or uplevel < up_depth_limit:
                    if up_depth_limit != None:
                        uplevel = uplevel + 1          
                    for relation in conceptrealations:
                        parentconcept = Concept().get(id=relation.conceptidfrom_id, include_subconcepts=False, 
                            include_parentconcepts=include_parentconcepts, include_relatedconcepts=include_relatedconcepts, 
                            exclude=exclude, include=include, depth_limit=depth_limit, 
                            up_depth_limit=up_depth_limit, downlevel=downlevel, uplevel=uplevel, nodetype=nodetype, semantic=semantic)
                        parentconcept.relationshiptype = relation.relationtype.pk
                        self.parentconcepts.append(parentconcept)

            if include_relatedconcepts:
                conceptrealations = models.ConceptRelations.objects.filter(Q(relationtype = 'related') | Q(relationtype__category = 'Mapping Properties'), Q(conceptidto = self.id) | Q(conceptidfrom = self.id))
                for relation in conceptrealations:
                    if relation.conceptidto_id != self.id:
                        relatedconcept = Concept().get(relation.conceptidto_id, include=['label'])
                        relatedconcept.relationshiptype = relation.relationtype.pk
                        self.relatedconcepts.append(relatedconcept)
                    if relation.conceptidfrom_id != self.id:
                        relatedconcept = Concept().get(relation.conceptidfrom_id, include=['label'])
                        relatedconcept.relationshiptype = relation.relationtype.pk
                        self.relatedconcepts.append(relatedconcept)

        return self
            
    def save(self):
        self.id = self.id if (self.id != '' and self.id != None) else str(uuid.uuid4())
        concept, created = models.Concepts.objects.get_or_create(pk=self.id, defaults={'legacyoid': self.legacyoid if self.legacyoid != '' else self.id, 'nodetype_id': self.nodetype})

        for value in self.values:
            if not isinstance(value, ConceptValue): 
                value = ConceptValue(value)
            value.conceptid = self.id
            value.save()        

        for parentconcept in self.parentconcepts:
            parentconcept.save()
            parentconcept.add_relation(self, parentconcept.relationshiptype)

        for subconcept in self.subconcepts:
            subconcept.save()
            self.add_relation(subconcept, subconcept.relationshiptype)

        for relatedconcept in self.relatedconcepts:
            self.add_relation(relatedconcept, relatedconcept.relationshiptype)
            
            if relatedconcept.relationshiptype == 'member':
                child_concepts = relatedconcept.get(include_subconcepts=True)
                def applyRelationship(concept):
                    for subconcept in concept.subconcepts:
                        concept.add_relation(subconcept, relatedconcept.relationshiptype)
                child_concepts.traverse(applyRelationship)
                

        return concept

    def delete(self, delete_self=False):
        """
        Deletes any subconcepts associated with this concept and additionally this concept if 'delete_self' is True
        If any parentconcepts or relatedconcepts are included then it will only delete the relationship to those concepts but not the concepts themselves
        If any values are passed, then those values as well as the relationship to those values will be deleted

        Note, django will automatically take care of deleting any db models that have a foreign key relationship to the model being deleted 
        (eg: deleting a concept model will also delete all values and relationships), but because we need to manage deleting 
        parent concepts and related concepts and values we have to do that here too

        """

        for subconcept in self.subconcepts:
            concepts_to_delete = Concept.gather_concepts_to_delete(subconcept)
            for key, concept in concepts_to_delete.iteritems():
                models.Concepts.objects.get(pk=key).delete()

{'fraction_non_alphanumeric': 0.08401331084469162, 'fraction_numerical': 0.01709440671012445, 'mean_word_length': 2.3730012300123002, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 20, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '490749', 'n_tokens_mistral': 6251, 'n_tokens_neox': 5478, 'n_words': 1473}