<commit_before>import os
import uuid
import importlib
import csv
import datetime
from time import time
from django.conf import settings
from django.db import connection, transaction
from django.contrib.auth.models import User
from django.forms.models import model_to_dict
from django.core.management.base import BaseCommand, CommandError
from arches.app.models.entity import Entity
from arches.app.models.resource import Resource
from arches.app.models.models import Concept
from arches.app.models.models import Value
from arches.app.models.models import ResourceXResource
from arches.app.models.concept import Concept
from arches.app.search.search_engine_factory import SearchEngineFactory
from arches.management.commands import utils
from optparse import make_option
from formats.archesfile import ArchesReader
from formats.archesjson import JsonReader
from formats.shpfile import ShapeReader
from arches.app.models.tile import Tile
from arches.app.models.models import ResourceInstance
from arches.app.models.models import FunctionXGraph
from arches.app.models.models import ResourceXResource
from arches.app.models.models import NodeGroup
from django.core.exceptions import ValidationError

def pre_import(tile, graph_id):
    for function in get_function_class_instances(tile, graph_id):
        try:
            function.on_import(tile)
        except NotImplementedError:
            pass
    return tile

def get_function_class_instances(tile, graph_id):
    ret = []
    functions = FunctionXGraph.objects.filter(graph_id=graph_id, config__triggering_nodegroups__contains=[tile['nodegroup_id']])
    for function in functions:
        mod_path = function.function.modulename.replace('.py', '')
        module = importlib.import_module('arches.app.functions.%s' % mod_path)
        func = getattr(module, function.function.classname)(function.config, tile['nodegroup_id'])
        ret.append(func)
    return ret

def validate_business_data(business_data):
    errors = []
    if type(business_data) == dict and business_data['resources']:
        for resource in business_data['resources']:
            graph_id = resource['resourceinstance']['graph_id']
            for tile in resource['tiles']:
                try:
                    pre_import(tile, graph_id)
                except ValidationError as e:
                    errors.append(e.args)
    return errors

def import_business_data(business_data):
    results = {'resource_count': 0, 'resources_saved': 0, 'tile_count': 0, 'tiles_saved': 0, 'relationship_count': 0}

    try:
        results['resource_count'] = len(business_data['resources'])
        for resource in business_data['resources']:
            if resource['resourceinstance'] != None:
                resource['resourceinstance']['resourceinstanceid'] = uuid.UUID(str(resource['resourceinstance']['resourceinstanceid']))
                resource['resourceinstance']['graphid'] = uuid.UUID(str(resource['resourceinstance']['graph_id']))

                resourceinstance = ResourceInstance.objects.update_or_create(
                    resourceinstanceid = resource['resourceinstance']['resourceinstanceid'],
                    graph_id = resource['resourceinstance']['graphid'],
                    resourceinstancesecurity = resource['resourceinstance']['resourceinstancesecurity']
                )

            if len(ResourceInstance.objects.filter(resourceinstanceid=resource['resourceinstance']['resourceinstanceid'])) == 1:
                results['resources_saved'] += 1

            if resource['tiles'] != []:
                results['tile_count'] += len(resource['tiles'])
                for tile in resource['tiles']:
                    tile['parenttile_id'] = uuid.UUID(str(tile['parenttile_id'])) if tile['parenttile_id'] else None
                    tile['nodegroup_id'] = NodeGroup(uuid.UUID(str(tile['nodegroup_id']))) if tile['nodegroup_id'] else None
                    tile['resourceinstance_id'] = ResourceInstance(uuid.UUID(str(tile['resourceinstance_id'])))
                    tile['tileid'] = uuid.UUID(str(tile['tileid']))
                    tile = Tile.objects.update_or_create(
                         resourceinstance = tile['resourceinstance_id'],
                        parenttile = Tile(uuid.UUID(str(tile['parenttile_id']))) if tile['parenttile_id'] else None,
                        nodegroup = tile['nodegroup_id'],
                        tileid = tile['tileid'],
                        data = tile['data']
                    )
                    if len(Tile.objects.filter(tileid=tile[0].tileid)) == 1:
                        results['tiles_saved'] += 1
        print results

    except (KeyError, TypeError) as e:
        print e, 'resources not in business data'

    try:
        results['relations_count'] = len(business_data['relations'])
        for relation in business_data['relations']:
            relation['resourcexid'] = uuid.UUID(str(relation['resourcexid']))
            relation['resourceinstanceidfrom'] = ResourceInstance(uuid.UUID(str(relation['resourceinstanceidfrom'])))
            relation['resourceinstanceidto'] = ResourceInstance(uuid.UUID(str(relation['resourceinstanceidto'])))
            relation['relationshiptype'] = uuid.UUID(str(relation['relationshiptype']))

            relation = ResourceXResource.objects.update_or_create(
                resourcexid = relation['resourcexid'],
                resourceinstanceidfrom = relation['resourceinstanceidfrom'],
                resourceinstanceidto = relation['resourceinstanceidto'],
                notes = relation['notes'],
                relationshiptype = relation['relationshiptype'],
                datestarted = relation['datestarted'],
                dateended = relation['dateended']
            )
            # print vars(relation)
            relation.update_or_create()
    except (KeyError, TypeError) as e:
        print e, 'relations not in business data'

    return results
<commit_msg>aded resourceimportreporter class re #1269<commit_after>import os
import uuid
import importlib
import csv
import datetime
from time import time
from django.conf import settings
from django.db import connection, transaction
from django.contrib.auth.models import User
from django.forms.models import model_to_dict
from django.core.management.base import BaseCommand, CommandError
from arches.app.models.entity import Entity
from arches.app.models.resource import Resource
from arches.app.models.models import Concept
from arches.app.models.models import Value
from arches.app.models.models import ResourceXResource
from arches.app.models.concept import Concept
from arches.app.search.search_engine_factory import SearchEngineFactory
from arches.management.commands import utils
from optparse import make_option
from formats.archesfile import ArchesReader
from formats.archesjson import JsonReader
from formats.shpfile import ShapeReader
from arches.app.models.tile import Tile
from arches.app.models.models import ResourceInstance
from arches.app.models.models import FunctionXGraph
from arches.app.models.models import ResourceXResource
from arches.app.models.models import NodeGroup
from django.core.exceptions import ValidationError

def pre_import(tile, graph_id):
    for function in get_function_class_instances(tile, graph_id):
        try:
            function.on_import(tile)
        except NotImplementedError:
            pass
    return tile

def get_function_class_instances(tile, graph_id):
    ret = []
    functions = FunctionXGraph.objects.filter(graph_id=graph_id, config__triggering_nodegroups__contains=[tile['nodegroup_id']])
    for function in functions:
        mod_path = function.function.modulename.replace('.py', '')
        module = importlib.import_module('arches.app.functions.%s' % mod_path)
        func = getattr(module, function.function.classname)(function.config, tile['nodegroup_id'])
        ret.append(func)
    return ret

def validate_business_data(business_data):
    errors = []
    if type(business_data) == dict and business_data['resources']:
        for resource in business_data['resources']:
            graph_id = resource['resourceinstance']['graph_id']
            for tile in resource['tiles']:
                try:
                    pre_import(tile, graph_id)
                except ValidationError as e:
                    errors.append(e.args)
    return errors


class ResourceImportReporter:
    def __init__(self, business_data):
        self.resources = 0
        self.tiles = 0
        self.resources_saved = 0
        self.tiles_saved = 0
        self.relations_saved = 0
        self.relations = 0

        if 'resources' in business_data:
            self.resources = business_data['resources']

        if 'relations' in business_data:
            self.relations = business_data['relations']

    def update_resources_saved(count=1):
        self.resources_saved += count
        print self.resources_saved

    def update_tiles_saved(count=1):
        self.tiles_saved += count
        print self.tiles_saved

    def update_relations_saved(count=1):
        self.tiles_saved += count
        print self.tiles_saved

    def report_results():
        print "{0}".format(self.resources)

def import_business_data(business_data):
    try:
        for resource in business_data['resources']:
            reporter = ResourceImportReporter(business_data)
            if resource['resourceinstance'] != None:
                resource['resourceinstance']['resourceinstanceid'] = uuid.UUID(str(resource['resourceinstance']['resourceinstanceid']))
                resource['resourceinstance']['graphid'] = uuid.UUID(str(resource['resourceinstance']['graph_id']))

                resourceinstance, created = ResourceInstance.objects.update_or_create(
                    resourceinstanceid = resource['resourceinstance']['resourceinstanceid'],
                    graph_id = resource['resourceinstance']['graphid'],
                    resourceinstancesecurity = resource['resourceinstance']['resourceinstancesecurity']
                )

            if len(ResourceInstance.objects.filter(resourceinstanceid=resource['resourceinstance']['resourceinstanceid'])) == 1:
                reporter.update_resources_saved()

            if resource['tiles'] != []:
                for tile in resource['tiles']:
                    Tile.objects.update_or_create(
                        resourceinstance = resourceinstance,
                        parenttile = Tile(uuid.UUID(str(tile['parenttile_id']))) if tile['parenttile_id'] else None,
                        nodegroup = NodeGroup(uuid.UUID(str(tile['nodegroup_id']))) if tile['nodegroup_id'] else None,
                        tileid = uuid.UUID(str(tile['tileid'])),
                        data = tile['data']
                    )
                    if len(Tile.objects.filter(tileid=tile[0].tileid)) == 1:
                        reporter.update_tiles_saved()

    except (KeyError, TypeError) as e:
        print e, 'resources not in business data'

    try:
        for relation in business_data['relations']:
            relation['resourcexid'] = uuid.UUID(str(relation['resourcexid']))
            relation['resourceinstanceidfrom'] = ResourceInstance(uuid.UUID(str(relation['resourceinstanceidfrom'])))
            relation['resourceinstanceidto'] = ResourceInstance(uuid.UUID(str(relation['resourceinstanceidto'])))
            relation['relationshiptype'] = uuid.UUID(str(relation['relationshiptype']))

            relation = ResourceXResource.objects.update_or_create(
                resourcexid = relation['resourcexid'],
                resourceinstanceidfrom = relation['resourceinstanceidfrom'],
                resourceinstanceidto = relation['resourceinstanceidto'],
                notes = relation['notes'],
                relationshiptype = relation['relationshiptype'],
                datestarted = relation['datestarted'],
                dateended = relation['dateended']
            )
            # print vars(relation)
            relation.update_or_create()

    except (KeyError, TypeError) as e:
        print e, 'relations not in business data'
{'fraction_non_alphanumeric': 0.0995973374969184, 'fraction_numerical': 0.0022187525680006574, 'mean_word_length': 3.0019730351857943, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3397485', 'n_tokens_mistral': 3158, 'n_tokens_neox': 3029, 'n_words': 664}