<commit_before>import os
import sys 
import uuid
import traceback
import unicodecsv
import concepts
import string
from time import time
from os import listdir
from os.path import isfile, join, exists
from django.db import connection, transaction
from django.conf import settings
from arches.app.models import models
from arches.app.models.concept import Concept, ConceptValue
from .. import utils

class Lookups(object):
    def __init__(self, *args, **kwargs):
        self.lookup = {}
        self.concept_relationships = []

    def add_lookup(self, concept=None, rownum=None):
        self.lookup[concept.legacyoid] = {'concept': concept, 'rownum': rownum}

    def get_lookup(self, legacyoid=''):
        if legacyoid in self.lookup:
            return self.lookup[legacyoid]['concept']
        else:
            raise Exception('Legacyoid (%s) not found.  Make sure your ParentConceptid in the csv file references a previously saved concept.' % (legacyoid))

    def add_relationship(self, source=None, type=None, target=None, rownum=None):
        self.concept_relationships.append({'source': source, 'type': type, 'target': target, 'rownum': rownum})

def load_authority_files(break_on_error=True):
    cursor = connection.cursor()
    file_list = []
    errors = []
    
    for path in settings.CONCEPT_SCHEME_LOCATIONS:
        if os.path.exists(path):
            print '\nLOADING AUTHORITY FILES (%s)' % (path)
            print '-----------------------'            
            
            for f in listdir(path):
                if isfile(join(path,f)) and '.values.csv' not in f and f != 'ENTITY_TYPE_X_ADOC.csv' and f[-4:] == '.csv':
                    file_list.append(f)
            
            file_list.sort()

            count = 1
            for file_name in file_list:
                errors = errors + load_authority_file(cursor, path, file_name)
                if count > 10:
                    pass
                    #break
                count = count + 1
            errors = errors + create_link_to_entity_types(cursor, path)
        else:
            errors.append('\n\nPath in settings.CONCEPT_SCHEME_LOCATIONS doesn\'t exist (%s)' % (path))  

    utils.write_to_file(os.path.join(settings.PACKAGE_ROOT, 'logs', 'authority_file_errors.txt'), '')
    if len(errors) > 0:
        utils.write_to_file(os.path.join(settings.PACKAGE_ROOT, 'logs', 'authority_file_errors.txt'), '\n'.join(errors))
        print "\n\nERROR: There were errors in some of the authority files."
        print "Please review the errors at %s, \ncorrect the errors and then rerun this script." % (os.path.join(settings.PACKAGE_ROOT, 'logs', 'authority_file_errors.txt'))
        if break_on_error:
            sys.exit(101)


def load_authority_file(cursor, path_to_authority_files, filename):
    print filename.upper()    

    start = time()
    value_types = models.ValueTypes.objects.all()
    filepath = os.path.join(path_to_authority_files, filename)
    unicodecsv.field_size_limit(sys.maxint)
    errors = []
    lookups = Lookups()

    #create nodes for each authority document file and relate them to the authority document node in the concept schema
    auth_doc_file_name = str(filename)
    display_file_name = string.capwords(auth_doc_file_name.replace('_',' ').replace('AUTHORITY DOCUMENT.csv', '').strip())
    if auth_doc_file_name.upper() != 'ARCHES RESOURCE CROSS-REFERENCE RELATIONSHIP TYPES.E32.CSV':
        concept = Concept()
        concept.id = str(uuid.uuid4())
        concept.legacyoid = auth_doc_file_name
        concept.addvalue({'value':display_file_name, 'language': 'en-us', 'type': 'prefLabel', 'datatype': 'text', 'category': 'label'})
        scheme_id = concept.id
    else:
        concept = Concept().get(id = '00000000-0000-0000-0000-000000000005')
        concept.legacyoid = 'ARCHES RESOURCE CROSS-REFERENCE RELATIONSHIP TYPES.E32.csv'
        scheme_id = concept.id

    lookups.add_relationship(source='00000000-0000-0000-0000-000000000004', type='narrower', target=concept.id)
    lookups.add_lookup(concept=concept, rownum=0)
    
    try:
        with open(filepath, 'rU') as f:
            rows = unicodecsv.DictReader(f, fieldnames=['CONCEPTID','PREFLABEL','ALTLABELS','PARENTCONCEPTID','CONCEPTTYPE','PROVIDER'], 
                encoding='utf-8-sig', delimiter=',', restkey='ADDITIONAL', restval='MISSING')
            rows.next() # skip header row
            for row in rows:              
                try:
                    if 'MISSING' in row:
                        raise Exception('The row wasn\'t parsed properly. Missing %s' % (row['MISSING']))
                    else:
                        concept = Concept()
                        concept.id = str(uuid.uuid4())
                        concept.legacyoid = row[u'CONCEPTID']
                        concept.addvalue({'value':row[u'PREFLABEL'], 'language': 'en-us', 'type': 'prefLabel', 'datatype': 'text', 'category': 'label'})
                        if row[u'ALTLABELS'] != '':
                            altlabel_list = row[u'ALTLABELS'].split(';')
                            for altlabel in altlabel_list:
                                concept.addvalue({'value':altlabel, 'language': 'en-us', 'type': 'altLabel', 'datatype': 'text', 'category': 'label'})                          
                        
                        #relationshiptype = 'includes' if row[u'CONCEPTTYPE'].upper() == 'INDEX' else ('has collection' if row[u'CONCEPTTYPE'].upper() == 'COLLECTOR' else '')
                        relationshiptype = 'narrower'
                        lookups.add_relationship(source=lookups.get_lookup(legacyoid=row[u'PARENTCONCEPTID']).id, type=relationshiptype, target=concept.id, rownum=rows.line_num)
                        
                        if row[u'PARENTCONCEPTID'] == '' or (row[u'CONCEPTTYPE'].upper() != 'INDEX' and row[u'CONCEPTTYPE'].upper() != 'COLLECTOR'):
                            raise Exception('The row has invalid values.')

                        lookups.add_lookup(concept=concept, rownum=rows.line_num)    <commit_msg>Fix problem where Resource to Resource Relationship Types was appearing duplicated in the RDM.<commit_after>import os
import sys 
import uuid
import traceback
import unicodecsv
import concepts
import string
from time import time
from os import listdir
from os.path import isfile, join, exists
from django.db import connection, transaction
from django.conf import settings
from arches.app.models import models
from arches.app.models.concept import Concept, ConceptValue
from .. import utils

class Lookups(object):
    def __init__(self, *args, **kwargs):
        self.lookup = {}
        self.concept_relationships = []

    def add_lookup(self, concept=None, rownum=None):
        self.lookup[concept.legacyoid] = {'concept': concept, 'rownum': rownum}

    def get_lookup(self, legacyoid=''):
        if legacyoid in self.lookup:
            return self.lookup[legacyoid]['concept']
        else:
            raise Exception('Legacyoid (%s) not found.  Make sure your ParentConceptid in the csv file references a previously saved concept.' % (legacyoid))

    def add_relationship(self, source=None, type=None, target=None, rownum=None):
        self.concept_relationships.append({'source': source, 'type': type, 'target': target, 'rownum': rownum})

def load_authority_files(break_on_error=True):
    cursor = connection.cursor()
    file_list = []
    errors = []
    
    for path in settings.CONCEPT_SCHEME_LOCATIONS:
        if os.path.exists(path):
            print '\nLOADING AUTHORITY FILES (%s)' % (path)
            print '-----------------------'            
            
            for f in listdir(path):
                if isfile(join(path,f)) and '.values.csv' not in f and f != 'ENTITY_TYPE_X_ADOC.csv' and f[-4:] == '.csv':
                    file_list.append(f)
            
            file_list.sort()

            count = 1
            for file_name in file_list:
                errors = errors + load_authority_file(cursor, path, file_name)
                if count > 10:
                    pass
                    #break
                count = count + 1
            errors = errors + create_link_to_entity_types(cursor, path)
        else:
            errors.append('\n\nPath in settings.CONCEPT_SCHEME_LOCATIONS doesn\'t exist (%s)' % (path))  

    utils.write_to_file(os.path.join(settings.PACKAGE_ROOT, 'logs', 'authority_file_errors.txt'), '')
    if len(errors) > 0:
        utils.write_to_file(os.path.join(settings.PACKAGE_ROOT, 'logs', 'authority_file_errors.txt'), '\n'.join(errors))
        print "\n\nERROR: There were errors in some of the authority files."
        print "Please review the errors at %s, \ncorrect the errors and then rerun this script." % (os.path.join(settings.PACKAGE_ROOT, 'logs', 'authority_file_errors.txt'))
        if break_on_error:
            sys.exit(101)


def load_authority_file(cursor, path_to_authority_files, filename):
    print filename.upper()    

    start = time()
    value_types = models.ValueTypes.objects.all()
    filepath = os.path.join(path_to_authority_files, filename)
    unicodecsv.field_size_limit(sys.maxint)
    errors = []
    lookups = Lookups()

    #create nodes for each authority document file and relate them to the authority document node in the concept schema
    auth_doc_file_name = str(filename)
    display_file_name = string.capwords(auth_doc_file_name.replace('_',' ').replace('AUTHORITY DOCUMENT.csv', '').strip())
    if auth_doc_file_name.upper() != 'ARCHES RESOURCE CROSS-REFERENCE RELATIONSHIP TYPES.E32.CSV':
        concept = Concept()
        concept.id = str(uuid.uuid4())
        concept.legacyoid = auth_doc_file_name
        concept.addvalue({'value':display_file_name, 'language': 'en-us', 'type': 'prefLabel', 'datatype': 'text', 'category': 'label'})
        scheme_id = concept.id
        lookups.add_relationship(source='00000000-0000-0000-0000-000000000004', type='narrower', target=concept.id)
    else:
        concept = Concept().get(id = '00000000-0000-0000-0000-000000000005')
        concept.legacyoid = 'ARCHES RESOURCE CROSS-REFERENCE RELATIONSHIP TYPES.E32.csv'
        scheme_id = concept.id

    lookups.add_lookup(concept=concept, rownum=0)
    
    try:
        with open(filepath, 'rU') as f:
            rows = unicodecsv.DictReader(f, fieldnames=['CONCEPTID','PREFLABEL','ALTLABELS','PARENTCONCEPTID','CONCEPTTYPE','PROVIDER'], 
                encoding='utf-8-sig', delimiter=',', restkey='ADDITIONAL', restval='MISSING')
            rows.next() # skip header row
            for row in rows:              
                try:
                    if 'MISSING' in row:
                        raise Exception('The row wasn\'t parsed properly. Missing %s' % (row['MISSING']))
                    else:
                        concept = Concept()
                        concept.id = str(uuid.uuid4())
                        concept.legacyoid = row[u'CONCEPTID']
                        concept.addvalue({'value':row[u'PREFLABEL'], 'language': 'en-us', 'type': 'prefLabel', 'datatype': 'text', 'category': 'label'})
                        if row[u'ALTLABELS'] != '':
                            altlabel_list = row[u'ALTLABELS'].split(';')
                            for altlabel in altlabel_list:
                                concept.addvalue({'value':altlabel, 'language': 'en-us', 'type': 'altLabel', 'datatype': 'text', 'category': 'label'})                          
                        
                        #relationshiptype = 'includes' if row[u'CONCEPTTYPE'].upper() == 'INDEX' else ('has collection' if row[u'CONCEPTTYPE'].upper() == 'COLLECTOR' else '')
                        relationshiptype = 'narrower'
                        lookups.add_relationship(source=lookups.get_lookup(legacyoid=row[u'PARENTCONCEPTID']).id, type=relationshiptype, target=concept.id, rownum=rows.line_num)
                        
                        if row[u'PARENTCONCEPTID'] == '' or (row[u'CONCEPTTYPE'].upper() != 'INDEX' and row[u'CONCEPTTYPE'].upper() != 'COLLECTOR'):
                            raise Exception('The row has invalid values.')

                        lookups.add_lookup(concept=concept, rownum=rows.line_num)    
{'fraction_non_alphanumeric': 0.12446142590033331, 'fraction_numerical': 0.013169661003170473, 'mean_word_length': 2.414376908132112, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 8, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1789606', 'n_tokens_mistral': 3805, 'n_tokens_neox': 3546, 'n_words': 825}