<commit_before>#!/usr/bin/python

# author: jhs
# created: 2013-01-28
# project: Rockefeller Archivematica - Archivists Toolkit Integration

# notes: this script creates a sql script, that can be run against an AT database
#        it will insert digital object records into AT for existing Archival Descriptions objects
#
# inputs:  requires information about the AT database, and the location of the DIP, plus some metadata supplied by the user
#
# first version of this script gets the input from prompts, next version will get it from MCP db.
# 
# example usage:
#
# python at_import.py --host=localhost --port=3306 --dbname="ATTEST" --dbuser=ATuser --dbpass=<PASSWORD> --dip_location="/home/jhs/dip" --dip_name=mydip --atuser=myuser --use_statement="Image-Service" --uri_prefix="http://www.rockarch.org/" 
#

import os
import sys
import MySQLdb
from time import localtime, strftime
import argparse
import logging
sys.path.append("/usr/lib/archivematica/archivematicaCommon")
import archivistsToolkit.atk as atk
import mets
from xml2obj import mets_file
import MySQLdb
import databaseInterface

#global variables
db = None
cursor = None
testMode =0 
base_fv_id = 1

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
logger.addHandler(logging.NullHandler())
logger.addHandler(logging.FileHandler('/tmp/at_upload.log', mode='a'))
    
def recursive_file_gen(mydir):
    for root, dirs, files in os.walk(mydir):
        for file in files:
            yield os.path.join(root, file)

def process_sql(str):
    global cursor 
    if testMode:
        print str
    else:
        cursor.execute(str)            
        newID = cursor.lastrowid
        return newID

def get_user_input():
    print "Archivematica import to AT script"
    print "Welcome\n"
    atdbhost = raw_input("AT database hostname:")
    atdbport = raw_input("AT database port:")
    atdbuser = raw_input("AT database user name:")
    atpass = raw_input("AT database user password:")
    atuser = raw_input("AT username:")
    atdb = raw_input("AT database name:")

    dip_location = raw_input("Location of DIP:")
    dip_name = raw_input("Name of DIP:")

    object_type = raw_input("Object Type:")
    ead_actuate = raw_input("EAD Actuate:")
    ead_show = raw_input("EAD Show:")
    use_statement = raw_input("Use Statement:")

    uri_prefix = raw_input("prefix for uri:")
    #aip = raw_input("Name of mets file:")
    #fileName = raw_input("File name:")
    return atdbhost, atdbport, atdbuser, atpass, atdb, dip_location, dip_name, atuser, object_type, ead_actuate, ead_show, use_statement, uri_prefix

def get_files_from_dip(dip_location, dip_name, dip_uuid):
    #need to find files in objects dir of dip:
    # go to dipLocation/dipName/objects
    # get a directory listing
    # for each item, set fileName and go
    try:
        mydir = dip_location + "objects/"
        mylist = list(recursive_file_gen(mydir))
        
        if len(mylist) > 0:
            return mylist
        else:
            logger.error("no files in " + mydir)
            raise ValueError("cannot find dip")
            exit(2)
    except Exception:
        raise
        exit(3)

def get_pairs(dip_uuid):
    pairs = dict()
    #connect to archivematica db, make a set of pairs from pairs table
   
    sql = """SELECT fileUUID, resourceId, resourceComponentId from AtkDIPObjectResourcePairing where dipUUID = '{}'""".format(dip_uuid)
    c, sqlLock = databaseInterface.querySQL(sql)
    dbresult = c.fetchall()
    for item in dbresult:
        ids = dict()
        ids['rid'] = item[1]
        ids['rcid'] = item[2]
        pairs[item[0]] =  ids
    sqlLock.release()
    return pairs

def delete_pairs(dip_uuid):
    sql = """delete from AtkDIPObjectResourcePairing where dipUUID = '{}'""".format(dip_uuid)
    c, sqlLock = databaseInterface.querySQL(sql)
    sqlLock.release()
      
def upload_to_atk(mylist, atuser, ead_actuate, ead_show, object_type, use_statement, uri_prefix, dip_uuid, access_conditions, use_conditions, restrictions, dip_location):
    
    logger.info("inputs: actuate '{}' show '{}' type '{}'  use_statement '{}' use_conditions '{}'".format(ead_actuate, ead_show, object_type, use_statement, use_conditions))
    if uri_prefix[-1] == '/':
        uri_prefix = uri_prefix + dip_uuid + "/objects/"
    else:
        uri_prefix = uri_prefix + "/" + dip_uuid + "/objects/"
        
    #get mets object if needed
    mets = None
    if restrictions == 'premis' or len(access_conditions) == 0 or len(use_conditions) == 0:
        try:
            logger.debug("looking for mets: {}".format(dip_uuid))
            mets_source = dip_location + 'METS.' + dip_uuid + '.xml'
            mets = mets_file(mets_source)
            logger.debug("found mets file")
        except Exception:
            raise
            exit(4)
            
    global db
    global cursor
    db = atk.connect_db(args.atdbhost, args.atdbport, args.atdbuser, args.atdbpass, args.atdb)
    cursor = db.cursor()
    
    #get a list of all the items in this collection
    #col = atk.collection_list(db, resource_id)
    #logger.debug("got collection_list: {}".format(len(col)))
    sql0 = "select max(fileVersionId) from fileversions"
    logger.info('sql0: ' + sql0)
    cursor.execute(sql0)
    data = cursor.fetchone()
    if not data[0]:
        newfVID = 1
    else:
        newfVID = int(data[0]) 
    logger.info('base file version id found is ' + str(data[0]))
    global base_fv_id 
    base_fv_id = newfVID        

    pairs = get_pairs(dip_uuid)
    #TODO test to make sure we got some pairs
    
    for f in mylist:
        base_fv_id+=1 
        logger.info( 'using ' + f)
        file_name = os.path.basename(f)
        logger.info('file_name is ' + file_name)
        uuid = file_name[0:36]
        #aipUUID = aip[5:41]
        access_restrictions = None
        access_rightsGrantedNote = None
        use_restrictions = None
        use_rightsGrantedNote = None
        #logging.info("looking for mets")
        if mets and mets[uuid]:
            #get premis info from mets
            for premis in mets[uuid]['premis']:
                logger.debug("{} rights = {}, note={}".format(premis, mets[uuid]['premis'][premis]['restriction'],mets[uuid]['premis'][premis]['rightsGrantedNote']))
                if premis == 'Disseminate':
                    access_restrictions = mets[uuid]['premis']['Disseminate']['restriction']
                    access_rightsGrantedNote = mets[uuid]['premis']['Disseminate']['rightsGrantedNote']
                if premis == 'Publish':
                    use_restrictions = mets[uuid]['premis']['Publish']['restriction']
                    use_rightsGrantedNote = mets[uuid]['premis']['Publish']['rightsGrantedNote']
        try:
            container1 = file_name[44:47]
            container2 = file_name[48:53]
        except:
            logger.error('file name does not have container ids in it')
            exit(5)
        logger.debug ("determine restrictions")<commit_msg>require lower case in premis act statements<commit_after>#!/usr/bin/python

# author: jhs
# created: 2013-01-28
# project: Rockefeller Archivematica - Archivists Toolkit Integration

# notes: this script creates a sql script, that can be run against an AT database
#        it will insert digital object records into AT for existing Archival Descriptions objects
#
# inputs:  requires information about the AT database, and the location of the DIP, plus some metadata supplied by the user
#
# first version of this script gets the input from prompts, next version will get it from MCP db.
# 
# example usage:
#
# python at_import.py --host=localhost --port=3306 --dbname="ATTEST" --dbuser=ATuser --dbpass=hello --dip_location="/home/jhs/dip" --dip_name=mydip --atuser=myuser --use_statement="Image-Service" --uri_prefix="http://www.rockarch.org/" 
#

import os
import sys
import MySQLdb
from time import localtime, strftime
import argparse
import logging
sys.path.append("/usr/lib/archivematica/archivematicaCommon")
import archivistsToolkit.atk as atk
import mets
from xml2obj import mets_file
import MySQLdb
import databaseInterface

#global variables
db = None
cursor = None
testMode =0 
base_fv_id = 1

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
logger.addHandler(logging.NullHandler())
logger.addHandler(logging.FileHandler('/tmp/at_upload.log', mode='a'))
    
def recursive_file_gen(mydir):
    for root, dirs, files in os.walk(mydir):
        for file in files:
            yield os.path.join(root, file)

def process_sql(str):
    global cursor 
    if testMode:
        print str
    else:
        cursor.execute(str)            
        newID = cursor.lastrowid
        return newID

def get_user_input():
    print "Archivematica import to AT script"
    print "Welcome\n"
    atdbhost = raw_input("AT database hostname:")
    atdbport = raw_input("AT database port:")
    atdbuser = raw_input("AT database user name:")
    atpass = raw_input("AT database user password:")
    atuser = raw_input("AT username:")
    atdb = raw_input("AT database name:")

    dip_location = raw_input("Location of DIP:")
    dip_name = raw_input("Name of DIP:")

    object_type = raw_input("Object Type:")
    ead_actuate = raw_input("EAD Actuate:")
    ead_show = raw_input("EAD Show:")
    use_statement = raw_input("Use Statement:")

    uri_prefix = raw_input("prefix for uri:")
    #aip = raw_input("Name of mets file:")
    #fileName = raw_input("File name:")
    return atdbhost, atdbport, atdbuser, atpass, atdb, dip_location, dip_name, atuser, object_type, ead_actuate, ead_show, use_statement, uri_prefix

def get_files_from_dip(dip_location, dip_name, dip_uuid):
    #need to find files in objects dir of dip:
    # go to dipLocation/dipName/objects
    # get a directory listing
    # for each item, set fileName and go
    try:
        mydir = dip_location + "objects/"
        mylist = list(recursive_file_gen(mydir))
        
        if len(mylist) > 0:
            return mylist
        else:
            logger.error("no files in " + mydir)
            raise ValueError("cannot find dip")
            exit(2)
    except Exception:
        raise
        exit(3)

def get_pairs(dip_uuid):
    pairs = dict()
    #connect to archivematica db, make a set of pairs from pairs table
   
    sql = """SELECT fileUUID, resourceId, resourceComponentId from AtkDIPObjectResourcePairing where dipUUID = '{}'""".format(dip_uuid)
    c, sqlLock = databaseInterface.querySQL(sql)
    dbresult = c.fetchall()
    for item in dbresult:
        ids = dict()
        ids['rid'] = item[1]
        ids['rcid'] = item[2]
        pairs[item[0]] =  ids
    sqlLock.release()
    return pairs

def delete_pairs(dip_uuid):
    sql = """delete from AtkDIPObjectResourcePairing where dipUUID = '{}'""".format(dip_uuid)
    c, sqlLock = databaseInterface.querySQL(sql)
    sqlLock.release()
      
def upload_to_atk(mylist, atuser, ead_actuate, ead_show, object_type, use_statement, uri_prefix, dip_uuid, access_conditions, use_conditions, restrictions, dip_location):
    
    logger.info("inputs: actuate '{}' show '{}' type '{}'  use_statement '{}' use_conditions '{}'".format(ead_actuate, ead_show, object_type, use_statement, use_conditions))
    if uri_prefix[-1] == '/':
        uri_prefix = uri_prefix + dip_uuid + "/objects/"
    else:
        uri_prefix = uri_prefix + "/" + dip_uuid + "/objects/"
        
    #get mets object if needed
    mets = None
    if restrictions == 'premis' or len(access_conditions) == 0 or len(use_conditions) == 0:
        try:
            logger.debug("looking for mets: {}".format(dip_uuid))
            mets_source = dip_location + 'METS.' + dip_uuid + '.xml'
            mets = mets_file(mets_source)
            logger.debug("found mets file")
        except Exception:
            raise
            exit(4)
            
    global db
    global cursor
    db = atk.connect_db(args.atdbhost, args.atdbport, args.atdbuser, args.atdbpass, args.atdb)
    cursor = db.cursor()
    
    #get a list of all the items in this collection
    #col = atk.collection_list(db, resource_id)
    #logger.debug("got collection_list: {}".format(len(col)))
    sql0 = "select max(fileVersionId) from fileversions"
    logger.info('sql0: ' + sql0)
    cursor.execute(sql0)
    data = cursor.fetchone()
    if not data[0]:
        newfVID = 1
    else:
        newfVID = int(data[0]) 
    logger.info('base file version id found is ' + str(data[0]))
    global base_fv_id 
    base_fv_id = newfVID        

    pairs = get_pairs(dip_uuid)
    #TODO test to make sure we got some pairs
    
    for f in mylist:
        base_fv_id+=1 
        logger.info( 'using ' + f)
        file_name = os.path.basename(f)
        logger.info('file_name is ' + file_name)
        uuid = file_name[0:36]
        #aipUUID = aip[5:41]
        access_restrictions = None
        access_rightsGrantedNote = None
        use_restrictions = None
        use_rightsGrantedNote = None
        #logging.info("looking for mets")
        if mets and mets[uuid]:
            #get premis info from mets
            for premis in mets[uuid]['premis']:
                logger.debug("{} rights = {}, note={}".format(premis, mets[uuid]['premis'][premis]['restriction'],mets[uuid]['premis'][premis]['rightsGrantedNote']))
                if premis == 'disseminate':
                    access_restrictions = mets[uuid]['premis']['disseminate']['restriction']
                    access_rightsGrantedNote = mets[uuid]['premis']['disseminate']['rightsGrantedNote']
                if premis == 'publish':
                    use_restrictions = mets[uuid]['premis']['publish']['restriction']
                    use_rightsGrantedNote = mets[uuid]['premis']['publish']['rightsGrantedNote']
        try:
            container1 = file_name[44:47]
            container2 = file_name[48:53]
        except:
            logger.error('file name does not have container ids in it')
            exit(5)
        logger.debug ("determine restrictions")
{'fraction_non_alphanumeric': 0.11223982382609932, 'fraction_numerical': 0.007245862044469703, 'mean_word_length': 2.94010635320459, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6985325', 'n_tokens_mistral': 4532, 'n_tokens_neox': 4347, 'n_words': 1153}