<commit_before>#!/usr/bin/python

# This file is part of Archivematica.
#
# Archivematica is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
# Archivematica is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Archivematica.  If not, see <http://www.gnu.org/licenses/>.

# @package Archivematica
# @subpackage Ingest
# @author <NAME> <<EMAIL>>
# @version svn: $Id$

#!/usr/bin/python

from archivematicaFreeSpaceChecker import checkSpace
from archivematicaLoadConfig import loadConfig
import sys
import os.path
import os
import logging
import subprocess
import shlex
import uuid
import lxml.etree as etree
from premisXMLlinker import xmlNormalize 
#from premisXMLlinker import 

#Delete the normalized file if free space below x number of bytes
spaceThreshold="10240"
archivmaticaVars=loadConfig("/etc/transcoder/transcoderConfig.conf")

#CONFIGURE THE FOLLOWING DIRECTORIES
accessFileDirectory = ""
fileDirectory = ""

#CONFIGURE THE FOLLOWING APPLICATION PATHS
#formatPoliciesPath = "/mnt/userver910/archivematica2/includes/archivematica/formatPolicies"
formatPoliciesPath = archivmaticaVars["formatPoliciesPath"]
convertPath = archivmaticaVars["convertPath"]
ffmpegPath = archivmaticaVars["ffmpegPath"]
theoraPath = archivmaticaVars["theoraPath"]
unoconvPath = archivmaticaVars["unoconvPath"]
unoconvAlternativePath = archivmaticaVars["unoconvAlternativePath"]
#xenaPath = archivmaticaVars["xenaPath"]
transcoderScriptsDir = archivmaticaVars["transcoderScriptsDir"]

#SET THE DEFAULT COMMAND
defaultCommand = "echo No default normalization tool defined."

#this script is passed fileIn, uuid
fileIn = sys.argv[1]
fileUUID = sys.argv[2]
accesspath = sys.argv[3]
xmlStuff = sys.argv[4] #yes/no
edate = ""
eid = ""
objectsPath = ""
logsPath = ""
if xmlStuff == "yes":
    edate = sys.argv[5]
    eid = sys.argv[6]
    objectsPath = sys.argv[7]
    logsPath = sys.argv[8]    



#get file name and extension
s = fileIn
#get indexes for python string array
#index of next char after last /
x1 = s.rfind('/')+1
#index of last .
x2 = s.rfind('.')
#index of next char after last .
x2mod = x2+1
#length of s
sLen = len(s)

if x2 < x1:
    x2mod = 0

fileTitle = s[x1:x2]
fileExtension = s[x2mod:sLen]
fileDirectory = s[:x1]
fileFullName = fileDirectory + fileTitle + "." + fileExtension

print >>sys.stderr, "\nNORMALIZING: " + fileTitle + "." + fileExtension + " {" + fileUUID + "}"

def findDirectory(root, tag=None, text=None):
    ret = []
    if (not tag) and (not text):
    #return all
        for element in root:
            ret.append(element)
    else:
        if tag:
            if not text:
                #match the tag
                for element in root:
                  if element.tag == tag:
                    ret.append(element)
            else:
                #match tag and text
                for element in root:
                  if element.tag == tag and element.text == text:
                    ret.append(element)
        else:
            #match the text
            for element in root:
                if element.text == text:
                  ret.append(element)
    return ret

def fillAttrib(attrib, var, fileExtension):
    tree = etree.parse(formatPoliciesPath + "/" + fileExtension.upper() + ".xml")
    root = tree.getroot()
#  print(etree.tostring(root))
 
    varsxml = findDirectory(root, "inherit")
    if varsxml[0].text :
        return fillAttrib( attrib, var, varsxml[0].text )
    
    varsxml = findDirectory(root, attrib)
    for varxml in varsxml:
        var.append(varxml.text)
#  print var

inherit = []
accessFormat = []
preservationFormat = []
accessConversionCommand = []
preservationConversionCommand =  []

def executeCommand(command, newUUID=""):
    #Replace replacement strings
    replacementDic = { \
    "%convertPath%": convertPath, \
    "%ffmpegPath%": ffmpegPath, \
    "%theoraPath%": theoraPath, \
    "%unoconvPath%": unoconvPath, \
    "%unoconvAlternativePath%": unoconvAlternativePath, \
    "%fileExtension%": fileExtension, \
    "%fileFullName%": fileFullName, \
    "%accessFileDirectory%": accesspath + fileUUID + "-", \
    "%preservationFileDirectory%": fileDirectory + newUUID, \
    "%fileDirectory%": fileDirectory,\
    "%fileTitle%": fileTitle, \
    "%normalizationScriptsDir%": transcoderScriptsDir, \
    "%transcoderScriptsDir%": transcoderScriptsDir, \
    "%accessFormat%": accessFormat[0].lower(), \
    "%preservationFormat%": preservationFormat[0].lower(), \
    "%fileUUID%": fileUUID , \
    "%objectsPath%": objectsPath, \
    "%eventUUID%": eid , \
    "%edate%": edate, \
    "%logsPath%": logsPath}
    <commit_msg>Related to issue Issue 379: 	Access copies overwritten when original files have the same name. Undoing previous change.<commit_after>#!/usr/bin/python

# This file is part of Archivematica.
#
# Archivematica is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
# Archivematica is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Archivematica.  If not, see <http://www.gnu.org/licenses/>.

# @package Archivematica
# @subpackage Ingest
# @author <NAME> <<EMAIL>>
# @version svn: $Id$

#!/usr/bin/python

from archivematicaFreeSpaceChecker import checkSpace
from archivematicaLoadConfig import loadConfig
import sys
import os.path
import os
import logging
import subprocess
import shlex
import uuid
import lxml.etree as etree
from premisXMLlinker import xmlNormalize 
#from premisXMLlinker import 

#Delete the normalized file if free space below x number of bytes
spaceThreshold="10240"
archivmaticaVars=loadConfig("/etc/transcoder/transcoderConfig.conf")

#CONFIGURE THE FOLLOWING DIRECTORIES
accessFileDirectory = ""
fileDirectory = ""

#CONFIGURE THE FOLLOWING APPLICATION PATHS
#formatPoliciesPath = "/mnt/userver910/archivematica2/includes/archivematica/formatPolicies"
formatPoliciesPath = archivmaticaVars["formatPoliciesPath"]
convertPath = archivmaticaVars["convertPath"]
ffmpegPath = archivmaticaVars["ffmpegPath"]
theoraPath = archivmaticaVars["theoraPath"]
unoconvPath = archivmaticaVars["unoconvPath"]
unoconvAlternativePath = archivmaticaVars["unoconvAlternativePath"]
#xenaPath = archivmaticaVars["xenaPath"]
transcoderScriptsDir = archivmaticaVars["transcoderScriptsDir"]

#SET THE DEFAULT COMMAND
defaultCommand = "echo No default normalization tool defined."

#this script is passed fileIn, uuid
fileIn = sys.argv[1]
fileUUID = sys.argv[2]
accesspath = sys.argv[3]
xmlStuff = sys.argv[4] #yes/no
edate = ""
eid = ""
objectsPath = ""
logsPath = ""
if xmlStuff == "yes":
    edate = sys.argv[5]
    eid = sys.argv[6]
    objectsPath = sys.argv[7]
    logsPath = sys.argv[8]    



#get file name and extension
s = fileIn
#get indexes for python string array
#index of next char after last /
x1 = s.rfind('/')+1
#index of last .
x2 = s.rfind('.')
#index of next char after last .
x2mod = x2+1
#length of s
sLen = len(s)

if x2 < x1:
    x2mod = 0

fileTitle = s[x1:x2]
fileExtension = s[x2mod:sLen]
fileDirectory = s[:x1]
fileFullName = fileDirectory + fileTitle + "." + fileExtension

print >>sys.stderr, "\nNORMALIZING: " + fileTitle + "." + fileExtension + " {" + fileUUID + "}"

def findDirectory(root, tag=None, text=None):
    ret = []
    if (not tag) and (not text):
    #return all
        for element in root:
            ret.append(element)
    else:
        if tag:
            if not text:
                #match the tag
                for element in root:
                  if element.tag == tag:
                    ret.append(element)
            else:
                #match tag and text
                for element in root:
                  if element.tag == tag and element.text == text:
                    ret.append(element)
        else:
            #match the text
            for element in root:
                if element.text == text:
                  ret.append(element)
    return ret

def fillAttrib(attrib, var, fileExtension):
    tree = etree.parse(formatPoliciesPath + "/" + fileExtension.upper() + ".xml")
    root = tree.getroot()
#  print(etree.tostring(root))
 
    varsxml = findDirectory(root, "inherit")
    if varsxml[0].text :
        return fillAttrib( attrib, var, varsxml[0].text )
    
    varsxml = findDirectory(root, attrib)
    for varxml in varsxml:
        var.append(varxml.text)
#  print var

inherit = []
accessFormat = []
preservationFormat = []
accessConversionCommand = []
preservationConversionCommand =  []

def executeCommand(command, newUUID=""):
    #Replace replacement strings
    replacementDic = { \
    "%convertPath%": convertPath, \
    "%ffmpegPath%": ffmpegPath, \
    "%theoraPath%": theoraPath, \
    "%unoconvPath%": unoconvPath, \
    "%unoconvAlternativePath%": unoconvAlternativePath, \
    "%fileExtension%": fileExtension, \
    "%fileFullName%": fileFullName, \
    "%accessFileDirectory%": accesspath, \
    "%preservationFileDirectory%": fileDirectory + newUUID, \
    "%fileDirectory%": fileDirectory,\
    "%fileTitle%": fileTitle, \
    "%normalizationScriptsDir%": transcoderScriptsDir, \
    "%transcoderScriptsDir%": transcoderScriptsDir, \
    "%accessFormat%": accessFormat[0].lower(), \
    "%preservationFormat%": preservationFormat[0].lower(), \
    "%fileUUID%": fileUUID , \
    "%objectsPath%": objectsPath, \
    "%eventUUID%": eid , \
    "%edate%": edate, \
    "%logsPath%": logsPath}
    
{'fraction_non_alphanumeric': 0.11144843795906376, 'fraction_numerical': 0.007345020076388209, 'mean_word_length': 3.4711033274956216, 'pattern_counts': {'":': 42, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 24}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1861626', 'n_tokens_mistral': 3196, 'n_tokens_neox': 3197, 'n_words': 898}