<commit_before>#!/usr/bin/python -OO

# This file is part of Archivematica.
#
# Copyright 2010-2011 Artefactual Systems Inc. <http://artefactual.com>
#
# Archivematica is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Archivematica is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Archivematica.  If not, see <http://www.gnu.org/licenses/>.

# @package Archivematica
# @subpackage Ingest
# @author <NAME> <<EMAIL>>
# @version svn: $Id$
import sys
import shlex
import subprocess
import os
import MySQLdb
import uuid
sys.path.append("/usr/lib/archivematica/archivematicaCommon")
import databaseInterface
from databaseFunctions import insertIntoEvents
from fileOperations import updateFileLocation
from archivematicaFunctions import unicodeToStr
       
if __name__ == '__main__':
    objectsDirectory = sys.argv[1]
    sipUUID =  sys.argv[2]
    date = sys.argv[3]
    taskUUID = sys.argv[4]
    groupType = sys.argv[5]
    groupType = "%%%s%%" % (groupType)
    groupSQL = sys.argv[6]
    groupID = sipUUID
    
    relativeReplacement = "%sobjects/" % (groupType) #"%SIPDirectory%objects/"
    

    #def executeCommand(taskUUID, requiresOutputLock = "no", sInput = "", sOutput = "", sError = "", execute = "", arguments = "", serverConnection = None):
    command = "sanitizeNames \"" + objectsDirectory + "\""
    lines = []
    commandVersion = "sanitizeNames -V"
    version = ""
    try:
        p = subprocess.Popen(shlex.split(command), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        #p.wait()
        output = p.communicate()
        retcode = p.returncode
        
        #print output

        #it executes check for errors
        if retcode != 0:
            print >>sys.stderr, "error code:" + retcode.__str__()
            print output[1]# sError
            quit(retcode)
        lines = output[0].split("\n")
        
        #GET VERSION
        p = subprocess.Popen(shlex.split(commandVersion), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    
        #p.wait()
        output = p.communicate()
        retcode = p.returncode
        
        #it executes check for errors
        if retcode != 0:
            print >>sys.stderr, "Error getting version; error code:" + retcode.__str__()
            print output[1]# sError
            quit(retcode)
        version = output[0].replace("\n", "")
    except OSError, ose:
        print >>sys.stderr, "Execution failed:", ose
        quit(2)

    eventDetail= "program=\"sanitizeNames\"; version=\"" + version + "\""
    for line in lines:
        detoxfiles = line.split(" -> ")
        if len(detoxfiles) > 1 :
            oldfile = detoxfiles[0].split('\n',1)[0]
            newfile = detoxfiles[1]
            #print "line: ", line
            if os.path.isfile(newfile):
                oldfile = oldfile.replace(objectsDirectory, relativeReplacement, 1)
                newfile = newfile.replace(objectsDirectory, relativeReplacement, 1)
                print oldfile, " -> ", newfile 
                
                if groupType == "%SIPDirectory%":
                    updateFileLocation(oldfile, newfile, "name cleanup", date, "prohibited characters removed:" + eventDetail, fileUUID=None, sipUUID=sipUUID)
                elif groupType == "%transferDirectory%":
                    updateFileLocation(oldfile, newfile, "name cleanup", date, "prohibited characters removed:" + eventDetail, fileUUID=None, transferUUID=sipUUID)
                else:
                    print >>sys.stderr, "bad group type", groupType
                    exit(3)
                
            elif os.path.isdir(newfile):
                oldfile = oldfile.replace(objectsDirectory, relativeReplacement, 1) + "/"
                newfile = newfile.replace(objectsDirectory, relativeReplacement, 1) + "/"
                directoryContents = []
                
                sql = "SELECT fileUUID, currentLocation FROM Files WHERE Files.removedTime = 0 AND Files.currentLocation LIKE '" + MySQLdb.escape_string(oldfile).replace("%","\%") + "%' AND " + groupSQL + " = '" + groupID + "';"
                 
                c, sqlLock = databaseInterface.querySQL(sql) 
                row = c.fetchone()
                while row != None:
                    print row
                    fileUUID = row[0] 
                    oldPath = row[1]
                    newPath = unicodeToStr(oldPath).replace(oldfile, newfile, 1)
                    directoryContents.append((fileUUID, oldPath, newPath))
                    row = c.fetchone()
                sqlLock.release()

                print oldfile, " -> ", newfile

                for fileUUID, oldPath, newPath in directoryContents:
                    updateFileLocation(oldPath, newPath, "name cleanup", date, "prohibited characters removed:" + eventDetail, fileUUID=fileUUID)
                    
<commit_msg>Issue 846:	ZippedWithBadNames Test SIP failed at Extract packages<commit_after>#!/usr/bin/python -OO

# This file is part of Archivematica.
#
# Copyright 2010-2011 Artefactual Systems Inc. <http://artefactual.com>
#
# Archivematica is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Archivematica is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Archivematica.  If not, see <http://www.gnu.org/licenses/>.

# @package Archivematica
# @subpackage Ingest
# @author <NAME> <<EMAIL>>
# @version svn: $Id$
import sys
import shlex
import subprocess
import os
import MySQLdb
import uuid
sys.path.append("/usr/lib/archivematica/archivematicaCommon")
import databaseInterface
from databaseFunctions import insertIntoEvents
from fileOperations import updateFileLocation
from archivematicaFunctions import unicodeToStr
       
if __name__ == '__main__':
    objectsDirectory = sys.argv[1]
    sipUUID =  sys.argv[2]
    date = sys.argv[3]
    taskUUID = sys.argv[4]
    groupType = sys.argv[5]
    groupType = "%%%s%%" % (groupType)
    groupSQL = sys.argv[6]
    groupID = sipUUID
    
    relativeReplacement = "%sobjects/" % (groupType) #"%SIPDirectory%objects/"
    

    #def executeCommand(taskUUID, requiresOutputLock = "no", sInput = "", sOutput = "", sError = "", execute = "", arguments = "", serverConnection = None):
    command = "sanitizeNames \"" + objectsDirectory + "\""
    lines = []
    commandVersion = "sanitizeNames -V"
    version = ""
    try:
        p = subprocess.Popen(shlex.split(command), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        #p.wait()
        output = p.communicate()
        retcode = p.returncode
        
        #print output

        #it executes check for errors
        if retcode != 0:
            print >>sys.stderr, "error code:" + retcode.__str__()
            print output[1]# sError
            quit(retcode)
        lines = output[0].split("\n")
        
        #GET VERSION
        p = subprocess.Popen(shlex.split(commandVersion), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    
        #p.wait()
        output = p.communicate()
        retcode = p.returncode
        
        #it executes check for errors
        if retcode != 0:
            print >>sys.stderr, "Error getting version; error code:" + retcode.__str__()
            print output[1]# sError
            quit(retcode)
        version = output[0].replace("\n", "")
    except OSError, ose:
        print >>sys.stderr, "Execution failed:", ose
        quit(2)

    eventDetail= "program=\"sanitizeNames\"; version=\"" + version + "\""
    for line in lines:
        detoxfiles = line.split(" -> ")
        if len(detoxfiles) > 1 :
            oldfile = detoxfiles[0].split('\n',1)[0]
            newfile = detoxfiles[1]
            #print "line: ", line
            if os.path.isfile(newfile):
                oldfile = oldfile.replace(objectsDirectory, relativeReplacement, 1)
                newfile = newfile.replace(objectsDirectory, relativeReplacement, 1)
                print oldfile, " -> ", newfile 
                
                if groupType == "%SIPDirectory%":
                    updateFileLocation(oldfile, newfile, "name cleanup", date, "prohibited characters removed:" + eventDetail, fileUUID=None, sipUUID=sipUUID)
                elif groupType == "%transferDirectory%":
                    updateFileLocation(oldfile, newfile, "name cleanup", date, "prohibited characters removed:" + eventDetail, fileUUID=None, transferUUID=sipUUID)
                else:
                    print >>sys.stderr, "bad group type", groupType
                    exit(3)
                
            elif os.path.isdir(newfile):
                oldfile = oldfile.replace(objectsDirectory, relativeReplacement, 1) + "/"
                newfile = newfile.replace(objectsDirectory, relativeReplacement, 1) + "/"
                directoryContents = []
                
                sql = "SELECT fileUUID, currentLocation FROM Files WHERE Files.removedTime = 0 AND Files.currentLocation LIKE '" + MySQLdb.escape_string(oldfile.replace("\\", "\\\\")).replace("%","\%") + "%' AND " + groupSQL + " = '" + groupID + "';"
                 
                c, sqlLock = databaseInterface.querySQL(sql) 
                row = c.fetchone()
                while row != None:
                    print row
                    fileUUID = row[0] 
                    oldPath = row[1]
                    newPath = unicodeToStr(oldPath).replace(oldfile, newfile, 1)
                    directoryContents.append((fileUUID, oldPath, newPath))
                    row = c.fetchone()
                sqlLock.release()

                print oldfile, " -> ", newfile

                for fileUUID, oldPath, newPath in directoryContents:
                    updateFileLocation(oldPath, newPath, "name cleanup", date, "prohibited characters removed:" + eventDetail, fileUUID=fileUUID)
                    

{'fraction_non_alphanumeric': 0.1092007434944238, 'fraction_numerical': 0.006970260223048327, 'mean_word_length': 2.312096029547553, 'pattern_counts': {'":': 4, '<': 13, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6610100', 'n_tokens_mistral': 3040, 'n_tokens_neox': 2942, 'n_words': 819}