<commit_before># -*- coding: utf-8 -*-

# BSD 3-Clause License
#
# Copyright (c) 2017, ColoredInsaneAsylums
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# * Neither the name of the copyright holder nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# DETAILS:
# File Name: technical.py
# Description: This file contains source code for the core functionality of the
#              technical schema workflow.
#
# Creator: <NAME> (milindsiddhanti at utexas dot edu)
#
# IMPORT NEEDED MODULES
import csv
import sys
import re

from datetime import datetime
from time import localtime, time, strftime
from subprocess import PIPE, Popen

from metadatautilspkg.globalvars import *
from metadatautilspkg.errorcodes import *
from metadatautilspkg.technical import *
from metadatautilspkg.dbfunctions import *
from metadatautilspkg.premis import *
from metadatautilspkg.metadatautils import *

def main():

    # Verify whether ImageMagick is installed in the system or not. Throws error if ImageMagick is not installed.
    output, error, exitcode = runCmd('identify -version')
    imgversion = output.decode('utf-8').split('\n')

    # remove null values from the list
    while '' in imgversion:
        imgversion.remove('')

    imgver = []
    for str in imgversion:
            if ': ' in str:
                imgver.append(str)
            else:
                globalvars.technicalErrorList.append([errorcodes.ERROR_INSTALL_IMAGEMAGICK["message"]])
                print_error(errorcodes.ERROR_INSTALL_IMAGEMAGICK["message"])
                errorCSV()
                exit(errorcodes.ERROR_INSTALL_IMAGEMAGICK["code"])

    version = {}
    for item in imgver:
        key, value = item.split(": ")
        key = key.strip(" ")
        value = value.strip(" ")
        version[key] = value

    if 'Version' in version:
        ver = version['Version']
        if 'ImageMagick' not in ver:
            globalvars.technicalErrorList.append([errorcodes.ERROR_INSTALL_IMAGEMAGICK["message"]])
            print_error(errorcodes.ERROR_INSTALL_IMAGEMAGICK["message"])
            errorCSV()
            exit(errorcodes.ERROR_INSTALL_IMAGEMAGICK["code"])
    else:
        globalvars.technicalErrorList.append([errorcodes.ERROR_INSTALL_IMAGEMAGICK["message"]])
        print_error(errorcodes.ERROR_INSTALL_IMAGEMAGICK["message"])
        errorCSV()
        exit(errorcodes.ERROR_INSTALL_IMAGEMAGICK["code"])

    argParser = defineCommandLineOptions()
    parseCommandLineArgs(argParser, sys.argv[1:])

    print_info("quiet mode: ", globalvars.quietMode)

    if globalvars.batchMode == True:  # Batch mode. Read and validate CSV file.
    # Read CSV file contents into globalvars.technicalList.
        try:
            # Open the CSV file in read-only mode.
            csvFileHandle = open (globalvars.csvFile, "r")
        except IOError as ioErrorCsvRead:
            print_error(ioErrorCsvRead)
            print_error(errorcodes.ERROR_CANNOT_OPEN_CSV_FILE["message"])
            exit(errorcodes.ERROR_CANNOT_OPEN_CSV_FILE["code"])

        # CSV file successfully opened.
        csvReader = csv.reader(csvFileHandle)  # Create an iterable object from the
                                                # CSV file using csv.reader().

        # Extract the first row to check if it is a header.
        firstRow = next(csvReader, None)

        print_info("Checking the header row. Header: {}".format(firstRow))
        if firstRow == None or len(firstRow) == 0:  # This also serves as a check for an empty CSV file
            print_error(errorcodes.ERROR_INVALID_HEADER_ROW["message"])
            globalvars.technicalErrorList.append([errorcodes.ERROR_INVALID_HEADER_ROW["message"]])
            errorCSV()
            exit(errorcodes.ERROR_INVALID_HEADER_ROW["code"])

        # This for loop reads and checks the format (i.errorcodes., presence of at least two
        # columns per row) of the CSV file, and populates 'globalvars.technicalList'

        rowNum = 1
        for row in csvReader:
            globalvars.technicalList.append(row)
            rowNum += 1

        csvFileHandle.close()  # Close the CSV file as it will not be needed
                            # from this point on.

    print_info("Filepath to extract technical information for: {}".format(len(globalvars.technicalList)))

    # READ-IN THE LABEL DICTIONARY
    globalvars.labels = readLabelDictionary()
    print_info("The following labels will be used for labeling metadata items in the database records:")
    print_info(globalvars.labels)

    # READ-IN THE CONTROLLED VOCABULARY
    globalvars.vocab = readControlledVocabulary()

    # CREATE DATABASE CONNECTION
    dbParams = init_db()  # TODO: there needs to be a check to determine if the
                        # database connection was successful or not.
    globalvars.dbHandle = dbParams["handle"]
    globalvars.dbCollection = dbParams["collection_name"]
    # print("db collection name: ".format(globalvars.dbCollection))

    # PROCESS ALL RECORDS
    for row in globalvars.technicalList:
        series = row[0]
        subseries = row[0]

        print_info("Technical data to be extracted for the files with series '{}' and subseries '{}' and extension '{}'".format(series, subseries, globalvars.ext))

        # function to extract technical properties of the files in technicalFileInfo.
        technicalStatus = technicalRecord(series, subseries, ver)

def errorCSV():
    # WRITE ALL ROWS THAT COULD NOT BE PROCESSED TO A CSV FILE
    if len(globalvars.technicalErrorList) > 0:
        errorsCSVFileName = ("technical_profile_errors_" + strftime("%Y-%m-%d_%H%M%S", localtime(time())) + ".csv")
<commit_msg>changes to technical.py script, added the arrange: <label> functionality to the script.<commit_after># -*- coding: utf-8 -*-

# BSD 3-Clause License
#
# Copyright (c) 2017, ColoredInsaneAsylums
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# * Neither the name of the copyright holder nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# DETAILS:
# File Name: technical.py
# Description: This file contains source code for the core functionality of the
#              technical schema workflow.
#
# Creator: <NAME> (milindsiddhanti at utexas dot edu)
#
# IMPORT NEEDED MODULES
import csv
import sys
import re

from datetime import datetime
from time import localtime, time, strftime
from subprocess import PIPE, Popen

from metadatautilspkg.globalvars import *
from metadatautilspkg.errorcodes import *
from metadatautilspkg.technical import *
from metadatautilspkg.dbfunctions import *
from metadatautilspkg.premis import *
from metadatautilspkg.metadatautils import *

def main():

    # Verify whether ImageMagick is installed in the system or not. Throws error if ImageMagick is not installed.
    output, error, exitcode = runCmd('identify -version')
    imgversion = output.decode('utf-8').split('\n')

    # remove null values from the list
    while '' in imgversion:
        imgversion.remove('')

    imgver = []
    for str in imgversion:
            if ': ' in str:
                imgver.append(str)
            else:
                globalvars.technicalErrorList.append([errorcodes.ERROR_INSTALL_IMAGEMAGICK["message"]])
                print_error(errorcodes.ERROR_INSTALL_IMAGEMAGICK["message"])
                errorCSV()
                exit(errorcodes.ERROR_INSTALL_IMAGEMAGICK["code"])

    version = {}
    for item in imgver:
        key, value = item.split(": ")
        key = key.strip(" ")
        value = value.strip(" ")
        version[key] = value

    if 'Version' in version:
        ver = version['Version']
        if 'ImageMagick' not in ver:
            globalvars.technicalErrorList.append([errorcodes.ERROR_INSTALL_IMAGEMAGICK["message"]])
            print_error(errorcodes.ERROR_INSTALL_IMAGEMAGICK["message"])
            errorCSV()
            exit(errorcodes.ERROR_INSTALL_IMAGEMAGICK["code"])
    else:
        globalvars.technicalErrorList.append([errorcodes.ERROR_INSTALL_IMAGEMAGICK["message"]])
        print_error(errorcodes.ERROR_INSTALL_IMAGEMAGICK["message"])
        errorCSV()
        exit(errorcodes.ERROR_INSTALL_IMAGEMAGICK["code"])

    argParser = defineCommandLineOptions()
    parseCommandLineArgs(argParser, sys.argv[1:])

    print_info("quiet mode: ", globalvars.quietMode)

    if globalvars.batchMode == True:  # Batch mode. Read and validate CSV file.
    # Read CSV file contents into globalvars.technicalList.
        try:
            # Open the CSV file in read-only mode.
            csvFileHandle = open (globalvars.csvFile, "r")
        except IOError as ioErrorCsvRead:
            print_error(ioErrorCsvRead)
            print_error(errorcodes.ERROR_CANNOT_OPEN_CSV_FILE["message"])
            exit(errorcodes.ERROR_CANNOT_OPEN_CSV_FILE["code"])

        # CSV file successfully opened.
        csvReader = csv.reader(csvFileHandle)  # Create an iterable object from the
                                                # CSV file using csv.reader().

        # Extract the first row to check if it is a header.
        firstRow = next(csvReader, None)

        print_info("Checking the header row. Header: {}".format(firstRow))
        if firstRow == None or len(firstRow) == 0:  # This also serves as a check for an empty CSV file
            print_error(errorcodes.ERROR_INVALID_HEADER_ROW["message"])
            globalvars.technicalErrorList.append([errorcodes.ERROR_INVALID_HEADER_ROW["message"]])
            errorCSV()
            exit(errorcodes.ERROR_INVALID_HEADER_ROW["code"])

        # Extract Arrange info from header row
        numArrangementInfoCols = 0
        arrangementInfoTags = {}
        for col in firstRow:
            if col.startswith(globalvars.ARRANGEMENT_INFO_MARKER):
                numArrangementInfoCols += 1
                if 'name' in col:
                    arrangementInfoTags[numArrangementInfoCols] = col.split(':')[-1]
                else:
                    arrangementInfoTags[numArrangementInfoCols] = col.split(':')[-1] + globalvars.ARRANGEMENT_INFO_LABEL_SUFFIX

            else:
                print_error("The column names should be with prefix {}".format(globalvars.ARRANGEMENT_INFO_MARKER))

        globalvars.technicalErrorList.append(firstRow + ["Comments"])

        # This for loop reads and checks the format (i.errorcodes., presence of at least two
        # columns per row) of the CSV file, and populates 'globalvars.technicalList'
        rowNum = 1
        for row in csvReader:
            if len(row) < globalvars.minNumCols:  # Check if the row has AT LEAST globalvars.minNumCols elements.
                print_error("Row number {} in {} is not a valid input. This row will not be processed.".format(rowNum, globalvars.csvFile))
                globalvars.adminerrorList.append(row + ["Not a valid input"])
                errorCSV()
            else:
                globalvars.technicalList.append(row)
            rowNum += 1

        csvFileHandle.close()  # Close the CSV file as it will not be needed from this point on.

    print_info("Series and Sub-series input in the csv is of length: {}".format(len(globalvars.technicalList)))

    # READ-IN THE LABEL DICTIONARY
    globalvars.labels = readLabelDictionary()
    print_info("The following labels will be used for labeling metadata items in the database records:")
    print_info(globalvars.labels)

    # READ-IN THE CONTROLLED VOCABULARY
    globalvars.vocab = readControlledVocabulary()

    # CREATE DATABASE CONNECTION
    dbParams = init_db()  # TODO: there needs to be a check to determine if the
                        # database connection was successful or not.
    globalvars.dbHandle = dbParams["handle"]
    globalvars.dbCollection = dbParams["collection_name"]
    # print("db collection name: ".format(globalvars.dbCollection))

    # PROCESS ALL RECORDS
    for row in globalvars.technicalList:

        arrangementInfo = {}

        for arrangementId in range(1, numArrangementInfoCols + 1):
            arrangementInfo[arrangementInfoTags[arrangementId]] = row[arrangementId - 1]

        series = arrangementInfo['seriesLabel']
        subseries = arrangementInfo['sub-seriesLabel']

        print_info("Technical data to be extracted for the files with series and subseries '{}' and extension '{}'".format(arrangementInfo, globalvars.ext))

        # function to extract technical properties of the files in technicalFileInfo.
        technicalStatus = technicalRecord(series, subseries, ver)

def errorCSV():
    # WRITE ALL ROWS THAT COULD NOT BE PROCESSED TO A CSV FILE
    if len(globalvars.technicalErrorList) > 0:
        errorsCSVFileName = ("technical_profile_errors_" + strftime("%Y-%m-%d_%H%M%S", localtime(time())) + ".csv")

{'fraction_non_alphanumeric': 0.08108108108108109, 'fraction_numerical': 0.0021337126600284497, 'mean_word_length': 3.1455373894398284, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5848232', 'n_tokens_mistral': 4319, 'n_tokens_neox': 4066, 'n_words': 1402}