<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Automate Transfers.

Helper script to automate running transfers through Archivematica.
"""

from __future__ import print_function, unicode_literals

import ast
import atexit
import base64
import logging
import os
import shutil
import subprocess
import sys
import time

import requests
from six.moves import configparser
from sqlalchemy.orm.exc import NoResultFound

# Allow execution as an executable and the script to be run at package level
# by ensuring that it can see itself.
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from transfers import defaults, errors, loggingconfig, models, utils
from transfers.amclient import AMClient
from transfers.transferargs import get_parser
from transfers.utils import fsencode, fsdecode

# Directory for various processing decisions, below.
THIS_DIR = os.path.abspath(os.path.dirname(__file__))

# Setup module level logging.
LOGGER = logging.getLogger('transfers')


def setup_automation_execution(pid_file):
    """Setup procedures for transfer.py."""
    atexit.register(manage_automation_execution, pid_file)


def manage_automation_execution(pid_file):
    """Cleanup procedures for transfer.py."""
    LOGGER.info("Running post-execution clean-up. Exiting script")
    os.remove(pid_file)
    models.cleanup_session()


def create_db_session(config_file):
    """Create and return a database session."""
    models.init_session(
        get_setting(config_file, 'databasefile',
                    os.path.join(THIS_DIR, 'transfers.db')))
    return models.Session()


def get_setting(config_file, setting, default=None):
    """Get an option value from the configuration file."""
    config = configparser.SafeConfigParser()
    section = "transfers"
    try:
        config.read(config_file)
        cfg = config.get(section, setting)
        LOGGER.info("Configuration values read for %s: %s", setting, cfg)
        return cfg
    except configparser.NoOptionError:
        LOGGER.warning("No option provided for %s", setting)
        return default
    except configparser.NoSectionError:
        LOGGER.warning("No section: %s in %s", section, config.sections())
        return default


def get_status(am_url, am_user, am_api_key, ss_url, ss_user, ss_api_key,
               unit_uuid, unit_type, hide_on_complete=False,
               delete_on_complete=False):
    """
    Get status of the SIP or Transfer with unit_uuid.

    :param str unit_uuid: UUID of the unit to query for.
    :param str unit_type: 'ingest' or 'transfer'
    :param bool hide_on_complete: Hide the unit in the dashboard if COMPLETE
    :returns: Dict with status of the unit from Archivematica or None.
    """
    # Get status
    url = "{}/api/{}/status/{}/".format(am_url, unit_type, unit_uuid)
    params = {'username': am_user, 'api_key': am_api_key}
    unit_info = utils._call_url_json(url, params)
    if isinstance(unit_info, int):
        if errors.error_lookup(unit_info) is not None:
            return errors.error_lookup(unit_info)
    # If complete, hide in dashboard
    if hide_on_complete and unit_info and \
            unit_info.get('status') == 'COMPLETE':
        LOGGER.info('Hiding %s %s in dashboard', unit_type, unit_uuid)
        url = "{}/api/{}/{}/delete/".format(am_url, unit_type, unit_uuid)
        LOGGER.debug('Method: DELETE; URL: %s; params: %s;', url, params)
        response = requests.delete(url, params=params)
        LOGGER.debug('Response: %s', response)
    # If Transfer is complete, get the SIP's status
    if unit_info and unit_type == 'transfer' and \
        unit_info.get('status') == 'COMPLETE' and \
            unit_info.get('sip_uuid') != 'BACKLOG':
        LOGGER.info('%s is a complete transfer, fetching SIP %s status.',
                    unit_uuid, unit_info.get('sip_uuid'))
        # Update DB to refer to this one
        unit = models.retrieve_unit_by_type_and_uuid(
            uuid=unit_uuid, unit_type=unit_type)
        models.update_unit_type_and_uuid(
            unit=unit, unit_type="ingest",
            uuid=unit_info.get('sip_uuid'))
        # Get SIP status
        url = "{}/api/ingest/status/{}/".\
            format(am_url, unit_info.get('sip_uuid'))
        unit_info = utils._call_url_json(url, params)
        if isinstance(unit_info, int):
            if errors.error_lookup(unit_info) is not None:
                return errors.error_lookup(unit_info)
        # If complete, hide in dashboard
        if hide_on_complete and unit_info and \
                unit_info.get('status') == 'COMPLETE':
            LOGGER.info('Hiding SIP %s in dashboard', unit.uuid)
            url = "{}/api/ingest/{}/delete/".format(am_url, unit.uuid)
            LOGGER.debug('Method: DELETE; URL: %s; params: %s;', url, params)
            response = requests.delete(url, params=params)
            LOGGER.debug('Response: %s', response)
        # If complete and SIP status is 'UPLOADED', delete transfer source
        # files
        if delete_on_complete and unit_info and \
                unit_info.get('status') == 'COMPLETE':
            am = AMClient(ss_url=ss_url, ss_user_name=ss_user,
                          ss_api_key=ss_api_key, package_uuid=unit.uuid)
            response = am.get_package_details()
            if response.get('status') == 'UPLOADED':
                LOGGER.info('Deleting source files for SIP %s from watched '
                            'directory', unit.uuid)
                try:
                    shutil.rmtree(unit.path)
                    LOGGER.info('Source files deleted for SIP %s '
                                'deleted', unit.uuid)
                except OSError as e:
                    LOGGER.warning('Error deleting source files: %s. If '
                                   'running this module remotely the '
                                   'script might not have access to the '
                                   'transfer source', e)
    return unit_info


def get_accession_id(dirname):
    """
    Call get-accession-number and return literal_eval stdout as accession ID.

    get-accession-number should be in the same directory as transfer.py. Its
    only output to stdout should be the accession number surrounded by
    quotes.  Eg. "accession number"

    :param str dirname: Directory name of folder to become transfer
    :returns: accession number or None.
    """
    script_path = os.path.join(THIS_DIR, 'get-accession-number')
    try:
        p = subprocess.Popen(
            [script_path, dirname], stdin=subprocess.PIPE,
            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    except OSError as err:
        LOGGER.warning('Error when trying to run %s', script_path)
        return None
    output, err = p.communicate()
    if p.returncode != 0:
        LOGGER.error('Error running %s %s: RC: %s; stdout: %s; stderr: %s',
                     script_path, dirname, p.returncode, output, err)
        return None
    output = fsdecode(output)
    try:
        return ast.literal_eval(output)
    except (ValueError, SyntaxError) as err:
        LOGGER.warning(
            "Unable to parse output from %s. Output: %s. %s",
            script_path, output, err)
        return None


def run_pre_transfer_scripts(config_file, transfer_path, transfer_type):
    """Wrapper for the run_scripts function. Pre-transfer functions want to
    modify the transfer itself, therefore the transfer path sent by the
    calling function function should at least be a valid one. If run_scripts
    results in an OSError exception then the calling function should take
    responsibility for working with that.
    """
    if not os.path.isdir(transfer_path):
        LOGGER.error(
            "Invalid transfer path for the pre-transfer scripts to work with")
    else:
        run_scripts(
            'pre-transfer',
            config_file,
            transfer_path,
            transfer_type,
        )


def run_scripts(directory, config_file, *args):
    """
    Run all executable scripts in directory relative to this file.

    :param str directory: Dir in the same folder as this file to run scripts
    :param args: All other parameters will be passed to called scripts.
    :return: None
    """
    directory = os.path.join(THIS_DIR, directory)
    if not os.path.isdir(directory):<commit_msg>Correct the behavior of Invalid transfer path (#96)<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Automate Transfers.

Helper script to automate running transfers through Archivematica.
"""

from __future__ import print_function, unicode_literals

import ast
import atexit
import base64
import logging
import os
import shutil
import subprocess
import sys
import time

import requests
from six.moves import configparser
from sqlalchemy.orm.exc import NoResultFound

# Allow execution as an executable and the script to be run at package level
# by ensuring that it can see itself.
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from transfers import defaults, errors, loggingconfig, models, utils
from transfers.amclient import AMClient
from transfers.transferargs import get_parser
from transfers.utils import fsencode, fsdecode

# Directory for various processing decisions, below.
THIS_DIR = os.path.abspath(os.path.dirname(__file__))

# Setup module level logging.
LOGGER = logging.getLogger('transfers')


def setup_automation_execution(pid_file):
    """Setup procedures for transfer.py."""
    atexit.register(manage_automation_execution, pid_file)


def manage_automation_execution(pid_file):
    """Cleanup procedures for transfer.py."""
    LOGGER.info("Running post-execution clean-up. Exiting script")
    os.remove(pid_file)
    models.cleanup_session()


def create_db_session(config_file):
    """Create and return a database session."""
    models.init_session(
        get_setting(config_file, 'databasefile',
                    os.path.join(THIS_DIR, 'transfers.db')))
    return models.Session()


def get_setting(config_file, setting, default=None):
    """Get an option value from the configuration file."""
    config = configparser.SafeConfigParser()
    section = "transfers"
    try:
        config.read(config_file)
        cfg = config.get(section, setting)
        LOGGER.info("Configuration values read for %s: %s", setting, cfg)
        return cfg
    except configparser.NoOptionError:
        LOGGER.warning("No option provided for %s", setting)
        return default
    except configparser.NoSectionError:
        LOGGER.warning("No section: %s in %s", section, config.sections())
        return default


def get_status(am_url, am_user, am_api_key, ss_url, ss_user, ss_api_key,
               unit_uuid, unit_type, hide_on_complete=False,
               delete_on_complete=False):
    """
    Get status of the SIP or Transfer with unit_uuid.

    :param str unit_uuid: UUID of the unit to query for.
    :param str unit_type: 'ingest' or 'transfer'
    :param bool hide_on_complete: Hide the unit in the dashboard if COMPLETE
    :returns: Dict with status of the unit from Archivematica or None.
    """
    # Get status
    url = "{}/api/{}/status/{}/".format(am_url, unit_type, unit_uuid)
    params = {'username': am_user, 'api_key': am_api_key}
    unit_info = utils._call_url_json(url, params)
    if isinstance(unit_info, int):
        if errors.error_lookup(unit_info) is not None:
            return errors.error_lookup(unit_info)
    # If complete, hide in dashboard
    if hide_on_complete and unit_info and \
            unit_info.get('status') == 'COMPLETE':
        LOGGER.info('Hiding %s %s in dashboard', unit_type, unit_uuid)
        url = "{}/api/{}/{}/delete/".format(am_url, unit_type, unit_uuid)
        LOGGER.debug('Method: DELETE; URL: %s; params: %s;', url, params)
        response = requests.delete(url, params=params)
        LOGGER.debug('Response: %s', response)
    # If Transfer is complete, get the SIP's status
    if unit_info and unit_type == 'transfer' and \
        unit_info.get('status') == 'COMPLETE' and \
            unit_info.get('sip_uuid') != 'BACKLOG':
        LOGGER.info('%s is a complete transfer, fetching SIP %s status.',
                    unit_uuid, unit_info.get('sip_uuid'))
        # Update DB to refer to this one
        unit = models.retrieve_unit_by_type_and_uuid(
            uuid=unit_uuid, unit_type=unit_type)
        models.update_unit_type_and_uuid(
            unit=unit, unit_type="ingest",
            uuid=unit_info.get('sip_uuid'))
        # Get SIP status
        url = "{}/api/ingest/status/{}/".\
            format(am_url, unit_info.get('sip_uuid'))
        unit_info = utils._call_url_json(url, params)
        if isinstance(unit_info, int):
            if errors.error_lookup(unit_info) is not None:
                return errors.error_lookup(unit_info)
        # If complete, hide in dashboard
        if hide_on_complete and unit_info and \
                unit_info.get('status') == 'COMPLETE':
            LOGGER.info('Hiding SIP %s in dashboard', unit.uuid)
            url = "{}/api/ingest/{}/delete/".format(am_url, unit.uuid)
            LOGGER.debug('Method: DELETE; URL: %s; params: %s;', url, params)
            response = requests.delete(url, params=params)
            LOGGER.debug('Response: %s', response)
        # If complete and SIP status is 'UPLOADED', delete transfer source
        # files
        if delete_on_complete and unit_info and \
                unit_info.get('status') == 'COMPLETE':
            am = AMClient(ss_url=ss_url, ss_user_name=ss_user,
                          ss_api_key=ss_api_key, package_uuid=unit.uuid)
            response = am.get_package_details()
            if response.get('status') == 'UPLOADED':
                LOGGER.info('Deleting source files for SIP %s from watched '
                            'directory', unit.uuid)
                try:
                    shutil.rmtree(unit.path)
                    LOGGER.info('Source files deleted for SIP %s '
                                'deleted', unit.uuid)
                except OSError as e:
                    LOGGER.warning('Error deleting source files: %s. If '
                                   'running this module remotely the '
                                   'script might not have access to the '
                                   'transfer source', e)
    return unit_info


def get_accession_id(dirname):
    """
    Call get-accession-number and return literal_eval stdout as accession ID.

    get-accession-number should be in the same directory as transfer.py. Its
    only output to stdout should be the accession number surrounded by
    quotes.  Eg. "accession number"

    :param str dirname: Directory name of folder to become transfer
    :returns: accession number or None.
    """
    script_path = os.path.join(THIS_DIR, 'get-accession-number')
    try:
        p = subprocess.Popen(
            [script_path, dirname], stdin=subprocess.PIPE,
            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    except OSError as err:
        LOGGER.warning('Error when trying to run %s', script_path)
        return None
    output, err = p.communicate()
    if p.returncode != 0:
        LOGGER.error('Error running %s %s: RC: %s; stdout: %s; stderr: %s',
                     script_path, dirname, p.returncode, output, err)
        return None
    output = fsdecode(output)
    try:
        return ast.literal_eval(output)
    except (ValueError, SyntaxError) as err:
        LOGGER.warning(
            "Unable to parse output from %s. Output: %s. %s",
            script_path, output, err)
        return None


def run_pre_transfer_scripts(config_file, transfer_path, transfer_type):
    """Wrapper for the run_scripts function. Pre-transfer functions want to
    modify the transfer itself, therefore the transfer path sent by the
    calling function function should at least be a valid one. If run_scripts
    results in an OSError exception then the calling function should take
    responsibility for working with that.
    """
    if not os.path.exists(transfer_path):
        LOGGER.error(
            "Invalid transfer path for the pre-transfer scripts to work with")
    else:
        run_scripts(
            'pre-transfer',
            config_file,
            transfer_path,
            transfer_type,
        )


def run_scripts(directory, config_file, *args):
    """
    Run all executable scripts in directory relative to this file.

    :param str directory: Dir in the same folder as this file to run scripts
    :param args: All other parameters will be passed to called scripts.
    :return: None
    """
    directory = os.path.join(THIS_DIR, directory)
    if not os.path.isdir(directory):
{'fraction_non_alphanumeric': 0.09524659408650128, 'fraction_numerical': 0.0005949193884228687, 'mean_word_length': 2.633808906182447, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3999758', 'n_tokens_mistral': 4900, 'n_tokens_neox': 4825, 'n_words': 1462}