<commit_before>__author__ = '<NAME> <<EMAIL>>'
__version__ = '0.1.0'

import os, os.path
import subprocess
import tempfile
import botocore.utils, botocore.exceptions
import boto3
import time
import logging, logging.handlers
import cupocore
import pymongo.errors


# TODO-refactor: Move old archive detection into own method, and add unique path detection, so not only triggered when adding new archives.
# TODO-ratelimit: #1 Add network rate limiting
# TODO-backupscount: #2 Add a way of specifying the amount of redundant backups that should be kept
# TODO-backupsage: #3 Specify the minimum amount of time that a backup should be kept for if there are more than <min amount> of backups remaining.


# Only the *files* in a given directory are archived, not the subdirectories.
# The contents of the subdirectories live in archives of their own (except for any directories that *they* contain)
# This means that only the changed files in a given directory need to be checked - and that if a file in a
# sub-sub-subdirectory is changed, the whole parent directory doesn't need to be re-uploaded.
# The name of each archive is equal to the name of the directory.

def archive_directory(top_dir, subdir, tmpdir):
    # We're only archiving the *files* in this directory, not the subdirectories.

    files = []
    full_backup_path = os.path.join(top_dir, subdir)
    dir_contents = os.listdir(full_backup_path)

    # Only add files to 'files' list, not subdirs
    for c in dir_contents:
        fpath = os.path.join(top_dir, subdir, c)
        if os.path.isfile(fpath) and not fpath.endswith(".ini"):
            # logger.info("Adding to archive list: {0}".format(c))
            files.append(fpath)

    if files:  # No point creating empty archives!
        archive_file_path = os.path.join(tmpdir, os.path.basename(subdir)) + ".7z"

        logger.info("Archiving %s to %s" % (subdir, archive_file_path))
        try:
            devnull = open(os.devnull, "w")
            subprocess.check_call(
                ["7z", "a", "-t7z", archive_file_path, os.path.join(full_backup_path, "*"),
                 "-m0=BZip2", "-y", "-aoa", "-xr-!*/", "-xr-!*sync-conflict*",
                 "-xr-!*desktop.ini", "-xr-!*.tmp", "-xr-!*thumbs.db"], stdout=devnull, stderr=devnull)
            devnull.close()
            logger.info("Created archive at %s" % archive_file_path)
            return archive_file_path
        except subprocess.CalledProcessError, e:
            ret_code = e.returncode
            if ret_code == 1:
                # Warning (Non fatal error(s)). For example, one or more files were locked by some
                # other application, so they were not compressed.
                logger.info("7-Zip: Non-fatal error (return code 1)")
            elif ret_code == 2:
                # Fatal error
                logger.info("7-Zip: Fatal error (return code 2)")
            elif ret_code == 7:
                # Command-line error
                logger.info("7-Zip: Command-line error (return code 7)\n%s"
                            % e.cmd)
            elif ret_code == 8:
                # Not enough memory for operation
                logger.info("7-Zip: Not enough memory for operation (return code 8)")
            elif ret_code == 255:
                # User stopped the process
                logger.info("7-Zip: User stopped the process (return code 255)")
            return None


def upload_archive(archive_path, aws_vault, archive_treehash, aws_account_id, dummy=False):
    logger.info("Uploading {0} to vault {1}".format(archive_path, aws_vault))

    try:
        if not dummy:
            response = {}
            with open(archive_path, 'rb') as body:
                response = boto_client.upload_archive(vaultName=aws_vault,
                                                      accountId=aws_account_id,
                                                      archiveDescription=archive_path,
                                                      body=body)
            aws_params = response

            # Returned fields from upload:
            # location -> (string)
            # The relative URI path of the newly added archive resource.
            # checksum -> (string)
            # The checksum of the archive computed by Amazon Glacier.
            # archiveId -> (string)
            # The ID of the archive. This value is also included as part of the location.


        else:
            # This is a dummy upload, for testing purposes. Create a fake
            # AWS URI and location, but don't touch the archive.
            logger.info("Dummy upload - not actually uploading archive!")
            aws_params = {}
            aws_params["archiveId"] = "{0}-hcrbackup-{1}".format(aws_vault, time.time())
            aws_params["location"] = "aws://dummy-uri-" + aws_params["archiveId"]
            aws_params["checksum"] = archive_treehash

        logger.info("Uploaded archive {archpath} \n \
                      Returned fields: \n \
                      \tlocation: {params[location]} \n \
                      \tchecksum: {params[checksum]} \n \
                      \tarchiveId: {params[archiveId]}".format(archpath=archive_path,
                                                               params=aws_params))
        return aws_params

    except botocore.exceptions.ChecksumError, e:
        logger.error("Upload failed - AWS checksum does not match local checksum")
        return None

    except botocore.exceptions.ConnectionClosedError, e:
        logger.error("Upload failed - connection to AWS server was unexpectedly closed")
        return None

    except botocore.exceptions.EndpointConnectionError, e:
        logger.error("Upload failed - unable to connect to AWS server")
        return None

    except botocore.exceptions.BotoCoreError, e:
        logger.error("Upload failed - {0}".format(e.msg))
        return None

    except:
        logger.error("Upload failed - unknown error!")
        return None


def delete_aws_archive(archive_id, aws_vault, aws_account_id):
    logger.info("Deleting archive with id {0} from vault {1}".format(
        archive_id, aws_vault))

    try:
        boto_client.delete_archive(vaultName=aws_vault,
                                   archiveId=archive_id)

        logger.info("Successfully deleted archive from AWS")
        return 1

    except Exception, e:
        logger.info("Failed to delete archive from AWS!")
        return None


def delete_redundant_archives(db, aws_vault_name, aws_account_id):
    redundant_archives = cupocore.mongoops.get_archives_to_delete(db)
    for arch in redundant_archives:
        deleted_aws = delete_aws_archive(arch["_id"], aws_vault_name, aws_account_id)
        if deleted_aws:
            cupocore.mongoops.delete_archive_document(db, arch["_id"])
            logger.info("Deleted archive with ID {0} from local database".format(arch["_id"]))
        else:
            logger.info("AWS deletion failed; not removing database entry")


def compare_files(length_a, hash_a, length_b, hash_b):
    return (length_a == length_b) & (hash_a == hash_b)


def list_dirs(top_dir):
    # Find all of the subdirectories in a given directory.
    logger.info("Finding subdirectories of {0}".format(top_dir))
    dirs = []
    for dirname, subdirs, files in os.walk(top_dir):
        for s in subdirs:
            dirs.append(os.path.relpath(os.path.join(dirname, s), top_dir))
            logger.info("Found subdirectory {0}".format(os.path.join(dirname, s), top_dir))
    return dirs

<commit_msg>Improved exception handling for AWS archive deletion<commit_after>__author__ = '<NAME> <<EMAIL>>'
__version__ = '0.1.0'

import os, os.path
import subprocess
import tempfile
import botocore.utils, botocore.exceptions
import boto3
import time
import logging, logging.handlers
import cupocore
import pymongo.errors


# TODO-refactor: Move old archive detection into own method, and add unique path detection, so not only triggered when adding new archives.
# TODO-ratelimit: #1 Add network rate limiting
# TODO-backupscount: #2 Add a way of specifying the amount of redundant backups that should be kept
# TODO-backupsage: #3 Specify the minimum amount of time that a backup should be kept for if there are more than <min amount> of backups remaining.


# Only the *files* in a given directory are archived, not the subdirectories.
# The contents of the subdirectories live in archives of their own (except for any directories that *they* contain)
# This means that only the changed files in a given directory need to be checked - and that if a file in a
# sub-sub-subdirectory is changed, the whole parent directory doesn't need to be re-uploaded.
# The name of each archive is equal to the name of the directory.

def archive_directory(top_dir, subdir, tmpdir):
    # We're only archiving the *files* in this directory, not the subdirectories.

    files = []
    full_backup_path = os.path.join(top_dir, subdir)
    dir_contents = os.listdir(full_backup_path)

    # Only add files to 'files' list, not subdirs
    for c in dir_contents:
        fpath = os.path.join(top_dir, subdir, c)
        if os.path.isfile(fpath) and not fpath.endswith(".ini"):
            # logger.info("Adding to archive list: {0}".format(c))
            files.append(fpath)

    if files:  # No point creating empty archives!
        archive_file_path = os.path.join(tmpdir, os.path.basename(subdir)) + ".7z"

        logger.info("Archiving %s to %s" % (subdir, archive_file_path))

        devnull = open(os.devnull, "w")
        try:
            subprocess.check_call(
                ["7z", "a", "-t7z", archive_file_path, os.path.join(full_backup_path, "*"),
                 "-m0=BZip2", "-y", "-aoa", "-xr-!*/", "-xr-!*sync-conflict*",
                 "-xr-!*desktop.ini", "-xr-!*.tmp", "-xr-!*thumbs.db"], stdout=devnull, stderr=devnull)
            logger.info("Created archive at %s" % archive_file_path)
            return archive_file_path

        except subprocess.CalledProcessError, e:
            ret_code = e.returncode
            if ret_code == 1:
                # Warning (Non fatal error(s)). For example, one or more files were locked by some
                # other application, so they were not compressed.
                logger.info("7-Zip: Non-fatal error (return code 1)")
            elif ret_code == 2:
                # Fatal error
                logger.info("7-Zip: Fatal error (return code 2)")
            elif ret_code == 7:
                # Command-line error
                logger.info("7-Zip: Command-line error (return code 7)\n%s"
                            % e.cmd)
            elif ret_code == 8:
                # Not enough memory for operation
                logger.info("7-Zip: Not enough memory for operation (return code 8)")
            elif ret_code == 255:
                # User stopped the process
                logger.info("7-Zip: User stopped the process (return code 255)")

        finally:
            devnull.close()
            return None


def upload_archive(archive_path, aws_vault, archive_treehash, aws_account_id, dummy=False):
    logger.info("Uploading {0} to vault {1}".format(archive_path, aws_vault))

    if not dummy:
        try:
            with open(archive_path, 'rb') as body:
                aws_params = boto_client.upload_archive(vaultName=aws_vault,
                                                        accountId=aws_account_id,
                                                        archiveDescription=archive_path,
                                                        body=body)

                # Returned fields from upload:
                # location -> (string)
                # The relative URI path of the newly added archive resource.
                # checksum -> (string)
                # The checksum of the archive computed by Amazon Glacier.
                # archiveId -> (string)
                # The ID of the archive. This value is also included as part of the location.

        except botocore.exceptions.ChecksumError, e:
            logger.error("Upload failed - AWS checksum does not match local checksum")
            return None

        except botocore.exceptions.ConnectionClosedError, e:
            logger.error("Upload failed - connection to AWS server was unexpectedly closed")
            return None

        except botocore.exceptions.EndpointConnectionError, e:
            logger.error("Upload failed - unable to connect to AWS server")
            return None

        except botocore.exceptions.BotoCoreError, e:
            logger.error("Upload failed - {0}".format(e.msg))
            return None

        except:
            logger.error("Upload failed - unknown error!")
            return None

    else:
        # This is a dummy upload, for testing purposes. Create a fake
        # AWS URI and location, but don't touch the archive.
        logger.info("Dummy upload - not actually uploading archive!")

        arch_id = "{0}-hcrbackup-{1}".format(aws_vault, time.time())

        aws_params = {"archiveId": arch_id,
                      "location": "aws://dummy-uri-{0}".format(arch_id),
                      "checksum": archive_treehash
                      }

    logger.info("Uploaded archive {archpath} \n \
                  Returned fields: \n \
                  \tlocation: {params[location]} \n \
                  \tchecksum: {params[checksum]} \n \
                  \tarchiveId: {params[archiveId]}".format(archpath=archive_path,
                                                           params=aws_params))
    return aws_params


def delete_aws_archive(archive_id, aws_vault):
    logger.info("Deleting archive with id {0} from vault {1}".format(
        archive_id, aws_vault))

    try:
        boto_client.delete_archive(vaultName=aws_vault,
                                   archiveId=archive_id)

        logger.info("Successfully deleted archive from AWS")
        return 1

    except botocore.exceptions.ConnectionClosedError, e:
        logger.error("AWS archive removal failed - connection to AWS server was unexpectedly closed")
        return None

    except botocore.exceptions.EndpointConnectionError, e:
        logger.error("AWS archive removal failed - unable to connect to AWS server")
        return None

    except botocore.exceptions.ClientError, e:
        logger.error("AWS archive removal failed - {0}".format(e.response["Message"]))
        return None

    except botocore.exceptions.BotoCoreError, e:
        logger.error("AWS archive removal failed - {0}".format(e.message))
        return None

    except Exception, e:
        logger.error("AWS archive removal failed - {0}".format(e.message))

def delete_redundant_archives(db, aws_vault_name, aws_account_id):
    redundant_archives = cupocore.mongoops.get_archives_to_delete(db)
    for arch in redundant_archives:
        deleted_aws = delete_aws_archive(arch["_id"], aws_vault_name, aws_account_id)
        if deleted_aws:
            cupocore.mongoops.delete_archive_document(db, arch["_id"])
            logger.info("Deleted archive with ID {0} from local database".format(arch["_id"]))
        else:
            logger.info("AWS deletion failed; not removing database entry")


def compare_files(length_a, hash_a, length_b, hash_b):
    return (length_a == length_b) & (hash_a == hash_b)


def list_dirs(top_dir):
    # Find all of the subdirectories in a given directory.
    logger.info("Finding subdirectories of {0}".format(top_dir))
    dirs = []
    for dirname, subdirs, files in os.walk(top_dir):
        for s in subdirs:
            dirs.append(os.path.relpath(os.path.join(dirname, s), top_dir))
            logger.info("Found subdirectory {0}".format(os.path.join(dirname, s), top_dir))
    return dirs


{'fraction_non_alphanumeric': 0.09486190987802566, 'fraction_numerical': 0.005687922644252038, 'mean_word_length': 2.2539584618548223, 'pattern_counts': {'":': 3, '<': 11, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '384167', 'n_tokens_mistral': 4542, 'n_tokens_neox': 4382, 'n_words': 1360}