<commit_before>#!/usr/bin/env python
# encoding: utf-8

"""Verify that all OSF Storage files have Glacier backups and parity files,
creating any missing backups.

TODO: Add check against Glacier inventory
Note: Must have par2 installed to run
"""

import os
import logging

import pyrax
from boto.glacier.layer2 import Layer2

from website.app import init_app
from website.addons.osfstorage import model

from scripts import utils as scripts_utils
from scripts.osfstorage import utils as storage_utils
from scripts.osfstorage import settings as storage_settings


container_primary = None
container_parity = None
vault = None

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)


def download_from_cloudfiles(version):
    path = os.path.join(storage_settings.AUDIT_TEMP_PATH, version.location['object'])
    if os.path.exists(path):
        return path
    obj = container_primary.get_object(version.location['object'])
    obj.download(storage_settings.AUDIT_TEMP_PATH)
    return path


def delete_temp_file(version):
    path = os.path.join(storage_settings.AUDIT_TEMP_PATH, version.location['object'])
    try:
        os.remove(path)
    except OSError:
        pass


def ensure_glacier(version, dry_run):
    if version.metadata.get('archive'):
        return
    logger.info('Glacier archive for version {0} not found'.format(version._id))
    if dry_run:
        return
    download_from_cloudfiles(version)
    file_path = os.path.join(storage_settings.AUDIT_TEMP_PATH, version.location['object'])
    glacier_id = vault.upload_archive(file_path, description=version.location['object'])
    version.metadata['archive'] = glacier_id
    version.save()


def check_parity_files(version):
    index = list(container_parity.list_all(prefix='{0}.par2'.format(version.location['object'])))
    vols = list(container_parity.list_all(prefix='{0}.vol'.format(version.location['object'])))
    return len(index) == 1 and len(vols) >= 1


def ensure_parity(version, dry_run):
    if check_parity_files(version):
        return
    logger.info('Parity files for version {0} not found'.format(version._id))
    if dry_run:
        return
    file_path = download_from_cloudfiles(version)
    parity_paths = storage_utils.create_parity_files(file_path)
    for parity_path in parity_paths:
        container_parity.create(parity_path)
        os.remove(parity_path)
    if not check_parity_files(version):
        logger.error('Parity files for version {0} not found after update'.format(version._id))


def ensure_backups(version, dry_run):
    if version.size == 0:
        logger.info('Skipping empty version {0}'.format(version._id))
        return
    ensure_glacier(version, dry_run)
    ensure_parity(version, dry_run)
    delete_temp_file(version)


def get_targets():
    return model.OsfStorageFileVersion.find()


def main(dry_run):
    for version in get_targets():
        ensure_backups(version, dry_run)


if __name__ == '__main__':
    import sys
    dry_run = 'dry' in sys.argv

    # Set up storage backends
    init_app(set_backends=True, routes=False)

    # Authenticate to Rackspace
    pyrax.settings.set('identity_type', 'rackspace')
    pyrax.set_credentials(
        storage_settings.USERNAME,
        storage_settings.API_KEY,
        region=storage_settings.REGION
    )
    container_primary = pyrax.cloudfiles.get_container(storage_settings.PRIMARY_CONTAINER_NAME)
    container_parity = pyrax.cloudfiles.get_container(storage_settings.PARITY_CONTAINER_NAME)<commit_msg>Fix to files audit<commit_after>#!/usr/bin/env python
# encoding: utf-8

"""Verify that all OSF Storage files have Glacier backups and parity files,
creating any missing backups.

TODO: Add check against Glacier inventory
Note: Must have par2 installed to run
"""

import os
import logging

import pyrax
from boto.glacier.layer2 import Layer2
from modularodm import Q

from website.app import init_app
from website.addons.osfstorage import model

from scripts import utils as scripts_utils
from scripts.osfstorage import utils as storage_utils
from scripts.osfstorage import settings as storage_settings


container_primary = None
container_parity = None
vault = None

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)


def download_from_cloudfiles(version):
    path = os.path.join(storage_settings.AUDIT_TEMP_PATH, version.location['object'])
    if os.path.exists(path):
        return path
    obj = container_primary.get_object(version.location['object'])
    obj.download(storage_settings.AUDIT_TEMP_PATH)
    return path


def delete_temp_file(version):
    path = os.path.join(storage_settings.AUDIT_TEMP_PATH, version.location['object'])
    try:
        os.remove(path)
    except OSError:
        pass


def ensure_glacier(version, dry_run):
    if version.metadata.get('archive'):
        return
    logger.info('Glacier archive for version {0} not found'.format(version._id))
    if dry_run:
        return
    download_from_cloudfiles(version)
    file_path = os.path.join(storage_settings.AUDIT_TEMP_PATH, version.location['object'])
    glacier_id = vault.upload_archive(file_path, description=version.location['object'])
    version.metadata['archive'] = glacier_id
    version.save()


def check_parity_files(version):
    index = list(container_parity.list_all(prefix='{0}.par2'.format(version.location['object'])))
    vols = list(container_parity.list_all(prefix='{0}.vol'.format(version.location['object'])))
    return len(index) == 1 and len(vols) >= 1


def ensure_parity(version, dry_run):
    if check_parity_files(version):
        return
    logger.info('Parity files for version {0} not found'.format(version._id))
    if dry_run:
        return
    file_path = download_from_cloudfiles(version)
    parity_paths = storage_utils.create_parity_files(file_path)
    for parity_path in parity_paths:
        container_parity.create(parity_path)
        os.remove(parity_path)
    if not check_parity_files(version):
        logger.error('Parity files for version {0} not found after update'.format(version._id))


def ensure_backups(version, dry_run):
    if version.size == 0:
        logger.info('Skipping empty version {0}'.format(version._id))
        return
    ensure_glacier(version, dry_run)
    ensure_parity(version, dry_run)
    delete_temp_file(version)


def get_targets():
    return model.OsfStorageFileVersion.find(Q('location.object', 'exists', True))


def main(dry_run):
    for version in get_targets():
        ensure_backups(version, dry_run)


if __name__ == '__main__':
    import sys
    dry_run = 'dry' in sys.argv

    # Set up storage backends
    init_app(set_backends=True, routes=False)

    # Authenticate to Rackspace
    pyrax.settings.set('identity_type', 'rackspace')
    pyrax.set_credentials(
        storage_settings.USERNAME,
        storage_settings.API_KEY,
        region=storage_settings.REGION
    )
    container_primary = pyrax.cloudfiles.get_container(storage_settings.PRIMARY_CONTAINER_NAME)
    container_parity = pyrax.cloudfiles.get_container(storage_settings.PARITY_CONTAINER_NAME)
{'fraction_non_alphanumeric': 0.09603487064116985, 'fraction_numerical': 0.003937007874015748, 'mean_word_length': 4.320119670905012, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6867372', 'n_tokens_mistral': 2267, 'n_tokens_neox': 2204, 'n_words': 476}