<commit_before>from __future__ import absolute_import
import base64
import logging
import os
import platform
import requests
from requests.auth import AuthBase
import slumber
import urllib

# archivematicaCommon
from archivematicaFunctions import get_setting

LOGGER = logging.getLogger("archivematica.common")


class ResourceNotFound(Exception):
    pass

class BadRequest(Exception):
    pass

class StorageServiceError(Exception):
    pass

######################### INTERFACE WITH STORAGE API #########################

############# HELPER FUNCTIONS #############

class TastypieApikeyAuth(AuthBase):
    def __init__(self, username, apikey):
        self.username = username
        self.apikey = apikey

    def __call__(self, r):
        r.headers['Authorization'] = "ApiKey {0}:{1}".format(self.username, self.apikey)
        return r


def _storage_service_url():
    # Get storage service URL from DashboardSetting model
    storage_service_url = get_setting('storage_service_url', None)
    if storage_service_url is None:
        LOGGER.error("Storage server not configured.")
        storage_service_url = 'http://localhost:8000/'
    # If the URL doesn't end in a /, add one
    if storage_service_url[-1] != '/':
        storage_service_url+='/'
    storage_service_url = storage_service_url+'api/v2/'
    LOGGER.debug("Storage service URL: {}".format(storage_service_url))
    return storage_service_url


def _storage_api():
    """ Returns slumber access to storage API. """
    storage_service_url = _storage_service_url()
    username = get_setting('storage_service_user', 'test')
    api_key = get_setting('storage_service_apikey', None)
    api = slumber.API(storage_service_url, auth=TastypieApikeyAuth(username, api_key))
    return api

def _storage_api_params():
    """ Returns API GET params username=USERNAME&api_key=KEY """
    username = get_setting('storage_service_user', 'test')
    api_key = get_setting('storage_service_apikey', None)
    return urllib.urlencode({'username': username, 'api_key': api_key})

def _storage_relative_from_absolute(location_path, space_path):
    """ Strip space_path and next / from location_path. """
    location_path = os.path.normpath(location_path)
    if location_path[0] == '/':
        strip = len(space_path)
        if location_path[strip] == '/':
            strip += 1
        location_path = location_path[strip:]
    return location_path

############# PIPELINE #############

def create_pipeline(create_default_locations=False, shared_path=None, api_username=None, api_key=None):
    api = _storage_api()
    pipeline = {
        'uuid': get_setting('dashboard_uuid'),
        'description': "Archivematica on {}".format(platform.node()),
        'create_default_locations': create_default_locations,
        'shared_path': shared_path,
        'api_username': api_username,
        'api_key': api_key,
    }
    LOGGER.info("Creating pipeline in storage service with {}".format(pipeline))
    try:
        api.pipeline.post(pipeline)
    except slumber.exceptions.HttpClientError as e:
        LOGGER.warning("Unable to create Archivematica pipeline in storage service from {} because {}".format(pipeline, e.content))
        return False
    except slumber.exceptions.HttpServerError as e:
        LOGGER.warning("Unable to create Archivematica pipeline in storage service from {} because {}".format(pipeline, e.content), exc_info=True)
        if 'column uuid is not unique' in e.content:
            pass
        else:
            raise
    return True

def _get_pipeline(uuid):
    api = _storage_api()
    try:
        pipeline = api.pipeline(uuid).get()
    except slumber.exceptions.HttpClientError as e:
        if e.response.status_code == 404:
            LOGGER.warning("This Archivematica instance is not registered with the storage service or has been disabled.")
        LOGGER.warning('Error fetching pipeline', exc_info=True)
        pipeline = None
    return pipeline

############# LOCATIONS #############

def get_location(path=None, purpose=None, space=None):
    """ Returns a list of storage locations, filtered by parameters.

    Queries the storage service and returns a list of storage locations,
    optionally filtered by purpose, containing space or path.

    purpose: How the storage is used.  Should reference storage service
        purposes, found in storage_service.locations.models.py
    path: Path to location.  If a space is passed in, paths starting with /
        have the space's path stripped.
    """
    api = _storage_api()
    offset = 0
    return_locations = []
    if space and path:
        path = _storage_relative_from_absolute(path, space['path'])
        space = space['uuid']
    pipeline = _get_pipeline(get_setting('dashboard_uuid'))
    if pipeline is None:
        return None
    while True:
        locations = api.location.get(pipeline__uuid=pipeline['uuid'],
                                     relative_path=path,
                                     purpose=purpose,
                                     space=space,
                                     offset=offset)
        LOGGER.debug("Storage locations retrieved: {}".format(locations))
        return_locations += locations['objects']
        if not locations['meta']['next']:
            break
        offset += locations['meta']['limit']

    LOGGER.info("Storage locations returned: {}".format(return_locations))
    return return_locations

def get_location_by_uri(uri):
    """ Get a specific location by the URI.  Only returns one location. """
    api = _storage_api()
    # TODO check that location is associated with this pipeline
    return api.location(uri).get()

def browse_location(uuid, path):
    """
    Browse files in a location. Encodes path in base64 for transimission, returns decoded entries.
    """
    api = _storage_api()
    path = base64.b64encode(path)
    browse = api.location(uuid).browse.get(path=path)
    browse['entries'] = map(base64.b64decode, browse['entries'])
    browse['directories'] = map(base64.b64decode, browse['directories'])
    browse['properties'] = {base64.b64decode(k): v for k, v in browse.get('properties', {}).items()}
    return browse

def copy_files(source_location, destination_location, files, api=None):
    """
    Copies `files` from `source_location` to `destination_location` using SS.

    source_location/destination_location: Dict with Location information, result
        of a call to get_location or get_location_by_uri.
    files: List of dicts with source and destination paths relative to
        source_location and destination_location, respectively.  All other
        fields ignored.
    """
    if api is None:
        api = _storage_api()
    pipeline = _get_pipeline(get_setting('dashboard_uuid'))
    move_files = {
        'origin_location': source_location['resource_uri'],
        'files': files,
        'pipeline': pipeline['resource_uri'],
    }
    try:
        ret = api.location(destination_location['uuid']).post(move_files)
    except slumber.exceptions.HttpClientError as e:
        LOGGER.warning("Unable to move files with {} because {}".format(move_files, e.content))
        return (None, e)
    except slumber.exceptions.HttpServerError as e:
        LOGGER.warning("Could not connect to storage service: {} ({})".format(
            e, e.content))
        return (None, e)
    return (ret, None)

def get_files_from_backlog(files):
    """
    Copies files from the backlog location to the currently processing location.
    See copy_files for more details.
    """
    api = _storage_api()<commit_msg>Decoding to Unicode prior to Slumber ``.post``.<commit_after>from __future__ import absolute_import
import base64
import logging
import os
import platform
import requests
from requests.auth import AuthBase
import slumber
import urllib

# archivematicaCommon
from archivematicaFunctions import get_setting

LOGGER = logging.getLogger("archivematica.common")


class ResourceNotFound(Exception):
    pass

class BadRequest(Exception):
    pass

class StorageServiceError(Exception):
    pass

######################### INTERFACE WITH STORAGE API #########################

############# HELPER FUNCTIONS #############

class TastypieApikeyAuth(AuthBase):
    def __init__(self, username, apikey):
        self.username = username
        self.apikey = apikey

    def __call__(self, r):
        r.headers['Authorization'] = "ApiKey {0}:{1}".format(self.username, self.apikey)
        return r


def _storage_service_url():
    # Get storage service URL from DashboardSetting model
    storage_service_url = get_setting('storage_service_url', None)
    if storage_service_url is None:
        LOGGER.error("Storage server not configured.")
        storage_service_url = 'http://localhost:8000/'
    # If the URL doesn't end in a /, add one
    if storage_service_url[-1] != '/':
        storage_service_url+='/'
    storage_service_url = storage_service_url+'api/v2/'
    LOGGER.debug("Storage service URL: {}".format(storage_service_url))
    return storage_service_url


def _storage_api():
    """ Returns slumber access to storage API. """
    storage_service_url = _storage_service_url()
    username = get_setting('storage_service_user', 'test')
    api_key = get_setting('storage_service_apikey', None)
    api = slumber.API(storage_service_url, auth=TastypieApikeyAuth(username, api_key))
    return api

def _storage_api_params():
    """ Returns API GET params username=USERNAME&api_key=KEY """
    username = get_setting('storage_service_user', 'test')
    api_key = get_setting('storage_service_apikey', None)
    return urllib.urlencode({'username': username, 'api_key': api_key})

def _storage_relative_from_absolute(location_path, space_path):
    """ Strip space_path and next / from location_path. """
    location_path = os.path.normpath(location_path)
    if location_path[0] == '/':
        strip = len(space_path)
        if location_path[strip] == '/':
            strip += 1
        location_path = location_path[strip:]
    return location_path

############# PIPELINE #############

def create_pipeline(create_default_locations=False, shared_path=None, api_username=None, api_key=None):
    api = _storage_api()
    pipeline = {
        'uuid': get_setting('dashboard_uuid'),
        'description': "Archivematica on {}".format(platform.node()),
        'create_default_locations': create_default_locations,
        'shared_path': shared_path,
        'api_username': api_username,
        'api_key': api_key,
    }
    LOGGER.info("Creating pipeline in storage service with {}".format(pipeline))
    try:
        api.pipeline.post(pipeline)
    except slumber.exceptions.HttpClientError as e:
        LOGGER.warning("Unable to create Archivematica pipeline in storage service from {} because {}".format(pipeline, e.content))
        return False
    except slumber.exceptions.HttpServerError as e:
        LOGGER.warning("Unable to create Archivematica pipeline in storage service from {} because {}".format(pipeline, e.content), exc_info=True)
        if 'column uuid is not unique' in e.content:
            pass
        else:
            raise
    return True

def _get_pipeline(uuid):
    api = _storage_api()
    try:
        pipeline = api.pipeline(uuid).get()
    except slumber.exceptions.HttpClientError as e:
        if e.response.status_code == 404:
            LOGGER.warning("This Archivematica instance is not registered with the storage service or has been disabled.")
        LOGGER.warning('Error fetching pipeline', exc_info=True)
        pipeline = None
    return pipeline

############# LOCATIONS #############

def get_location(path=None, purpose=None, space=None):
    """ Returns a list of storage locations, filtered by parameters.

    Queries the storage service and returns a list of storage locations,
    optionally filtered by purpose, containing space or path.

    purpose: How the storage is used.  Should reference storage service
        purposes, found in storage_service.locations.models.py
    path: Path to location.  If a space is passed in, paths starting with /
        have the space's path stripped.
    """
    api = _storage_api()
    offset = 0
    return_locations = []
    if space and path:
        path = _storage_relative_from_absolute(path, space['path'])
        space = space['uuid']
    pipeline = _get_pipeline(get_setting('dashboard_uuid'))
    if pipeline is None:
        return None
    while True:
        locations = api.location.get(pipeline__uuid=pipeline['uuid'],
                                     relative_path=path,
                                     purpose=purpose,
                                     space=space,
                                     offset=offset)
        LOGGER.debug("Storage locations retrieved: {}".format(locations))
        return_locations += locations['objects']
        if not locations['meta']['next']:
            break
        offset += locations['meta']['limit']

    LOGGER.info("Storage locations returned: {}".format(return_locations))
    return return_locations

def get_location_by_uri(uri):
    """ Get a specific location by the URI.  Only returns one location. """
    api = _storage_api()
    # TODO check that location is associated with this pipeline
    return api.location(uri).get()

def browse_location(uuid, path):
    """
    Browse files in a location. Encodes path in base64 for transimission, returns decoded entries.
    """
    api = _storage_api()
    path = base64.b64encode(path)
    browse = api.location(uuid).browse.get(path=path)
    browse['entries'] = map(base64.b64decode, browse['entries'])
    browse['directories'] = map(base64.b64decode, browse['directories'])
    browse['properties'] = {base64.b64decode(k): v for k, v in browse.get('properties', {}).items()}
    return browse

def copy_files(source_location, destination_location, files, api=None):
    """
    Copies `files` from `source_location` to `destination_location` using SS.

    source_location/destination_location: Dict with Location information, result
        of a call to get_location or get_location_by_uri.
    files: List of dicts with source and destination paths relative to
        source_location and destination_location, respectively.  All other
        fields ignored.
    """
    if api is None:
        api = _storage_api()
    pipeline = _get_pipeline(get_setting('dashboard_uuid'))
    move_files = {
        'origin_location': source_location['resource_uri'],
        'files': files,
        'pipeline': pipeline['resource_uri'],
    }

    # Here we attempt to decode the 'source' attributes of each move-file to
    # Unicode prior to passing to Slumber's ``post`` method. Slumber will do
    # this anyway and will choke in certain specific cases, specifically where
    # the JavaScript of the dashboard has base-64-encoded a Latin-1-encoded
    # string.
    for file_ in move_files['files']:
        try:
            file_['source'] = file_['source'].decode('utf8')
        except UnicodeDecodeError:
            try:
                file_['source'] = file_['source'].decode('latin-1')
            except UnicodeError:
                pass

    try:
        ret = api.location(destination_location['uuid']).post(move_files)
    except slumber.exceptions.HttpClientError as e:
        LOGGER.warning("Unable to move files with {} because {}".format(move_files, e.content))
        return (None, e)
    except slumber.exceptions.HttpServerError as e:
        LOGGER.warning("Could not connect to storage service: {} ({})".format(
            e, e.content))
        return (None, e)
    return (ret, None)

def get_files_from_backlog(files):
    """
    Copies files from the backlog location to the currently processing location.
    See copy_files for more details.
    """
    api = _storage_api()
{'fraction_non_alphanumeric': 0.11305060180225597, 'fraction_numerical': 0.004600163841451887, 'mean_word_length': 3.2810898300512545, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3129109', 'n_tokens_mistral': 4451, 'n_tokens_neox': 4389, 'n_words': 1224}