<commit_before>#!/usr/bin/python

# (C) Copyright 2017 IBM Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Authors:
<NAME> (<EMAIL>)
"""

"""
This file contains implementation of the SwiftHLM Handler component of the
SwiftHLM - a Swift extension for high latency media support.  SwiftHLM Handler
is installed on a Swift storage node, and invoked by a SwiftHLM proxy
middleware (SwiftHLM Middleware) when processing STATUS request or by SwiftHLM
Dispatcher process when processing MIGRATE or RECALL request. For an object or
a list of objects from a container, SwiftHLM Handler creates a list that maps
objects to file path(s) and submits the list to the HLM backend for migration,
recall or status processing.  
"""

from sys import stdin, stdout 
from collections import defaultdict
#from swift.common import read_config_file
import ConfigParser
from ConfigParser import RawConfigParser
from swift.common.utils import readconf
from swift.common.utils import json, get_logger, split_path
import logging

from swift.obj.server import ObjectController
from swift.common.storage_policy import POLICIES
from swift.common.exceptions import ConnectionTimeout, DiskFileQuarantined, \
    DiskFileNotExist, DiskFileCollision, DiskFileNoSpace, DiskFileDeleted, \
    DiskFileDeviceUnavailable, DiskFileExpired, ChunkReadTimeout, \
    DiskFileXattrNotSupported
from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \
    HTTPInternalServerError, HTTPNoContent, HTTPNotFound, \
    HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \
    HTTPClientDisconnect, HTTPMethodNotAllowed, Request, Response, \
    HTTPInsufficientStorage, HTTPForbidden, HTTPException, HeaderKeyDict, \
    HTTPConflict, HTTPServerError
import os
import imp

#scor aux
from swift.proxy.controllers.base import get_container_info

#scor aux
from swift.common.utils import hash_path

#
import swifthlm.dummy_connector
import importlib

# SwiftHLM Handler maps objects to files and submits requests to Storage
# Backend via Backend Connector
class Handler(object):

    def __init__(self):
        self.request_in = ''
        self.request_out = {}
        self.response_out = {}

        # Config
        configFile = r'/etc/swift/object-server.conf'
        self.conf = readconf(configFile) 
        # readconf does not load the [DEFAULT] section, adding that manually
        rcp = RawConfigParser()
        cf = open(configFile, 'r')
        rcp.readfp(cf)
        full_conf = self.conf.copy()
        full_conf.update(rcp.defaults())
        cf.close()
        self.conf = full_conf

        # Logging
        hlm_stor_node_config = self.conf.get('hlm', None)
        if hlm_stor_node_config:
            hlm_stor_node_log_level = hlm_stor_node_config.get('set log_level',
                    None)
        if hlm_stor_node_log_level:
            self.conf['log_level'] = hlm_stor_node_log_level
        self.logger = get_logger(self.conf, name='hlm-handler',
                log_route='swifthlm', fmt="%(server)s: %(msecs)03d "
                "[%(filename)s:%(funcName)20s():%(lineno)s] %(message)s")

        self.logger.info('info: Initialized Handler')
        self.logger.debug('dbg: Initialized Handler')
        #self.logger.info('conf: %s', self.conf)

        # Generic backend interface (GBI) configuration options
        self.gbi_provide_dirpaths_instead_of_filepaths = False
        conf_gbi_provide_dirpaths_instead_of_filepaths = \
                hlm_stor_node_config.get(\
                    'gbi_provide_dirpaths_instead_of_filepaths',
                    'False')
        if conf_gbi_provide_dirpaths_instead_of_filepaths == 'True':
            self.gbi_provide_dirpaths_instead_of_filepaths = True

        # Backend connector (directory and .py filename) can be configured in 
        # /etc/swift/object-server.conf
        # If nothing is configured a dummy backend connector, that is provided
        # and installed with SwiftHLM is used by default
        swifthlm_connector_module = hlm_stor_node_config.get(
                'swifthlm_connector_module',
                '')
        swifthlm_connector_dir = hlm_stor_node_config.get(
                'swifthlm_connector_dir',
                '')
        swifthlm_connector_filename = hlm_stor_node_config.get(
                'swifthlm_connector_filename',
                '') 
        swifthlm_connector_path = swifthlm_connector_dir + '/' + \
                swifthlm_connector_filename
        if swifthlm_connector_module:
            self.logger.debug('swifthlm_connector_module: %s',
                    swifthlm_connector_module)
            self.swifthlm_connector_mod = \
                importlib.import_module(swifthlm_connector_module, package=None)
        elif swifthlm_connector_filename:
            swifthlm_connector_module = swifthlm_connector_filename[:-3]
            self.logger.debug('swifthlm_connector_path: %s',
                    swifthlm_connector_path)
            self.swifthlm_connector_mod = \
                imp.load_source(swifthlm_connector_module,
                swifthlm_connector_path)
        else:
            self.logger.debug('Using default swifthlm_connector_module: %s',
                    'swifthlm.dummy_connector')
            self.swifthlm_connector_mod = swifthlm.dummy_connector

    # Receive request from dispatcher
    def receive_request(self):
        
        self.logger.debug('Receiving request from Dispatcher')
        self.request_in = str(stdin.read())

        return
       
    # Map objects to their local storage server data replicas
    # and create a request for the generic backend interface 
    def map_objects_to_targets(self):

        self.logger.debug('Mapping objects to files')
        self.logger.debug('request_in(first 1024 bytes): %s',
                str(self.request_in)[0:1023])
    
        request_in_dict = json.loads(self.request_in)
        #TODO consider modifying incoming request in place
        self.request_out = {}
        self.request_out['request'] = request_in_dict['request']
        objects_and_files = []
        for obj_and_dev in request_in_dict['objects']:
            obj_and_file = {}
            obj_and_file['object'] = obj_and_dev['object'] 
            self.logger.debug('obj: %s', obj_and_dev)        
            try:
                (account, container, obj) = split_path(obj_and_dev['object'],
                        3, 3, False)
            except ValueError:
                self.logger.debug('split_path exception')        
                raise
            device = obj_and_dev['device']
            # TODO, can can storage_policy_index be determined from storage
            # node to not have to pass from proxy?                      
#           # container_info = get_container_info(
#           #     {'PATH_INFO': '/v1/%s/%s' % (account, container)},
#           #     self.app, swift_source='LE')
#           # storage_policy_index = container_info['storage_policy']
#           # obj_ring = self.get_object_ring(storage_policy_index)
            swift_dir = request_in_dict['swift_dir']<commit_msg>Fix GET for DLO or object names containing virtual folders (a/b/c.txt)<commit_after>#!/usr/bin/python

# (C) Copyright 2017 IBM Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Authors:
<NAME> (<EMAIL>)
"""

"""
This file contains implementation of the SwiftHLM Handler component of the
SwiftHLM - a Swift extension for high latency media support.  SwiftHLM Handler
is installed on a Swift storage node, and invoked by a SwiftHLM proxy
middleware (SwiftHLM Middleware) when processing STATUS request or by SwiftHLM
Dispatcher process when processing MIGRATE or RECALL request. For an object or
a list of objects from a container, SwiftHLM Handler creates a list that maps
objects to file path(s) and submits the list to the HLM backend for migration,
recall or status processing.  
"""

from sys import stdin, stdout 
from collections import defaultdict
#from swift.common import read_config_file
import ConfigParser
from ConfigParser import RawConfigParser
from swift.common.utils import readconf
from swift.common.utils import json, get_logger, split_path
import logging

from swift.obj.server import ObjectController
from swift.common.storage_policy import POLICIES
from swift.common.exceptions import ConnectionTimeout, DiskFileQuarantined, \
    DiskFileNotExist, DiskFileCollision, DiskFileNoSpace, DiskFileDeleted, \
    DiskFileDeviceUnavailable, DiskFileExpired, ChunkReadTimeout, \
    DiskFileXattrNotSupported
from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \
    HTTPInternalServerError, HTTPNoContent, HTTPNotFound, \
    HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \
    HTTPClientDisconnect, HTTPMethodNotAllowed, Request, Response, \
    HTTPInsufficientStorage, HTTPForbidden, HTTPException, HeaderKeyDict, \
    HTTPConflict, HTTPServerError
import os
import imp

#scor aux
from swift.proxy.controllers.base import get_container_info

#scor aux
from swift.common.utils import hash_path

#
import swifthlm.dummy_connector
import importlib

# SwiftHLM Handler maps objects to files and submits requests to Storage
# Backend via Backend Connector
class Handler(object):

    def __init__(self):
        self.request_in = ''
        self.request_out = {}
        self.response_out = {}

        # Config
        configFile = r'/etc/swift/object-server.conf'
        self.conf = readconf(configFile) 
        # readconf does not load the [DEFAULT] section, adding that manually
        rcp = RawConfigParser()
        cf = open(configFile, 'r')
        rcp.readfp(cf)
        full_conf = self.conf.copy()
        full_conf.update(rcp.defaults())
        cf.close()
        self.conf = full_conf

        # Logging
        hlm_stor_node_config = self.conf.get('hlm', None)
        if hlm_stor_node_config:
            hlm_stor_node_log_level = hlm_stor_node_config.get('set log_level',
                    None)
        if hlm_stor_node_log_level:
            self.conf['log_level'] = hlm_stor_node_log_level
        self.logger = get_logger(self.conf, name='hlm-handler',
                log_route='swifthlm', fmt="%(server)s: %(msecs)03d "
                "[%(filename)s:%(funcName)20s():%(lineno)s] %(message)s")

        self.logger.info('info: Initialized Handler')
        self.logger.debug('dbg: Initialized Handler')
        #self.logger.info('conf: %s', self.conf)

        # Generic backend interface (GBI) configuration options
        self.gbi_provide_dirpaths_instead_of_filepaths = False
        conf_gbi_provide_dirpaths_instead_of_filepaths = \
                hlm_stor_node_config.get(\
                    'gbi_provide_dirpaths_instead_of_filepaths',
                    'False')
        if conf_gbi_provide_dirpaths_instead_of_filepaths == 'True':
            self.gbi_provide_dirpaths_instead_of_filepaths = True

        # Backend connector (directory and .py filename) can be configured in 
        # /etc/swift/object-server.conf
        # If nothing is configured a dummy backend connector, that is provided
        # and installed with SwiftHLM is used by default
        swifthlm_connector_module = hlm_stor_node_config.get(
                'swifthlm_connector_module',
                '')
        swifthlm_connector_dir = hlm_stor_node_config.get(
                'swifthlm_connector_dir',
                '')
        swifthlm_connector_filename = hlm_stor_node_config.get(
                'swifthlm_connector_filename',
                '') 
        swifthlm_connector_path = swifthlm_connector_dir + '/' + \
                swifthlm_connector_filename
        if swifthlm_connector_module:
            self.logger.debug('swifthlm_connector_module: %s',
                    swifthlm_connector_module)
            self.swifthlm_connector_mod = \
                importlib.import_module(swifthlm_connector_module, package=None)
        elif swifthlm_connector_filename:
            swifthlm_connector_module = swifthlm_connector_filename[:-3]
            self.logger.debug('swifthlm_connector_path: %s',
                    swifthlm_connector_path)
            self.swifthlm_connector_mod = \
                imp.load_source(swifthlm_connector_module,
                swifthlm_connector_path)
        else:
            self.logger.debug('Using default swifthlm_connector_module: %s',
                    'swifthlm.dummy_connector')
            self.swifthlm_connector_mod = swifthlm.dummy_connector

    # Receive request from dispatcher
    def receive_request(self):
        
        self.logger.debug('Receiving request from Dispatcher')
        self.request_in = str(stdin.read())

        return
       
    # Map objects to their local storage server data replicas
    # and create a request for the generic backend interface 
    def map_objects_to_targets(self):

        self.logger.debug('Mapping objects to files')
        self.logger.debug('request_in(first 1024 bytes): %s',
                str(self.request_in)[0:1023])
    
        request_in_dict = json.loads(self.request_in)
        #TODO consider modifying incoming request in place
        self.request_out = {}
        self.request_out['request'] = request_in_dict['request']
        objects_and_files = []
        for obj_and_dev in request_in_dict['objects']:
            obj_and_file = {}
            obj_and_file['object'] = obj_and_dev['object'] 
            self.logger.debug('obj: %s', obj_and_dev)        
            try:
                (account, container, obj) = split_path(obj_and_dev['object'],
                                                       3, 3, True)
            except ValueError:
                self.logger.debug('split_path exception')        
                raise
            device = obj_and_dev['device']
            # TODO, can can storage_policy_index be determined from storage
            # node to not have to pass from proxy?                      
#           # container_info = get_container_info(
#           #     {'PATH_INFO': '/v1/%s/%s' % (account, container)},
#           #     self.app, swift_source='LE')
#           # storage_policy_index = container_info['storage_policy']
#           # obj_ring = self.get_object_ring(storage_policy_index)
            swift_dir = request_in_dict['swift_dir']
{'fraction_non_alphanumeric': 0.07843776488231075, 'fraction_numerical': 0.003260089978483406, 'mean_word_length': 2.7620799607554574, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2806287', 'n_tokens_mistral': 4492, 'n_tokens_neox': 4299, 'n_words': 1099}