<commit_before># Copyright (c) 2010-2013 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

""" Sproxyd Disk File Interface for Swift Object Server"""

import hashlib
import pickle
import base64
import random
import urllib
from contextlib import contextmanager

from eventlet.timeout import Timeout

from swift.common.bufferedhttp import http_connect_raw
from swift.common.exceptions import ConnectionTimeout, \
    DiskFileQuarantined, DiskFileNotExist, \
    DiskFileDeleted, DiskFileNotOpen, \
    DiskFileError
from swift.common.swob import multi_range_iterator

class SproxydException(DiskFileError):
    """
    Sproxyd Exception
    """
    def __init__(self, msg, ipaddr='', port=0, path='',
                 http_status=0, http_reason=''):
        Exception.__init__(self, msg)
        self.msg = msg
        self.ipaddr = ipaddr
        self.port = port
        self.base_path = path
        self.http_status = http_status
        self.http_reason = http_reason

    def __str__(self):
        if self.ipaddr:
            self.msg += ' %s' % self.ipaddr
        if self.port:
            self.msg += ':%d' % int(self.port)
        if self.base_path:
            self.msg += '%s' % self.base_path
        if self.http_status:
            self.msg += ' %d' % self.http_status
        if self.http_reason:
            self.msg += ' %s' % self.http_reason
        return self.msg


class ScalitySproxydFileSystem(object):
    """
    A sproxyd file system scheme.
    """

    def __init__(self, conf, logger):
        self.logger = logger
        self.conn_timeout = int(conf.get('sproxyd_conn_timeout', 10))
        self.proxy_timeout = int(conf.get('sproxyd_proxy_timeout', 3))
        self.base_path = conf.get('sproxyd_path', '/proxy/chord').rstrip('/') + '/'
        self.hosts = conf.get('sproxyd_host', 'localhost:81').split(",")
        random.shuffle(self.hosts)

    def do_connect(self, ipaddr, port, method, path, headers=None,
                   query_string=None, ssl=False):
        """
        stubable function for connecting
        """
        safe_path = self.base_path + urllib.quote(path)
        conn = http_connect_raw(
            ipaddr, port, method,
            safe_path, headers, query_string, ssl)
        return conn

    def conn_getresponse(self, conn):
        """
        stubable function for getting conn responses
        """
        return conn.getresponse()

    def get_next_host(self):
        server = random.choice(self.hosts)
        (host, port) = server.split(':')
        return host, port

    def get_meta(self, name):
        """
        Open a connection and get usermd"
        """
        self.logger.debug("HEAD " + self.base_path + name)
        headers = {}
        conn = None
        try:
            with ConnectionTimeout(self.conn_timeout):
                (ipaddr, port) = self.get_next_host()
                conn = self.do_connect(
                    ipaddr, port, 'HEAD',
                    name, headers, None, False)
            with Timeout(self.proxy_timeout):
                resp = self.conn_getresponse(conn)
                if resp.status == 200:
                    headers = dict(resp.getheaders())
                    usermd = base64.b64decode(headers["x-scal-usermd"])
                    metadata = pickle.loads(usermd)
                elif resp.status == 404:
                    metadata = None
                else:
                    msg = resp.read()
                    raise SproxydException(
                        'get_meta: %s' % msg,
                        ipaddr=ipaddr, port=port,
                        path=self.base_path, http_status=resp.status,
                        http_reason=resp.reason)
        except (EOFError) as err:
            print "EOFError"
            return None
        finally:
            if conn:
                conn.close()
        self.logger.debug("Metadata for " + self.base_path + name + " : " + str(metadata))
        return metadata

    def put_meta(self, name, metadata):
        """
        Connect to sproxyd and put usermd
        """
        self.logger.debug("PUT_meta " + self.base_path + name + " : " + str(metadata))
        if metadata is None:
            raise SproxydException("no usermd")
        headers = {}
        headers["x-scal-cmd"] = "update-usermd"
        usermd = pickle.dumps(metadata)
        headers["x-scal-usermd"] = base64.b64encode(usermd)
        conn = None
        try:
            with ConnectionTimeout(self.conn_timeout):
                (ipaddr, port) = self.get_next_host()
                conn = self.do_connect(
                    ipaddr, port, 'PUT',
                    name, headers, None, False)
            with Timeout(self.proxy_timeout):
                resp = self.conn_getresponse(conn)
                if resp.status == 200:
                    resp.read()
                elif resp.status == 500:
                    msg = resp.read()
                    raise SproxydException(
                        'put_meta: %s' % msg,
                        ipaddr=ipaddr, port=port,
                        path=self.base_path, http_status=resp.status,
                        http_reason=resp.reason)
        finally:
            if conn:
                conn.close()

    def del_object(self, name):
        """
        Connect to sproxyd and delete object
        """
        self.logger.debug("DELETE " + self.base_path + name)
        headers = {}
        conn = None
        try:
            with ConnectionTimeout(self.conn_timeout):
                (ipaddr, port) = self.get_next_host()
                conn = self.do_connect(
                    ipaddr, port, 'DELETE',
                    name, headers, None, False)
            with Timeout(self.proxy_timeout):
                resp = self.conn_getresponse(conn)
                if resp.status == 200 or resp.status == 404:
                    resp.read()
                elif resp.status == 500:
                    msg = resp.read()
                    raise SproxydException(
                        'del_object: %s' % msg, ipaddr=ipaddr, port=port,
                        path=self.base_path, http_status=resp.status,
                        http_reason=resp.reason)
        finally:
            if conn:<commit_msg>Raise an exception for everything unexpected<commit_after># Copyright (c) 2010-2013 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

""" Sproxyd Disk File Interface for Swift Object Server"""

import hashlib
import pickle
import base64
import random
import urllib
from contextlib import contextmanager

from eventlet.timeout import Timeout

from swift.common.bufferedhttp import http_connect_raw
from swift.common.exceptions import ConnectionTimeout, \
    DiskFileQuarantined, DiskFileNotExist, \
    DiskFileDeleted, DiskFileNotOpen, \
    DiskFileError
from swift.common.swob import multi_range_iterator

class SproxydException(DiskFileError):
    """
    Sproxyd Exception
    """
    def __init__(self, msg, ipaddr='', port=0, path='',
                 http_status=0, http_reason=''):
        Exception.__init__(self, msg)
        self.msg = msg
        self.ipaddr = ipaddr
        self.port = port
        self.base_path = path
        self.http_status = http_status
        self.http_reason = http_reason

    def __str__(self):
        if self.ipaddr:
            self.msg += ' %s' % self.ipaddr
        if self.port:
            self.msg += ':%d' % int(self.port)
        if self.base_path:
            self.msg += '%s' % self.base_path
        if self.http_status:
            self.msg += ' %d' % self.http_status
        if self.http_reason:
            self.msg += ' %s' % self.http_reason
        return self.msg


class ScalitySproxydFileSystem(object):
    """
    A sproxyd file system scheme.
    """

    def __init__(self, conf, logger):
        self.logger = logger
        self.conn_timeout = int(conf.get('sproxyd_conn_timeout', 10))
        self.proxy_timeout = int(conf.get('sproxyd_proxy_timeout', 3))
        self.base_path = conf.get('sproxyd_path', '/proxy/chord').rstrip('/') + '/'
        self.hosts = conf.get('sproxyd_host', 'localhost:81').split(",")
        random.shuffle(self.hosts)

    def do_connect(self, ipaddr, port, method, path, headers=None,
                   query_string=None, ssl=False):
        """
        stubable function for connecting
        """
        safe_path = self.base_path + urllib.quote(path)
        conn = http_connect_raw(
            ipaddr, port, method,
            safe_path, headers, query_string, ssl)
        return conn

    def conn_getresponse(self, conn):
        """
        stubable function for getting conn responses
        """
        return conn.getresponse()

    def get_next_host(self):
        server = random.choice(self.hosts)
        (host, port) = server.split(':')
        return host, port

    def get_meta(self, name):
        """
        Open a connection and get usermd"
        """
        self.logger.debug("HEAD " + self.base_path + name)
        headers = {}
        conn = None
        try:
            with ConnectionTimeout(self.conn_timeout):
                (ipaddr, port) = self.get_next_host()
                conn = self.do_connect(
                    ipaddr, port, 'HEAD',
                    name, headers, None, False)
            with Timeout(self.proxy_timeout):
                resp = self.conn_getresponse(conn)
                if resp.status == 200:
                    headers = dict(resp.getheaders())
                    usermd = base64.b64decode(headers["x-scal-usermd"])
                    metadata = pickle.loads(usermd)
                elif resp.status == 404:
                    metadata = None
                else:
                    msg = resp.read()
                    raise SproxydException(
                        'get_meta: %s' % msg,
                        ipaddr=ipaddr, port=port,
                        path=self.base_path, http_status=resp.status,
                        http_reason=resp.reason)
        except (EOFError) as err:
            print "EOFError"
            return None
        finally:
            if conn:
                conn.close()
        self.logger.debug("Metadata for " + self.base_path + name + " : " + str(metadata))
        return metadata

    def put_meta(self, name, metadata):
        """
        Connect to sproxyd and put usermd
        """
        self.logger.debug("PUT_meta " + self.base_path + name + " : " + str(metadata))
        if metadata is None:
            raise SproxydException("no usermd")
        headers = {}
        headers["x-scal-cmd"] = "update-usermd"
        usermd = pickle.dumps(metadata)
        headers["x-scal-usermd"] = base64.b64encode(usermd)
        conn = None
        try:
            with ConnectionTimeout(self.conn_timeout):
                (ipaddr, port) = self.get_next_host()
                conn = self.do_connect(
                    ipaddr, port, 'PUT',
                    name, headers, None, False)
            with Timeout(self.proxy_timeout):
                resp = self.conn_getresponse(conn)
                if resp.status == 200:
                    resp.read()
                else:
                    msg = resp.read()
                    raise SproxydException(
                        'put_meta: %s' % msg,
                        ipaddr=ipaddr, port=port,
                        path=self.base_path, http_status=resp.status,
                        http_reason=resp.reason)
        finally:
            if conn:
                conn.close()

    def del_object(self, name):
        """
        Connect to sproxyd and delete object
        """
        self.logger.debug("DELETE " + self.base_path + name)
        headers = {}
        conn = None
        try:
            with ConnectionTimeout(self.conn_timeout):
                (ipaddr, port) = self.get_next_host()
                conn = self.do_connect(
                    ipaddr, port, 'DELETE',
                    name, headers, None, False)
            with Timeout(self.proxy_timeout):
                resp = self.conn_getresponse(conn)
                if resp.status == 200 or resp.status == 404:
                    resp.read()
                else:
                    msg = resp.read()
                    raise SproxydException(
                        'del_object: %s' % msg, ipaddr=ipaddr, port=port,
                        path=self.base_path, http_status=resp.status,
                        http_reason=resp.reason)
        finally:
            if conn:
{'fraction_non_alphanumeric': 0.09955018066514269, 'fraction_numerical': 0.006931642209276602, 'mean_word_length': 1.723293172690763, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '395043', 'n_tokens_mistral': 3889, 'n_tokens_neox': 3682, 'n_words': 934}