<commit_before># Copyright 2010-2011 <NAME>
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Storage backend for RBD
   (RADOS (Reliable Autonomic Distributed Object Store) Block Device)"""
from __future__ import absolute_import
from __future__ import with_statement

import hashlib
import math

from oslo.config import cfg
import six
import six.moves.urllib.parse as urlparse
from six import text_type

from glance.common import exception
from glance.common import utils
from glance import i18n
from glance.openstack.common import excutils
import glance.openstack.common.log as logging
from glance.openstack.common import units
import glance.store.base
import glance.store.location

try:
    import rados
    import rbd
except ImportError:
    rados = None
    rbd = None

DEFAULT_POOL = 'images'
DEFAULT_CONFFILE = '/etc/ceph/ceph.conf'
DEFAULT_USER = None    # let librados decide based on the Ceph conf file
DEFAULT_CHUNKSIZE = 8  # in MiB
DEFAULT_SNAPNAME = 'snap'

LOG = logging.getLogger(__name__)
_LI = i18n._LI

rbd_opts = [
    cfg.IntOpt('rbd_store_chunk_size', default=DEFAULT_CHUNKSIZE,
               help=_('RADOS images will be chunked into objects of this size '
                      '(in megabytes). For best performance, this should be '
                      'a power of two.')),
    cfg.StrOpt('rbd_store_pool', default=DEFAULT_POOL,
               help=_('RADOS pool in which images are stored.')),
    cfg.StrOpt('rbd_store_user', default=DEFAULT_USER,
               help=_('RADOS user to authenticate as (only applicable if '
                      'using Cephx. If <None>, a default will be chosen based '
                      'on the client. section in rbd_store_ceph_conf).')),
    cfg.StrOpt('rbd_store_ceph_conf', default=DEFAULT_CONFFILE,
               help=_('Ceph configuration file path. '
                      'If <None>, librados will locate the default config. '
                      'If using cephx authentication, this file should '
                      'include a reference to the right keyring '
                      'in a client.<USER> section.')),
]

CONF = cfg.CONF
CONF.register_opts(rbd_opts)


class StoreLocation(glance.store.location.StoreLocation):
    """
    Class describing a RBD URI. This is of the form:

        rbd://image

        or

        rbd://fsid/pool/image/snapshot
    """

    def process_specs(self):
        # convert to ascii since librbd doesn't handle unicode
        for key, value in six.iteritems(self.specs):
            self.specs[key] = str(value)
        self.fsid = self.specs.get('fsid')
        self.pool = self.specs.get('pool')
        self.image = self.specs.get('image')
        self.snapshot = self.specs.get('snapshot')

    def get_uri(self):
        if self.fsid and self.pool and self.snapshot:
            # ensure nothing contains / or any other url-unsafe character
            safe_fsid = urlparse.quote(self.fsid, '')
            safe_pool = urlparse.quote(self.pool, '')
            safe_image = urlparse.quote(self.image, '')
            safe_snapshot = urlparse.quote(self.snapshot, '')
            return "rbd://%s/%s/%s/%s" % (safe_fsid, safe_pool,
                                          safe_image, safe_snapshot)
        else:
            return "rbd://%s" % self.image

    def parse_uri(self, uri):
        prefix = 'rbd://'
        if not uri.startswith(prefix):
            reason = _('URI must start with rbd://')
            msg = _LI("Invalid URI: %s") % reason
            LOG.info(msg)
            raise exception.BadStoreUri(message=reason)
        # convert to ascii since librbd doesn't handle unicode
        try:
            ascii_uri = str(uri)
        except UnicodeError:
            reason = _('URI contains non-ascii characters')
            msg = _LI("Invalid URI: %s") % reason
            LOG.info(msg)
            raise exception.BadStoreUri(message=reason)
        pieces = ascii_uri[len(prefix):].split('/')
        if len(pieces) == 1:
            self.fsid, self.pool, self.image, self.snapshot = \
                (None, None, pieces[0], None)
        elif len(pieces) == 4:
            self.fsid, self.pool, self.image, self.snapshot = \
                map(urlparse.unquote, pieces)
        else:
            reason = _('URI must have exactly 1 or 4 components')
            msg = _LI("Invalid URI: %s") % reason
            LOG.info(msg)
            raise exception.BadStoreUri(message=reason)
        if any(map(lambda p: p == '', pieces)):
            reason = _('URI cannot contain empty components')
            msg = _LI("Invalid URI: %s") % reason
            LOG.info(msg)
            raise exception.BadStoreUri(message=reason)


class ImageIterator(object):
    """
    Reads data from an RBD image, one chunk at a time.
    """

    def __init__(self, name, store):
        self.name = name
        self.pool = store.pool
        self.user = store.user
        self.conf_file = store.conf_file
        self.chunk_size = store.chunk_size

    def __iter__(self):
        try:
            with rados.Rados(conffile=self.conf_file,
                             rados_id=self.user) as conn:
                with conn.open_ioctx(self.pool) as ioctx:
                    with rbd.Image(ioctx, self.name) as image:
                        img_info = image.stat()
                        size = img_info['size']
                        bytes_left = size
                        while bytes_left > 0:
                            length = min(self.chunk_size, bytes_left)
                            data = image.read(size - bytes_left, length)
                            bytes_left -= len(data)
                            yield data
                        raise StopIteration()
        except rbd.ImageNotFound:
            raise exception.NotFound(
                _('RBD image %s does not exist') % self.name)


class Store(glance.store.base.Store):
    """An implementation of the RBD backend adapter."""

    EXAMPLE_URL = "rbd://<FSID>/<POOL>/<IMAGE>/<SNAP>"<commit_msg>Fix RBD store to use READ_CHUNKSIZE<commit_after># Copyright 2010-2011 <NAME>
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Storage backend for RBD
   (RADOS (Reliable Autonomic Distributed Object Store) Block Device)"""
from __future__ import absolute_import
from __future__ import with_statement

import hashlib
import math

from oslo.config import cfg
import six
import six.moves.urllib.parse as urlparse
from six import text_type

from glance.common import exception
from glance.common import utils
from glance import i18n
from glance.openstack.common import excutils
import glance.openstack.common.log as logging
from glance.openstack.common import units
import glance.store.base
import glance.store.location

try:
    import rados
    import rbd
except ImportError:
    rados = None
    rbd = None

DEFAULT_POOL = 'images'
DEFAULT_CONFFILE = '/etc/ceph/ceph.conf'
DEFAULT_USER = None    # let librados decide based on the Ceph conf file
DEFAULT_CHUNKSIZE = 8  # in MiB
DEFAULT_SNAPNAME = 'snap'

LOG = logging.getLogger(__name__)
_LI = i18n._LI

rbd_opts = [
    cfg.IntOpt('rbd_store_chunk_size', default=DEFAULT_CHUNKSIZE,
               help=_('RADOS images will be chunked into objects of this size '
                      '(in megabytes). For best performance, this should be '
                      'a power of two.')),
    cfg.StrOpt('rbd_store_pool', default=DEFAULT_POOL,
               help=_('RADOS pool in which images are stored.')),
    cfg.StrOpt('rbd_store_user', default=DEFAULT_USER,
               help=_('RADOS user to authenticate as (only applicable if '
                      'using Cephx. If <None>, a default will be chosen based '
                      'on the client. section in rbd_store_ceph_conf).')),
    cfg.StrOpt('rbd_store_ceph_conf', default=DEFAULT_CONFFILE,
               help=_('Ceph configuration file path. '
                      'If <None>, librados will locate the default config. '
                      'If using cephx authentication, this file should '
                      'include a reference to the right keyring '
                      'in a client.<USER> section.')),
]

CONF = cfg.CONF
CONF.register_opts(rbd_opts)


class StoreLocation(glance.store.location.StoreLocation):
    """
    Class describing a RBD URI. This is of the form:

        rbd://image

        or

        rbd://fsid/pool/image/snapshot
    """

    def process_specs(self):
        # convert to ascii since librbd doesn't handle unicode
        for key, value in six.iteritems(self.specs):
            self.specs[key] = str(value)
        self.fsid = self.specs.get('fsid')
        self.pool = self.specs.get('pool')
        self.image = self.specs.get('image')
        self.snapshot = self.specs.get('snapshot')

    def get_uri(self):
        if self.fsid and self.pool and self.snapshot:
            # ensure nothing contains / or any other url-unsafe character
            safe_fsid = urlparse.quote(self.fsid, '')
            safe_pool = urlparse.quote(self.pool, '')
            safe_image = urlparse.quote(self.image, '')
            safe_snapshot = urlparse.quote(self.snapshot, '')
            return "rbd://%s/%s/%s/%s" % (safe_fsid, safe_pool,
                                          safe_image, safe_snapshot)
        else:
            return "rbd://%s" % self.image

    def parse_uri(self, uri):
        prefix = 'rbd://'
        if not uri.startswith(prefix):
            reason = _('URI must start with rbd://')
            msg = _LI("Invalid URI: %s") % reason
            LOG.info(msg)
            raise exception.BadStoreUri(message=reason)
        # convert to ascii since librbd doesn't handle unicode
        try:
            ascii_uri = str(uri)
        except UnicodeError:
            reason = _('URI contains non-ascii characters')
            msg = _LI("Invalid URI: %s") % reason
            LOG.info(msg)
            raise exception.BadStoreUri(message=reason)
        pieces = ascii_uri[len(prefix):].split('/')
        if len(pieces) == 1:
            self.fsid, self.pool, self.image, self.snapshot = \
                (None, None, pieces[0], None)
        elif len(pieces) == 4:
            self.fsid, self.pool, self.image, self.snapshot = \
                map(urlparse.unquote, pieces)
        else:
            reason = _('URI must have exactly 1 or 4 components')
            msg = _LI("Invalid URI: %s") % reason
            LOG.info(msg)
            raise exception.BadStoreUri(message=reason)
        if any(map(lambda p: p == '', pieces)):
            reason = _('URI cannot contain empty components')
            msg = _LI("Invalid URI: %s") % reason
            LOG.info(msg)
            raise exception.BadStoreUri(message=reason)


class ImageIterator(object):
    """
    Reads data from an RBD image, one chunk at a time.
    """

    def __init__(self, name, store):
        self.name = name
        self.pool = store.pool
        self.user = store.user
        self.conf_file = store.conf_file
        self.chunk_size = store.READ_CHUNKSIZE

    def __iter__(self):
        try:
            with rados.Rados(conffile=self.conf_file,
                             rados_id=self.user) as conn:
                with conn.open_ioctx(self.pool) as ioctx:
                    with rbd.Image(ioctx, self.name) as image:
                        img_info = image.stat()
                        size = img_info['size']
                        bytes_left = size
                        while bytes_left > 0:
                            length = min(self.chunk_size, bytes_left)
                            data = image.read(size - bytes_left, length)
                            bytes_left -= len(data)
                            yield data
                        raise StopIteration()
        except rbd.ImageNotFound:
            raise exception.NotFound(
                _('RBD image %s does not exist') % self.name)


class Store(glance.store.base.Store):
    """An implementation of the RBD backend adapter."""

    EXAMPLE_URL = "rbd://<FSID>/<POOL>/<IMAGE>/<SNAP>"
{'fraction_non_alphanumeric': 0.09667627864622856, 'fraction_numerical': 0.0034906662619517376, 'mean_word_length': 2.060613098002787, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2053765', 'n_tokens_mistral': 3789, 'n_tokens_neox': 3676, 'n_words': 1108}