<commit_before># Copyright (c) 2014 OpenStack Foundation
# Copyright (c) 2017 OpenIO SAS
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from six.moves.urllib.parse import parse_qs, quote, unquote, urlencode
from swift.common.middleware import versioned_writes as vw
from swift.common.swob import Request, HTTPException
from swift.common.utils import config_true_value, json, \
    register_swift_info, split_path, closing_if_possible, close_if_possible
from swift.proxy.controllers.base import get_container_info, get_object_info


VERSIONING_SUFFIX = '+versioning'


def swift3_versioned_object_name(object_name, version_id=None):
    if version_id is not None:
        version_id = '/%s' % version_id
    return '%03x%s%s' % (len(object_name), object_name, version_id)


def swift3_split_object_name_version(object_name):
    if '/' not in object_name or \
            len(object_name) < 3 or \
            not object_name[:3].isdigit():
        return object_name, None
    return object_name[3:].rsplit('/', 1)


def get_unversioned_container(container):
    if container.endswith(VERSIONING_SUFFIX):
        return container[:-len(VERSIONING_SUFFIX)]
    return container


def is_deleted(obj):
    return obj.get('content_type') == vw.DELETE_MARKER_CONTENT_TYPE


class OioVersionedWritesContext(vw.VersionedWritesContext):

    def handle_container_listing(self, env, start_response):
        # This code may be clearer by using Request(env).get_response()
        # instead of self._app_call(env)
        api_vers, account, container_name = split_path(
            env['PATH_INFO'], 3, 3, True)
        sub_env = env.copy()
        orig_container = get_unversioned_container(container_name)
        if orig_container != container_name:
            # Check that container_name is actually the versioning
            # container for orig_container
            sub_env['PATH_INFO'] = '/%s/%s/%s' % (api_vers, account,
                                                  orig_container)
            info = get_container_info(sub_env, self.app,
                                      swift_source='VW')
            if info.get('sysmeta', {}).get('versions-location') != \
                    container_name:
                # We were wrong, do a standard listing
                orig_container = container_name

        if orig_container != container_name:
            qs = parse_qs(sub_env.get('QUERY_STRING', ''))
            if 'marker' in qs:
                marker, _ = swift3_split_object_name_version(qs['marker'][0])
                qs['marker'] = [marker]
            if 'prefix' in qs:
                prefix, _ = swift3_split_object_name_version(qs['prefix'][0])
                qs['prefix'] = prefix
            sub_env['QUERY_STRING'] = urlencode(qs, True)
            sub_env['oio.query'] = {'versions': True}

        resp = super(OioVersionedWritesContext, self).handle_container_request(
            sub_env, lambda x, y, z: None)

        if orig_container != container_name and \
                self._response_status == '200 OK':
            with closing_if_possible(resp):
                versioned_objects = json.loads("".join(resp))

            # Discard the latest version of each object, because it is
            # not supposed to appear in the versioning container.
            latest = dict()
            for obj in versioned_objects:
                ver = int(obj.get('version', '0'))
                if ver > latest.get(obj['name'], 0):
                    latest[obj['name']] = ver
            versioned_objects = [obj for obj in versioned_objects
                                 if int(obj.get('version', '0')) !=
                                 latest[obj['name']] or
                                 is_deleted(obj)]<commit_msg>versioned_writes: enforce JSON format for subrequests<commit_after># Copyright (c) 2014 OpenStack Foundation
# Copyright (c) 2017 OpenIO SAS
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from six.moves.urllib.parse import parse_qs, quote, unquote, urlencode
from swift.common.middleware import versioned_writes as vw
from swift.common.swob import Request, HTTPException
from swift.common.utils import config_true_value, json, \
    register_swift_info, split_path, closing_if_possible, close_if_possible
from swift.proxy.controllers.base import get_container_info, get_object_info


VERSIONING_SUFFIX = '+versioning'


def swift3_versioned_object_name(object_name, version_id=None):
    if version_id is not None:
        version_id = '/%s' % version_id
    return '%03x%s%s' % (len(object_name), object_name, version_id)


def swift3_split_object_name_version(object_name):
    if '/' not in object_name or \
            len(object_name) < 3 or \
            not object_name[:3].isdigit():
        return object_name, None
    return object_name[3:].rsplit('/', 1)


def get_unversioned_container(container):
    if container.endswith(VERSIONING_SUFFIX):
        return container[:-len(VERSIONING_SUFFIX)]
    return container


def is_deleted(obj):
    return obj.get('content_type') == vw.DELETE_MARKER_CONTENT_TYPE


class OioVersionedWritesContext(vw.VersionedWritesContext):

    def handle_container_listing(self, env, start_response):
        # This code may be clearer by using Request(env).get_response()
        # instead of self._app_call(env)
        api_vers, account, container_name = split_path(
            env['PATH_INFO'], 3, 3, True)
        sub_env = env.copy()
        orig_container = get_unversioned_container(container_name)
        if orig_container != container_name:
            # Check that container_name is actually the versioning
            # container for orig_container
            sub_env['PATH_INFO'] = '/%s/%s/%s' % (api_vers, account,
                                                  orig_container)
            info = get_container_info(sub_env, self.app,
                                      swift_source='VW')
            if info.get('sysmeta', {}).get('versions-location') != \
                    container_name:
                # We were wrong, do a standard listing
                orig_container = container_name

        if orig_container != container_name:
            qs = parse_qs(sub_env.get('QUERY_STRING', ''))
            if 'marker' in qs:
                marker, _ = swift3_split_object_name_version(qs['marker'][0])
                qs['marker'] = [marker]
            if 'prefix' in qs:
                prefix, _ = swift3_split_object_name_version(qs['prefix'][0])
                qs['prefix'] = prefix
            qs['format'] = 'json'
            sub_env['QUERY_STRING'] = urlencode(qs, True)
            sub_env['oio.query'] = {'versions': True}

        resp = super(OioVersionedWritesContext, self).handle_container_request(
            sub_env, lambda x, y, z: None)

        if orig_container != container_name and \
                self._response_status == '200 OK':
            with closing_if_possible(resp):
                versioned_objects = json.loads("".join(resp))

            # Discard the latest version of each object, because it is
            # not supposed to appear in the versioning container.
            latest = dict()
            for obj in versioned_objects:
                ver = int(obj.get('version', '0'))
                if ver > latest.get(obj['name'], 0):
                    latest[obj['name']] = ver
            versioned_objects = [obj for obj in versioned_objects
                                 if int(obj.get('version', '0')) !=
                                 latest[obj['name']] or
                                 is_deleted(obj)]
{'fraction_non_alphanumeric': 0.09458517313502207, 'fraction_numerical': 0.007436672089240065, 'mean_word_length': 2.4775757575757575, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2359133', 'n_tokens_mistral': 2451, 'n_tokens_neox': 2361, 'n_words': 658}