<commit_before># Copyright (c) 2010-2012 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import cPickle as pickle
from collections import defaultdict
from gzip import GzipFile
from os.path import getmtime
from struct import unpack_from
from time import time
import os

from swift.common.utils import hash_path, validate_configuration
from swift.common.ring.utils import tiers_for_dev


class RingData(object):
    """Partitioned consistent hashing ring data (used for serialization)."""

    def __init__(self, replica2part2dev_id, devs, part_shift):
        self.devs = devs
        self._replica2part2dev_id = replica2part2dev_id
        self._part_shift = part_shift

    def to_dict(self):
        return {'devs': self.devs,
                'replica2part2dev_id': self._replica2part2dev_id,
                'part_shift': self._part_shift}


class Ring(object):
    """
    Partitioned consistent hashing ring.

    :param pickle_gz_path: path to ring file
    :param reload_time: time interval in seconds to check for a ring change
    """

    def __init__(self, pickle_gz_path, reload_time=15, ring_name=None):
        # can't use the ring unless HASH_PATH_SUFFIX is set
        validate_configuration()
        if ring_name:
            self.pickle_gz_path = os.path.join(pickle_gz_path,
                    ring_name + '.ring.gz')
        else:
            self.pickle_gz_path = os.path.join(pickle_gz_path)
        self.reload_time = reload_time
        self._reload(force=True)

    def _reload(self, force=False):
        self._rtime = time() + self.reload_time
        if force or self.has_changed():
            ring_data = pickle.load(GzipFile(self.pickle_gz_path, 'rb'))
            if not hasattr(ring_data, 'devs'):
                ring_data = RingData(ring_data['replica2part2dev_id'],
                    ring_data['devs'], ring_data['part_shift'])
            self._mtime = getmtime(self.pickle_gz_path)
            self.devs = ring_data.devs

            self._replica2part2dev_id = ring_data._replica2part2dev_id
            self._part_shift = ring_data._part_shift
            self._rebuild_tier_data()

    def _rebuild_tier_data(self):
        self.tier2devs = defaultdict(list)
        for dev in self.devs:
            if not dev:
                continue
            for tier in tiers_for_dev(dev):
                self.tier2devs[tier].append(dev)

        tiers_by_length = defaultdict(list)
        for tier in self.tier2devs.keys():
            tiers_by_length[len(tier)].append(tier)
        self.tiers_by_length = sorted(tiers_by_length.values(),
                                      key=lambda x: len(x[0]))
        for tiers in self.tiers_by_length:
            tiers.sort()

    @property
    def replica_count(self):
        """Number of replicas used in the ring."""
        return len(self._replica2part2dev_id)<commit_msg>Use buffered gzip stream with python 2.7.<commit_after># Copyright (c) 2010-2012 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import cPickle as pickle
from collections import defaultdict
from gzip import GzipFile
from os.path import getmtime
from struct import unpack_from
from time import time
import os
from io import BufferedReader

from swift.common.utils import hash_path, validate_configuration
from swift.common.ring.utils import tiers_for_dev


class RingData(object):
    """Partitioned consistent hashing ring data (used for serialization)."""

    def __init__(self, replica2part2dev_id, devs, part_shift):
        self.devs = devs
        self._replica2part2dev_id = replica2part2dev_id
        self._part_shift = part_shift

    def to_dict(self):
        return {'devs': self.devs,
                'replica2part2dev_id': self._replica2part2dev_id,
                'part_shift': self._part_shift}


class Ring(object):
    """
    Partitioned consistent hashing ring.

    :param pickle_gz_path: path to ring file
    :param reload_time: time interval in seconds to check for a ring change
    """

    def __init__(self, pickle_gz_path, reload_time=15, ring_name=None):
        # can't use the ring unless HASH_PATH_SUFFIX is set
        validate_configuration()
        if ring_name:
            self.pickle_gz_path = os.path.join(pickle_gz_path,
                    ring_name + '.ring.gz')
        else:
            self.pickle_gz_path = os.path.join(pickle_gz_path)
        self.reload_time = reload_time
        self._reload(force=True)

    def _reload(self, force=False):
        self._rtime = time() + self.reload_time
        if force or self.has_changed():
            ring_data = pickle.load(self._get_gz_file())
            if not hasattr(ring_data, 'devs'):
                ring_data = RingData(ring_data['replica2part2dev_id'],
                    ring_data['devs'], ring_data['part_shift'])
            self._mtime = getmtime(self.pickle_gz_path)
            self.devs = ring_data.devs

            self._replica2part2dev_id = ring_data._replica2part2dev_id
            self._part_shift = ring_data._part_shift
            self._rebuild_tier_data()

    def _get_gz_file(self):
        gz_file = GzipFile(self.pickle_gz_path, 'rb')
        if hasattr(gz_file, '_checkReadable'):
            return BufferedReader(gz_file)
        else:
            # Python 2.6 doesn't support BufferedIO
            return gz_file

    def _rebuild_tier_data(self):
        self.tier2devs = defaultdict(list)
        for dev in self.devs:
            if not dev:
                continue
            for tier in tiers_for_dev(dev):
                self.tier2devs[tier].append(dev)

        tiers_by_length = defaultdict(list)
        for tier in self.tier2devs.keys():
            tiers_by_length[len(tier)].append(tier)
        self.tiers_by_length = sorted(tiers_by_length.values(),
                                      key=lambda x: len(x[0]))
        for tiers in self.tiers_by_length:
            tiers.sort()

    @property
    def replica_count(self):
        """Number of replicas used in the ring."""
        return len(self._replica2part2dev_id)
{'fraction_non_alphanumeric': 0.08222473818284744, 'fraction_numerical': 0.01075573167279932, 'mean_word_length': 2.6559751681324366, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1043522', 'n_tokens_mistral': 2191, 'n_tokens_neox': 2099, 'n_words': 576}