<commit_before>#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import time
from webob import Request, Response

from swift.common.utils import split_path, cache_from_env, get_logger
from swift.proxy.server import get_container_memcache_key


class MaxSleepTimeHit(Exception):
    pass


class RateLimitMiddleware(object):
    """
    Rate limiting middleware
    """

    def __init__(self, app, conf, logger=None):
        self.app = app
        if logger:
            self.logger = logger
        else:
            self.logger = get_logger(conf)
        self.account_rate_limit = float(conf.get('account_ratelimit', 200.0))
        self.max_sleep_time_seconds = float(conf.get('max_sleep_time_seconds',
                                                   60))
        self.clock_accuracy = int(conf.get('clock_accuracy', 1000))
        self.rate_limit_whitelist = [acc.strip() for acc in
            conf.get('account_whitelist', '').split(',')
            if acc.strip()]
        self.rate_limit_blacklist = [acc.strip() for acc in
            conf.get('account_blacklist', '').split(',')
            if acc.strip()]
        self.memcache_client = None
        conf_limits = []
        for conf_key in conf.keys():
            if conf_key.startswith('container_limit_'):
                cont_size = int(conf_key[len('container_limit_'):])
                rate = float(conf[conf_key])
                conf_limits.append((cont_size, rate))

        conf_limits.sort()
        self.container_limits = []
        while conf_limits:
            cur_size, cur_rate = conf_limits.pop(0)
            if conf_limits:
                next_size, next_rate = conf_limits[0]
                slope = (float(next_rate) - float(cur_rate)) \
                      / (next_size - cur_size)

                def new_scope(cur_size, slope, cur_rate):
                    # making new scope for variables
                    return lambda x: (x - cur_size) * slope + cur_rate
                line_func = new_scope(cur_size, slope, cur_rate)
            else:
                line_func = lambda x: cur_rate

            self.container_limits.append((cur_size, cur_rate, line_func))

    def get_container_maxrate(self, container_size):
        last_func = None
        if container_size:
            container_size = int(container_size)
            for size, rate, func in self.container_limits:
                if container_size < size:
                    break
                last_func = func

            if last_func:
                return last_func(container_size)
        return None

    def get_ratelimitable_key_tuples(self, req_method,
                                     account_name, container_name, obj_name):
        """
        Returns a list of key (used in memcache), ratelimit tuples. Keys 
        should be checked in order.
        """
        keys = []
        if account_name and (
            not (container_name or obj_name) or
            (container_name and not obj_name and req_method == 'PUT')):
            keys.append(("ratelimit/%s" % account_name,
                         self.account_rate_limit))
<commit_msg>getting rid account ratelimiting default<commit_after>#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import time
from webob import Request, Response

from swift.common.utils import split_path, cache_from_env, get_logger
from swift.proxy.server import get_container_memcache_key


class MaxSleepTimeHit(Exception):
    pass


class RateLimitMiddleware(object):
    """
    Rate limiting middleware
    """

    def __init__(self, app, conf, logger=None):
        self.app = app
        if logger:
            self.logger = logger
        else:
            self.logger = get_logger(conf)
        self.account_rate_limit = float(conf.get('account_ratelimit', 0))
        self.max_sleep_time_seconds = float(conf.get('max_sleep_time_seconds',
                                                   60))
        self.clock_accuracy = int(conf.get('clock_accuracy', 1000))
        self.rate_limit_whitelist = [acc.strip() for acc in
            conf.get('account_whitelist', '').split(',')
            if acc.strip()]
        self.rate_limit_blacklist = [acc.strip() for acc in
            conf.get('account_blacklist', '').split(',')
            if acc.strip()]
        self.memcache_client = None
        conf_limits = []
        for conf_key in conf.keys():
            if conf_key.startswith('container_limit_'):
                cont_size = int(conf_key[len('container_limit_'):])
                rate = float(conf[conf_key])
                conf_limits.append((cont_size, rate))

        conf_limits.sort()
        self.container_limits = []
        while conf_limits:
            cur_size, cur_rate = conf_limits.pop(0)
            if conf_limits:
                next_size, next_rate = conf_limits[0]
                slope = (float(next_rate) - float(cur_rate)) \
                      / (next_size - cur_size)

                def new_scope(cur_size, slope, cur_rate):
                    # making new scope for variables
                    return lambda x: (x - cur_size) * slope + cur_rate
                line_func = new_scope(cur_size, slope, cur_rate)
            else:
                line_func = lambda x: cur_rate

            self.container_limits.append((cur_size, cur_rate, line_func))

    def get_container_maxrate(self, container_size):
        last_func = None
        if container_size:
            container_size = int(container_size)
            for size, rate, func in self.container_limits:
                if container_size < size:
                    break
                last_func = func

            if last_func:
                return last_func(container_size)
        return None

    def get_ratelimitable_key_tuples(self, req_method,
                                     account_name, container_name, obj_name):
        """
        Returns a list of key (used in memcache), ratelimit tuples. Keys 
        should be checked in order.
        """
        keys = []
        if self.account_rate_limit and account_name and (
            not (container_name or obj_name) or
            (container_name and not obj_name and req_method == 'PUT')):
            keys.append(("ratelimit/%s" % account_name,
                         self.account_rate_limit))

{'fraction_non_alphanumeric': 0.08569472963723478, 'fraction_numerical': 0.003969883641341547, 'mean_word_length': 2.07362221287337, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2216177', 'n_tokens_mistral': 2031, 'n_tokens_neox': 1961, 'n_words': 552}