<commit_before>"""
Client API to Swift

Copyright 2011 <NAME>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

__all__ = ['VERSION', 'CHUNK_SIZE', 'Client']


from os import umask
from urllib import quote
from urlparse import urlparse, urlunparse

try:
    from eventlet.green.httplib import HTTPException, HTTPSConnection
except ImportError:
    from httplib import HTTPException, HTTPSConnection

try:
    from swift.common.bufferedhttp \
        import BufferedHTTPConnection as HTTPConnection
except ImportError:
    try:
        from eventlet.green.httplib import HTTPConnection
    except ImportError:
        from httplib import HTTPConnection

try:
    import simplejson as json
except ImportError:
    import json

try:
    from eventlet import sleep
except ImportError:
    from time import sleep

try:
    from swift.proxy.server import Application as SwiftProxy
    from webob import Request
except ImportError:
    SwiftProxy = None

from swiftly import VERSION


CHUNK_SIZE = 65536


def _quote(value, safe='/:'):
    if isinstance(value, unicode):
        value = value.encode('utf8')
    return quote(value, safe)


class _LocalMemcache(object):

    def __init__(self):
        self.cache = {}

    def set(self, key, value, serialize=True, timeout=0):
        self.cache[key] = value

    def get(self, key):
        return self.cache.get(key)

    def incr(self, key, delta=1, timeout=0):
        if key not in self.cache:
            self.cache[key] = max(0, delta)
        else:
            self.cache[key] += delta
        if self.cache[key] < 0:
            self.cache[key] = 0
        return self.cache[key]

    def decr(self, key, delta=1, timeout=0):
        return self.incr(key, delta=-delta, timeout=timeout)

    def delete(self, key):
        if key in self.cache:
            del self.cache[key]

    def set_multi(self, mapping, server_key, serialize=True, timeout=0):
        for k, v in mapping.iteritems():
            self.cache[k] = v

    def get_multi(self, keys, server_key):
        return [self.cache.get(k) for k in keys]


class _IterReader(object):

    def __init__(self, iterator):
        self.iterator = iter(iterator)

    def read(self, size=-1):
        try:
            return self.iterator.next()
        except StopIteration:
            return ''


class _FileIter(object):

    def __init__(self, fp, chunked=False):
        self.fp = fp
        self.chunked = chunked
        self.final_chunk_sent = False

    def __iter__(self):
        return self

    def next(self):
        chunk = self.fp.read(CHUNK_SIZE)
        if not chunk:<commit_msg>Better behaved _LocalMemcache<commit_after>"""
Client API to Swift

Copyright 2011 <NAME>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

__all__ = ['VERSION', 'CHUNK_SIZE', 'Client']


from os import umask
from urllib import quote
from urlparse import urlparse, urlunparse

try:
    from eventlet.green.httplib import HTTPException, HTTPSConnection
except ImportError:
    from httplib import HTTPException, HTTPSConnection

try:
    from swift.common.bufferedhttp \
        import BufferedHTTPConnection as HTTPConnection
except ImportError:
    try:
        from eventlet.green.httplib import HTTPConnection
    except ImportError:
        from httplib import HTTPConnection

try:
    import simplejson as json
except ImportError:
    import json

try:
    from eventlet import sleep
except ImportError:
    from time import sleep

try:
    from swift.proxy.server import Application as SwiftProxy
    from webob import Request
except ImportError:
    SwiftProxy = None

from swiftly import VERSION


CHUNK_SIZE = 65536


def _quote(value, safe='/:'):
    if isinstance(value, unicode):
        value = value.encode('utf8')
    return quote(value, safe)


class _Node(object):

    def __init__(self, key, val, prv, nxt):
        self.key = key
        self.val = val
        self.prv = prv
        self.nxt = nxt


class _LocalMemcache(object):

    def __init__(self):
        self.cache = {}
        self.first = None
        self.last = None
        self.count = 0
        self.max_count = 1000

    def set(self, key, value, serialize=True, timeout=0):
        self.delete(key)
        self.last = node = _Node(key, value, self.last, None)
        if node.prv:
            node.prv.nxt = node
        self.cache[key] = node
        self.count += 1
        if self.count > self.max_count:
            self.delete(self.first.key)

    def get(self, key):
        node = self.cache.get(key)
        return node.val if node else None

    def incr(self, key, delta=1, timeout=0):
        node = self.cache.get(key)
        if node:
            node.val += delta
            return node.val
        else:
            self.set(key, delta)
            return delta

    def decr(self, key, delta=1, timeout=0):
        return self.incr(key, delta=-delta, timeout=timeout)

    def delete(self, key):
        node = self.cache.get(key)
        if node:
            del self.cache[key]
            if node.prv:
                node.prv.nxt = node.nxt
            if node.nxt:
                wax.nxt.prv = node.prv
            if self.first == node:
                self.first = node.nxt
            if self.last == node:
                self.last = node.prv
            self.count -= 1

    def set_multi(self, mapping, server_key, serialize=True, timeout=0):
        for key, value in mapping.iteritems():
            self.set(key, value)

    def get_multi(self, keys, server_key):
        return [self.get(k) for k in keys]


class _IterReader(object):

    def __init__(self, iterator):
        self.iterator = iter(iterator)

    def read(self, size=-1):
        try:
            return self.iterator.next()
        except StopIteration:
            return ''


class _FileIter(object):

    def __init__(self, fp, chunked=False):
        self.fp = fp
        self.chunked = chunked
        self.final_chunk_sent = False

    def __iter__(self):
        return self

    def next(self):
        chunk = self.fp.read(CHUNK_SIZE)
        if not chunk:
{'fraction_non_alphanumeric': 0.08671507902605724, 'fraction_numerical': 0.0074042432009112915, 'mean_word_length': 2.506739890164753, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '304161', 'n_tokens_mistral': 2117, 'n_tokens_neox': 2032, 'n_words': 681}