<commit_before># -*- coding: utf-8 -*-
"""
    tipfy.ext.sharded_counter
    ~~~~~~~~~~~~~~~~~~~~~~~~~

    A general purpose sharded counter implementation for the datastore.

    :copyright: 2008 <NAME>.
    :copyright: 2010 <NAME>.
    :license: Apache, see LICENSE.txt for more details.
"""
import string
import random
import logging

from google.appengine.api import memcache
from google.appengine.ext import db
from google.appengine.runtime import apiproxy_errors

from tipfy import get_config

#: Default configuration values for this module. Keys are:
#:
#: - ``shards``: The amount of shards to use.
default_config = {
    'shards': 10,
}


class MemcachedCount(object):
    # Allows negative numbers in unsigned memcache
    DELTA_ZERO = 500000

    @property
    def namespace(self):
        return __name__ + '.' + self.__class__.__name__

    def __init__(self, name):
        self.key = 'MemcachedCount' + name

    def get_count(self):
        value = memcache.get(self.key, namespace=self.namespace)
        if value is None:
            return 0
        else:
            return string.atoi(value) - MemcachedCount.DELTA_ZERO

    def set_count(self, value):
        memcache.set(self.key, str(MemcachedCount.DELTA_ZERO + value),
            namespace=self.namespace)

    def delete_count(self):
        memcache.delete(self.key)

    count = property(get_count, set_count, delete_count)

    def increment(self, incr=1):
        value = memcache.get(self.key, namespace=self.namespace)
        if value is None:
            self.count = incr
        elif incr > 0:
            memcache.incr(self.key, incr, namespace=self.namespace)
        elif incr < 0:
            memcache.decr(self.key, -incr, namespace=self.namespace)


class Counter(object):
    """A counter using sharded writes to prevent contentions.

    Should be used for counters that handle a lot of concurrent use.
    Follows pattern described in Google I/O talk:
        http://sites.google.com/site/io/building-scalable-web-applications-with-google-app-engine

    Memcache is used for caching counts and if a cached count is available, it is
    the most correct. If there are datastore put issues, we store the un-put values
    into a delayed_incr memcache that will be applied as soon as the next shard put
    is successful. Changes will only be lost if we lose memcache before a successful
    datastore shard put or there's a failure/error in memcache.

    Usage:
        hits = Counter('hits')
        hits.increment()
        my_hits = hits.count
        hits.get_count(nocache=True)  # Forces non-cached count of all shards
        hits.count = 6                # Set the counter to arbitrary value
        hits.increment(incr=-1)       # Decrement
        hits.increment(10)
    """
    #: Number of shards to use.
    shards = None

    def __init__(self, name):
        self.name = name
        self.memcached = MemcachedCount('counter:' + name)
        self.delayed_incr = MemcachedCount('delayed:' + name)

    @property
    def number_of_shards(self):
        return self.shards or get_config(__name__, 'shards')

    def delete(self):<commit_msg>Formatting.<commit_after># -*- coding: utf-8 -*-
"""
    tipfy.ext.sharded_counter
    ~~~~~~~~~~~~~~~~~~~~~~~~~

    A general purpose sharded counter implementation for the datastore.

    :copyright: 2008 <NAME>.
    :copyright: 2010 <NAME>.
    :license: Apache, see LICENSE.txt for more details.
"""
import string
import random
import logging

from google.appengine.api import memcache
from google.appengine.ext import db
from google.appengine.runtime import apiproxy_errors

from tipfy import get_config

#: Default configuration values for this module. Keys are:
#:
#: - ``shards``: The amount of shards to use.
default_config = {
    'shards': 10,
}


class MemcachedCount(object):
    # Allows negative numbers in unsigned memcache
    DELTA_ZERO = 500000

    @property
    def namespace(self):
        return __name__ + '.' + self.__class__.__name__

    def __init__(self, name):
        self.key = 'MemcachedCount' + name

    def get_count(self):
        value = memcache.get(self.key, namespace=self.namespace)
        if value is None:
            return 0
        else:
            return string.atoi(value) - MemcachedCount.DELTA_ZERO

    def set_count(self, value):
        memcache.set(self.key, str(MemcachedCount.DELTA_ZERO + value),
            namespace=self.namespace)

    def delete_count(self):
        memcache.delete(self.key)

    count = property(get_count, set_count, delete_count)

    def increment(self, incr=1):
        value = memcache.get(self.key, namespace=self.namespace)
        if value is None:
            self.count = incr
        elif incr > 0:
            memcache.incr(self.key, incr, namespace=self.namespace)
        elif incr < 0:
            memcache.decr(self.key, -incr, namespace=self.namespace)


class Counter(object):
    """A counter using sharded writes to prevent contentions.

    Should be used for counters that handle a lot of concurrent use.
    Follows pattern described in Google I/O talk:

        http://sites.google.com/site/io/building-scalable-web-applications-with-google-app-engine

    Memcache is used for caching counts and if a cached count is available,
    it is the most correct. If there are datastore put issues, we store the
    un-put values into a delayed_incr memcache that will be applied as soon
    as the next shard put is successful. Changes will only be lost if we lose
    memcache before a successful datastore shard put or there's a
    failure/error in memcache.

    Example usage:

        # Build a new counter that uses the unique key name 'hits'.
        hits = Counter('hits')
        # Increment by 1.
        hits.increment()
        # Increment by 10.
        hits.increment(10)
        # Decrement by 3.
        hits.increment(-3)
        # This is the current count.
        my_hits = hits.count
        # Forces fetching a non-cached count of all shards.
        hits.get_count(nocache=True)
        # Set the counter to an arbitrary value.
        hits.count = 6
    """
    #: Number of shards to use.
    shards = None

    def __init__(self, name):
        self.name = name
        self.memcached = MemcachedCount('counter:' + name)
        self.delayed_incr = MemcachedCount('delayed:' + name)

    @property
    def number_of_shards(self):
        return self.shards or get_config(__name__, 'shards')

    def delete(self):
{'fraction_non_alphanumeric': 0.09468822170900693, 'fraction_numerical': 0.008314087759815243, 'mean_word_length': 2.6679841897233203, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '257609', 'n_tokens_mistral': 1974, 'n_tokens_neox': 1875, 'n_words': 607}