<commit_before>"""
Copyright (c) 2008-2011, <NAME> <<EMAIL>>

Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.

THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
"""

import web
import cPickle as pickle
import memcache
from collections import deque
from time import time

import steam
from optf2.backend import config
from optf2.backend import log

memcached = memcache.Client([config.ini.get("cache", "memcached-address")],
                            pickleProtocol = pickle.HIGHEST_PROTOCOL)

# Keeps track of connection times, until I think of a better way
last_server_checks = {}

class cache:
    """ Cache retrieval/setting functions """

    def _get_generic_aco(self, baseclass, keyprefix, freshcallback = None, stale = False, appid = None):
        """ Initializes and caches Aggresively Cached Objects from steamodd """

        modulename = self._mod_id
        language = self._language
        lm = None
        ctime = int(time())
        memkey = "{0}-{1}-{2}".format(keyprefix, modulename, language)

        oldobj = memcached.get(memkey)
        if stale: return oldobj
        if oldobj:
            if (ctime - last_server_checks.get(memkey, 0)) < config.ini.getint("cache", keyprefix + "-check-interval"):
                return oldobj
            lm = oldobj.get_last_modified()

        result = None
        try:
            timeout = config.ini.get("steam", "connect-timeout")
            datatimeout = config.ini.get("steam", "download-timeout")
            if not appid: result = baseclass(lang = language, lm = lm, timeout = timeout, data_timeout = datatimeout)
            else: result = baseclass(appid, lang = language, lm = lm, timeout = timeout, data_timeout = datatimeout)
            if freshcallback: freshcallback(result)
            result._get()
            memcached.set(memkey, result, min_compress_len = 1048576)<commit_msg>Use config.getint, proper last-modified kwarg<commit_after>"""
Copyright (c) 2008-2011, <NAME> <<EMAIL>>

Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.

THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
"""

import web
import cPickle as pickle
import memcache
from collections import deque
from time import time

import steam
from optf2.backend import config
from optf2.backend import log

memcached = memcache.Client([config.ini.get("cache", "memcached-address")],
                            pickleProtocol = pickle.HIGHEST_PROTOCOL)

# Keeps track of connection times, until I think of a better way
last_server_checks = {}

class cache:
    """ Cache retrieval/setting functions """

    def _get_generic_aco(self, baseclass, keyprefix, freshcallback = None, stale = False, appid = None):
        """ Initializes and caches Aggresively Cached Objects from steamodd """

        modulename = self._mod_id
        language = self._language
        lm = None
        ctime = int(time())
        memkey = "{0}-{1}-{2}".format(keyprefix, modulename, language)

        oldobj = memcached.get(memkey)
        if stale: return oldobj
        if oldobj:
            if (ctime - last_server_checks.get(memkey, 0)) < config.ini.getint("cache", keyprefix + "-check-interval"):
                return oldobj
            lm = oldobj.get_last_modified()

        result = None
        try:
            timeout = config.ini.getint("steam", "connect-timeout")
            datatimeout = config.ini.getint("steam", "download-timeout")
            if not appid: result = baseclass(lang = language, last_modified = lm, timeout = timeout, data_timeout = datatimeout)
            else: result = baseclass(appid, lang = language, last_modified = lm, timeout = timeout, data_timeout = datatimeout)
            if freshcallback: freshcallback(result)
            result._get()
            memcached.set(memkey, result, min_compress_len = 1048576)
{'fraction_non_alphanumeric': 0.08199679230152365, 'fraction_numerical': 0.008420208500400962, 'mean_word_length': 3.245957446808511, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7287990', 'n_tokens_mistral': 1440, 'n_tokens_neox': 1336, 'n_words': 513}