<commit_before>import base64
import os
import requests
import time
import zlib

from . import version
from compat import bytes_
from compat import text_
from errors import APIException
from requests.adapters import HTTPAdapter

try:
    from urllib.parse import urlparse, urlunparse, parse_qsl
except ImportError:  # pragma: no cover
    from urlparse import urlparse, urlunparse, parse_qsl

try:
    import pickle
except ImportError:  # pragma: no cover
    import cPickle as pickle

import logging
import re

logger = logging.getLogger("pycrest.eve")
cache_re = re.compile(r'max-age=([0-9]+)')


class APICache(object):
    def put(self, key, value):
        raise NotImplementedError

    def get(self, key):
        raise NotImplementedError

    def invalidate(self, key):
        raise NotImplementedError


class FileCache(APICache):
    def __init__(self, path):
        self._cache = {}
        self.path = path
        if not os.path.isdir(self.path):
            os.mkdir(self.path, 0o700)

    def _getpath(self, key):
        return os.path.join(self.path, str(hash(key)) + '.cache')

    def put(self, key, value):
        with open(self._getpath(key), 'wb') as f:
            f.write(zlib.compress(pickle.dumps(value, -1)))
        self._cache[key] = value

    def get(self, key):
        if key in self._cache:
            return self._cache[key]

        try:
            with open(self._getpath(key), 'rb') as f:
                return pickle.loads(zlib.decompress(f.read()))
        except IOError as ex:
            if ex.errno == 2:  # file does not exist (yet)
                return None
            else:
                raise

    def invalidate(self, key):
        self._cache.pop(key, None)

        try:
            os.unlink(self._getpath(key))
        except OSError as ex:
            if ex.errno == 2:  # does not exist
                pass
            else:
                raise


class DictCache(APICache):
    def __init__(self):
        self._dict = {}

    def get(self, key):
        return self._dict.get(key, None)

    def put(self, key, value):
        self._dict[key] = value

    def invalidate(self, key):
        self._dict.pop(key, None)


class APIConnection(object):
    def __init__(self, additional_headers=None, user_agent=None, cache_dir=None, cache=None):
        # Set up a Requests Session
        session = requests.Session()
        if additional_headers is None:
            additional_headers = {}
        if user_agent is None:
            user_agent = "PyCrest/{0}".format(version)
        session.headers.update({
            "User-Agent": user_agent,
        })
        session.headers.update(additional_headers)
        session.mount(
            'https://public-crest.eveonline.com',
            HTTPAdapter(
                pool_connections=20,
                pool_maxsize=20,
            )
        )

        self._session = session
        if cache:
            if isinstance(cache, APICache):
                self.cache = cache  # Inherit from parents
            elif isinstance(cache, type):
                self.cache = cache()  # Instantiate a new cache
        elif cache_dir:
            self.cache_dir = cache_dir
            self.cache = FileCache(self.cache_dir)
        else:
            self.cache = DictCache()

        self.current_headers = {}

    def get(self, resource, params=None, headers={}):
        logger.debug('Getting resource %s', resource)
        if params is None:
            params = {}

        # remove params from resource URI (needed for paginated stuff)
        parsed_uri = urlparse(resource)
        qs = parsed_uri.query
        resource = urlunparse(parsed_uri._replace(query=''))
        prms = {}
        for tup in parse_qsl(qs):
            prms[tup[0]] = tup[1]

        # params supplied to self.get() override parsed params
        for key in params:
            prms[key] = params[key]

        # check cache
        key = (resource, frozenset(self._session.headers.items()), frozenset(prms.items()))
        cached = self.cache.get(key)
        if cached and cached['cached_until'] > time.time():
            logger.debug('Cache hit for resource %s (params=%s)', resource, prms)
            return cached
        elif cached:
            logger.debug('Cache stale for resource %s (params=%s)', resource, prms)
            self.cache.invalidate(key)
        else:
            logger.debug('Cache miss for resource %s (params=%s', resource, prms)

        logger.debug('Getting resource %s (params=%s)', resource, prms)
        res = self._session.get(resource, params=prms, headers=headers)
        if res.status_code != 200:
            raise APIException("Got unexpected status code from server: %i\nReason: %s\nURI: %s\nParams: %s" % (res.status_code, res.reason, resource, prms))

        ret = res.json()

        # define current session header in a class attributes
        # as it will change each time we'll do get(), since we
        # pass "self" to APIObjects when we create them.
        # this way, we may access to request headers, at any time
        self.current_headers = res.headers

        # cache result
        expires = self._get_expires(res)
        if expires > 0:
            ret.update({'cached_until': time.time() + expires})
            self.cache.put(key, ret)<commit_msg>Allow "no cache" on pycrest, add the use off given cache (without instanciation)<commit_after>import base64
import os
import requests
import time
import zlib

from . import version
from compat import bytes_
from compat import text_
from errors import APIException
from requests.adapters import HTTPAdapter

try:
    from urllib.parse import urlparse, urlunparse, parse_qsl
except ImportError:  # pragma: no cover
    from urlparse import urlparse, urlunparse, parse_qsl

try:
    import pickle
except ImportError:  # pragma: no cover
    import cPickle as pickle

import logging
import re

logger = logging.getLogger("pycrest.eve")
cache_re = re.compile(r'max-age=([0-9]+)')


class APICache(object):
    def put(self, key, value):
        raise NotImplementedError

    def get(self, key):
        raise NotImplementedError

    def invalidate(self, key):
        raise NotImplementedError


class FileCache(APICache):
    def __init__(self, path):
        self._cache = {}
        self.path = path
        if not os.path.isdir(self.path):
            os.mkdir(self.path, 0o700)

    def _getpath(self, key):
        return os.path.join(self.path, str(hash(key)) + '.cache')

    def put(self, key, value):
        with open(self._getpath(key), 'wb') as f:
            f.write(zlib.compress(pickle.dumps(value, -1)))
        self._cache[key] = value

    def get(self, key):
        if key in self._cache:
            return self._cache[key]

        try:
            with open(self._getpath(key), 'rb') as f:
                return pickle.loads(zlib.decompress(f.read()))
        except IOError as ex:
            if ex.errno == 2:  # file does not exist (yet)
                return None
            else:
                raise

    def invalidate(self, key):
        self._cache.pop(key, None)

        try:
            os.unlink(self._getpath(key))
        except OSError as ex:
            if ex.errno == 2:  # does not exist
                pass
            else:
                raise


class DictCache(APICache):
    def __init__(self):
        self._dict = {}

    def get(self, key):
        return self._dict.get(key, None)

    def put(self, key, value):
        self._dict[key] = value

    def invalidate(self, key):
        self._dict.pop(key, None)


class APIConnection(object):
    def __init__(self, additional_headers=None, user_agent=None, cache_dir=None, cache='dict'):
        # Set up a Requests Session
        session = requests.Session()
        if additional_headers is None:
            additional_headers = {}
        if user_agent is None:
            user_agent = "PyCrest/{0}".format(version)
        session.headers.update({
            "User-Agent": user_agent,
        })
        session.headers.update(additional_headers)
        session.mount(
            'https://public-crest.eveonline.com',
            HTTPAdapter(
                pool_connections=20,
                pool_maxsize=20,
            )
        )

        self._session = session
        if cache:
            if isinstance(cache, APICache):
                self.cache = cache  # Inherit from parents
            elif isinstance(cache, type):
                self.cache = cache()  # Instantiate a new cache
            else:
                self.cache = cache
        elif cache_dir:
            self.cache_dir = cache_dir
            self.cache = FileCache(self.cache_dir)
        elif cache == 'dict':
            self.cache = DictCache()
        else:
            self.cache = None

        self.current_headers = {}

    def get(self, resource, params=None, headers={}):
        logger.debug('Getting resource %s', resource)
        if params is None:
            params = {}

        # remove params from resource URI (needed for paginated stuff)
        parsed_uri = urlparse(resource)
        qs = parsed_uri.query
        resource = urlunparse(parsed_uri._replace(query=''))
        prms = {}
        for tup in parse_qsl(qs):
            prms[tup[0]] = tup[1]

        # params supplied to self.get() override parsed params
        for key in params:
            prms[key] = params[key]

        # check cache
        if self.cache:
            key = (resource, frozenset(self._session.headers.items()), frozenset(prms.items()))
            cached = self.cache.get(key)
            if cached and cached['cached_until'] > time.time():
                logger.debug('Cache hit for resource %s (params=%s)', resource, prms)
                return cached
            elif cached:
                logger.debug('Cache stale for resource %s (params=%s)', resource, prms)
                self.cache.invalidate(key)
            else:
                logger.debug('Cache miss for resource %s (params=%s', resource, prms)

        logger.debug('Getting resource %s (params=%s)', resource, prms)
        res = self._session.get(resource, params=prms, headers=headers)
        if res.status_code != 200:
            raise APIException("Got unexpected status code from server: %i\nReason: %s\nURI: %s\nParams: %s" % (res.status_code, res.reason, resource, prms))

        ret = res.json()

        # define current session header in a class attributes
        # as it will change each time we'll do get(), since we
        # pass "self" to APIObjects when we create them.
        # this way, we may access to request headers, at any time
        self.current_headers = res.headers

        # cache result
        expires = self._get_expires(res)
        if expires > 0:
            ret.update({'cached_until': time.time() + expires})
            if self.cache:
                self.cache.put(key, ret)
{'fraction_non_alphanumeric': 0.10280117173196632, 'fraction_numerical': 0.004027828634199927, 'mean_word_length': 2.0653759820426485, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4422121', 'n_tokens_mistral': 3227, 'n_tokens_neox': 3097, 'n_words': 889}