<commit_before>from __future__ import unicode_literals

import json
import logging
import time
import urllib2

logger = logging.getLogger(__name__)


class Dirble(object):
    """Light wrapper for Dirble API lookup.

    Important things to note:
    - The client will do exponential back off when requests fail or timeout.
    - The client will cache results aggressively.
    - Failed requests will return an empty default type appropriate for the
      lookup in question, normally a empty dict or list.
    - The data returned comes direct from the API's JSON.
    - For stations, only the minimal data set returned by station lists is
      guaranteed to be there.
    """
    def __init__(self, api_key, timeout):
        self._base_uri = 'http://dirble.com/dirapi/%s/apikey/' + api_key
        self._cache = {}
        self._stations = {}
        self._timeout = timeout / 1000.0
        self._backoff_until = time.time()
        self._backoff_max = 60
        self._backoff = 1

    def flush(self):
        self._cache = {}
        self._stations = {}

    def categories(self):
        uri = self._base_uri % 'primaryCategories'
        return self._fetch(uri, [])

    def sub_categories(self, id):
        uri = (self._base_uri + '/primaryid/%s') % ('childCategories', id)
        return self._fetch(uri, [])

    def stations(self, id):
        uri = (self._base_uri + '/id/%s') % ('stations', id)
        stations = self._fetch(uri, [])
        for station in stations:
            self._stations.setdefault(station['id'], station)
        return stations

    def station(self, id):
        id = int(id)  # Ensure we are consistent for cache key.
        if id in self._stations:
            return self._stations[id]
        uri = (self._base_uri + '/id/%s') % ('station', id)
        station = self._fetch(uri, {})
        if station:
            self._stations.setdefault(station['id'], station)
        return station

    def _fetch(self, uri, default):
        if uri in self._cache:
            logger.debug('Cache hit: %s', uri)
            return self._cache[uri]

        if time.time() < self._backoff_until:
            logger.debug('Back off fallack used: %s', uri)
            return default
<commit_msg>Update client internal API as prep for refactor.<commit_after>from __future__ import unicode_literals

import json
import logging
import time
import urllib2

logger = logging.getLogger(__name__)


class Dirble(object):
    """Light wrapper for Dirble API lookup.

    Important things to note:
    - The client will do exponential back off when requests fail or timeout.
    - The client will cache results aggressively.
    - Failed requests will return an empty default type appropriate for the
      lookup in question, normally a empty dict or list.
    - The data returned comes direct from the API's JSON.
    - For stations, only the minimal data set returned by station lists is
      guaranteed to be there.
    """
    def __init__(self, api_key, timeout):
        self._base_uri = 'http://dirble.com/dirapi/%s/apikey/' + api_key
        self._cache = {}
        self._stations = {}
        self._timeout = timeout / 1000.0
        self._backoff_until = time.time()
        self._backoff_max = 60
        self._backoff = 1

    def flush(self):
        self._cache = {}
        self._stations = {}

    def categories(self):
        return self._fetch('primaryCategories', '', [])

    def sub_categories(self, category):
        path = '/primaryid/%s' % category
        return self._fetch('childCategories', path, [])

    def stations(self, category):
        path = '/id/%s' % category
        stations = self._fetch('stations', path, [])
        for station in stations:
            self._stations.setdefault(station['id'], station)
        return stations

    def station(self, identifier):
        identifier = int(identifier)  # Ensure we are consistent for cache key.
        if identifier in self._stations:
            return self._stations[identifier]
        path = '/id/%s' % identifier
        station = self._fetch('station', path, {})
        if station:
            self._stations.setdefault(station['id'], station)
        return station

    def _fetch(self, variant, path, default):
        uri = (self._base_uri % variant) + path
        if uri in self._cache:
            logger.debug('Cache hit: %s', uri)
            return self._cache[uri]

        if time.time() < self._backoff_until:
            logger.debug('Back off fallback used: %s', uri)
            return default

{'fraction_non_alphanumeric': 0.10255281690140845, 'fraction_numerical': 0.003961267605633803, 'mean_word_length': 2.37667161961367, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1645593', 'n_tokens_mistral': 1273, 'n_tokens_neox': 1236, 'n_words': 410}