<commit_before>from django.core.cache import cache
from django.conf import settings
from datetime import datetime
import chardet
import json
import logging
import re
import time
import urllib2

logger      = logging.getLogger(__name__)
API_URL     = settings.FAEXPORT_API_URL
API_TIMEOUT = settings.FAEXPORT_API_TIMEOUT
USER_AGENT  = 'Mozilla/5.0 (Windows NT 6.1; rv:31.0) Gecko/20100101 Firefox/31.0'
USER_REGEX  = re.compile(r'^http://www.furaffinity.net/user/([-_.~\[\]\w]+)/')
MEDIA_REGEX = re.compile(r'(^http:| src=")/(/[adt]\.facdn\.net|themes)(/.*?\.[a-zA-Z0-9]+)(#[^"]+)?($|")')

MEDIA_PREFIX = {
    '/a.facdn.net' : settings.MEDIA_URL + 'avatar',
    '/d.facdn.net' : settings.MEDIA_URL + 'data',
    '/t.facdn.net' : settings.MEDIA_URL + 'thumbnail',
    'themes'       : settings.MEDIA_URL + 'themes',
}

class RegexFetcher(object):
    def __init__(self):
        pass

    def __call__(self, match):
        url_root    = match.group(2)
        url_path    = urllib2.quote(match.group(3).encode('utf-8'), '/:@')
        local_url   = MEDIA_PREFIX[url_root] + url_path

        if match.group(1) == 'http:':
            return local_url
        else:
            return match.group(1) + local_url + match.group(5)

def api_fetch(api_key, extra_args=''):
    cache_key = api_key + extra_args
    data = cache.get(cache_key)

    if data is None:
        logger.debug('CACHE MISS: {}'.format(cache_key))
        request_url = '{}/{}.json{}'.format(API_URL, api_key, extra_args)
        request_obj = urllib2.Request(request_url)
        request_obj.add_header('User-Agent', USER_AGENT)

        opener      = urllib2.build_opener()
        raw_data    = opener.open(request_obj).read()
        data        = json.loads(raw_data)

        cache.set(cache_key, data, API_TIMEOUT)
    else:
        logger.debug('CACHE HIT: {}'.format(cache_key))

    return data

def api_fetch_watchs(name, watch_view, limit=None):
    watchs = []
    index = 1

    while True:
        try:
            batch = api_fetch(
                'user/{}/{}'.format(name, watch_view),
                '?page={}'.format(index) if index > 1 else '',
            )<commit_msg>Escape invalid characters in API fetch URL<commit_after>from django.core.cache import cache
from django.conf import settings
from datetime import datetime
import chardet
import json
import logging
import re
import time
import urllib2

logger      = logging.getLogger(__name__)
API_URL     = settings.FAEXPORT_API_URL
API_TIMEOUT = settings.FAEXPORT_API_TIMEOUT
USER_AGENT  = 'Mozilla/5.0 (Windows NT 6.1; rv:31.0) Gecko/20100101 Firefox/31.0'
USER_REGEX  = re.compile(r'^http://www.furaffinity.net/user/([-_.~\[\]\w]+)/')
MEDIA_REGEX = re.compile(r'(^http:| src=")/(/[adt]\.facdn\.net|themes)(/.*?\.[a-zA-Z0-9]+)(#[^"]+)?($|")')

MEDIA_PREFIX = {
    '/a.facdn.net' : settings.MEDIA_URL + 'avatar',
    '/d.facdn.net' : settings.MEDIA_URL + 'data',
    '/t.facdn.net' : settings.MEDIA_URL + 'thumbnail',
    'themes'       : settings.MEDIA_URL + 'themes',
}

class RegexFetcher(object):
    def __init__(self):
        pass

    def __call__(self, match):
        url_root    = match.group(2)
        url_path    = urllib2.quote(match.group(3).encode('utf-8'), '/:@')
        local_url   = MEDIA_PREFIX[url_root] + url_path

        if match.group(1) == 'http:':
            return local_url
        else:
            return match.group(1) + local_url + match.group(5)

def api_fetch(api_key, extra_args=''):
    cache_key = api_key + extra_args
    data = cache.get(cache_key)

    if data is None:
        logger.debug('CACHE MISS: {}'.format(cache_key))
        request_url = '{}/{}.json{}'.format(API_URL, api_key, extra_args)
        request_url = urllib2.quote(request_url, ':/?&=')
        request_obj = urllib2.Request(request_url)
        request_obj.add_header('User-Agent', USER_AGENT)

        opener      = urllib2.build_opener()
        raw_data    = opener.open(request_obj).read()
        data        = json.loads(raw_data)

        cache.set(cache_key, data, API_TIMEOUT)
    else:
        logger.debug('CACHE HIT: {}'.format(cache_key))

    return data

def api_fetch_watchs(name, watch_view, limit=None):
    watchs = []
    index = 1

    while True:
        try:
            batch = api_fetch(
                'user/{}/{}'.format(name, watch_view),
                '?page={}'.format(index) if index > 1 else '',
            )
{'fraction_non_alphanumeric': 0.15339233038348082, 'fraction_numerical': 0.014749262536873156, 'mean_word_length': 2.7643040136635353, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2790593', 'n_tokens_mistral': 1603, 'n_tokens_neox': 1544, 'n_words': 284}