<commit_before>import webapp2
from google.appengine.api import urlfetch
import json
from PIL import Image, ImageDraw, ImageFont
from google.appengine.api import memcache
import StringIO
import jinja2
import os
from decimal import * #used fixed point math for better accuracy
from google.appengine import runtime # for catching DeadlineExceededError
from google.appengine.api import urlfetch_errors # "

JINJA_ENVIRONMENT = jinja2.Environment(
    loader=jinja2.FileSystemLoader(os.path.dirname(__file__)))

def urlfetch_cache(url,exchange):
    # fetches a url, but using memcache to not hammer the exchanges server
    data = memcache.get(url)
    if data is not None:
        return process_json(data, exchange)
    else:
        try:
            result = urlfetch.fetch(url,deadline=30) #timeout after 10 sec
            if result.status_code == 200:
                value = process_json(result.content, exchange)
                memcache.add(url, result.content, 30) #cache for 30 sec
                memcache.add('longcache'+url, result.content, 3000) #also cache for 5min in case of timeouts
                return value
            else:
                return 'Error: '+exchange+' status code '+str(result.status_code) #'Error accessing Vircurex API'
        except runtime.DeadlineExceededError: #raised if the overall request times out
            data = memcache.get('longcache'+url)
            if data is not None: return process_json(data, exchange)
            else: return 'Error: timeout'
        except runtime.apiproxy_errors.DeadlineExceededError: #raised if an RPC exceeded its deadline (set)
            data = memcache.get('longcache'+url)
            if data is not None: return process_json(data, exchange)
            else: return 'Error: timeout'
        except urlfetch_errors.DeadlineExceededError: #raised if the URLFetch times out
            data = memcache.get('longcache'+url)
            if data is not None: return process_json(data, exchange)
            else: return 'Error: timeout'
            
def process_json(txt, exchange):
    #should probably add error handling in case bad json is passed<commit_msg>timeout error message<commit_after>import webapp2
from google.appengine.api import urlfetch
import json
from PIL import Image, ImageDraw, ImageFont
from google.appengine.api import memcache
import StringIO
import jinja2
import os
from decimal import * #used fixed point math for better accuracy
from google.appengine import runtime # for catching DeadlineExceededError
from google.appengine.api import urlfetch_errors # "

JINJA_ENVIRONMENT = jinja2.Environment(
    loader=jinja2.FileSystemLoader(os.path.dirname(__file__)))

def urlfetch_cache(url,exchange):
    # fetches a url, but using memcache to not hammer the exchanges server
    data = memcache.get(url)
    if data is not None:
        return process_json(data, exchange)
    else:
        try:
            result = urlfetch.fetch(url,deadline=30) #timeout after 10 sec
            if result.status_code == 200:
                value = process_json(result.content, exchange)
                memcache.add(url, result.content, 30) #cache for 30 sec
                memcache.add('longcache'+url, result.content, 3000) #also cache for 5min in case of timeouts
                return value
            else:
                return 'Error: '+exchange+' status code '+str(result.status_code) #'Error accessing Vircurex API'
        except runtime.DeadlineExceededError: #raised if the overall request times out
            data = memcache.get('longcache'+url)
            if data is not None: return process_json(data, exchange)
            else: return 'Error: '+exchange+' timeout'
        except runtime.apiproxy_errors.DeadlineExceededError: #raised if an RPC exceeded its deadline (set)
            data = memcache.get('longcache'+url)
            if data is not None: return process_json(data, exchange)
            else: return 'Error: '+exchange+' timeout'
        except urlfetch_errors.DeadlineExceededError: #raised if the URLFetch times out
            data = memcache.get('longcache'+url)
            if data is not None: return process_json(data, exchange)
            else: return 'Error: '+exchange+' timeout'
            
def process_json(txt, exchange):
    #should probably add error handling in case bad json is passed
{'fraction_non_alphanumeric': 0.07421064761465776, 'fraction_numerical': 0.00921871398939848, 'mean_word_length': 2.981651376146789, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1499950', 'n_tokens_mistral': 1222, 'n_tokens_neox': 1142, 'n_words': 404}