<commit_before>import os
import sys
import asyncio
import json

from aiohttp import web
from jinja2 import Environment, FileSystemLoader
import pylibmc

from check import check, Crawlability


BASE_DIR = os.path.abspath(os.path.dirname(__file__))
TEMPLATE_DIR = os.path.join(BASE_DIR, 'templates')
STATIC_DIR = os.path.join(BASE_DIR, 'static')
env = Environment(loader=FileSystemLoader(TEMPLATE_DIR))

MEMCACHIER_SERVERS = os.environ.get('MEMCACHIER_SERVERS', '').split(',')
MEMCACHIER_USERNAME = os.environ.get('MEMCACHIER_USERNAME', '')
MEMCACHIER_PASSWORD = os.environ.get('MEMCACHIER_PASSWORD', '')

CACHE_DISABLED = os.environ.get('CACHE_DISABLED') == '1'
CACHE_TIMEOUT_SECONDS = 5 * 60

mc = pylibmc.Client(MEMCACHIER_SERVERS, binary=True,
                    username=MEMCACHIER_USERNAME, password=<PASSWORD>,
                    behaviors={"tcp_nodelay": True,
                               "ketama": True,
                               "no_block": True,})


def _get_from_cache(bytes_url):
    if CACHE_DISABLED:
        return None

    value = mc.get(bytes_url)
    if value is None:
        return None

    d = json.loads(value)
    c = Crawlability(**d)
    return c


def _store_into_cache(bytes_url, c):
    if CACHE_DISABLED:
        return

    mc.set(bytes_url, json.dumps(c.to_cacheable_dict()), time=CACHE_TIMEOUT_SECONDS)<commit_msg>Cache all of Crawlability object<commit_after>import os
import sys
import asyncio

from aiohttp import web
from jinja2 import Environment, FileSystemLoader
import pylibmc

from check import check, Crawlability


BASE_DIR = os.path.abspath(os.path.dirname(__file__))
TEMPLATE_DIR = os.path.join(BASE_DIR, 'templates')
STATIC_DIR = os.path.join(BASE_DIR, 'static')
env = Environment(loader=FileSystemLoader(TEMPLATE_DIR))

MEMCACHIER_SERVERS = os.environ.get('MEMCACHIER_SERVERS', '').split(',')
MEMCACHIER_USERNAME = os.environ.get('MEMCACHIER_USERNAME', '')
MEMCACHIER_PASSWORD = os.environ.get('MEMCACHIER_PASSWORD', '')

CACHE_DISABLED = os.environ.get('CACHE_DISABLED') == '1'
CACHE_TIMEOUT_SECONDS = 5 * 60

mc = pylibmc.Client(MEMCACHIER_SERVERS, binary=True,
                    username=MEMCACHIER_USERNAME, password=<PASSWORD>,
                    behaviors={"tcp_nodelay": True,
                               "ketama": True,
                               "no_block": True,})


def _get_from_cache(bytes_url):
    if CACHE_DISABLED:
        return None

    return mc.get(bytes_url)


def _store_into_cache(bytes_url, c):
    if CACHE_DISABLED:
        return

    mc.set(bytes_url, c, time=CACHE_TIMEOUT_SECONDS)
{'fraction_non_alphanumeric': 0.10813953488372093, 'fraction_numerical': 0.003875968992248062, 'mean_word_length': 3.6420863309352516, 'pattern_counts': {'":': 6, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '147427', 'n_tokens_mistral': 912, 'n_tokens_neox': 887, 'n_words': 163}