<commit_before>from flask import json, request, abort, make_response, g
from flask import render_template
import sys, os
import datetime, re, copy
from werkzeug.security import check_password_hash
from collections import defaultdict
import redis
import analytics
import requests

from totalimpact import app, tiredis, collection, incoming_email
from totalimpact import item as item_module
from totalimpact.models import MemberItems, NotAuthenticatedError
from totalimpact.providers import provider as provider_module
from totalimpact.providers.provider import ProviderFactory, ProviderItemNotFoundError, ProviderError, ProviderServerError, ProviderTimeout
from totalimpact import unicode_helpers
from totalimpact import default_settings
from totalimpact import REDIS_MAIN_DATABASE_NUMBER
import logging


logger = logging.getLogger("ti.views")
logger.setLevel(logging.DEBUG)

mydao = None
myredis = tiredis.from_url(os.getenv("REDIS_URL"), db=REDIS_MAIN_DATABASE_NUMBER)  # main app is on DB 0

# logger.debug(u"Building reference sets")
myrefsets = None
myrefsets_histograms = None
# try:
#     (myrefsets, myrefsets_histograms) = collection.build_all_reference_lookups(myredis, mydao)
#     logger.debug(u"Reference sets dict has %i keys" %len(myrefsets.keys()))
# except (LookupError, AttributeError), e:
#     logger.error(u"Exception %s: Unable to load reference sets" % (e.__repr__()))

def set_db(url, db):
    """useful for unit testing, where you want to use a local database
    """
    global mydao 
    mydao = None
    return mydao
<commit_msg>turn refsets back on<commit_after>from flask import json, request, abort, make_response, g
from flask import render_template
import sys, os
import datetime, re, copy
from werkzeug.security import check_password_hash
from collections import defaultdict
import redis
import analytics
import requests

from totalimpact import app, tiredis, collection, incoming_email
from totalimpact import item as item_module
from totalimpact.models import MemberItems, NotAuthenticatedError
from totalimpact.providers import provider as provider_module
from totalimpact.providers.provider import ProviderFactory, ProviderItemNotFoundError, ProviderError, ProviderServerError, ProviderTimeout
from totalimpact import unicode_helpers
from totalimpact import default_settings
from totalimpact import REDIS_MAIN_DATABASE_NUMBER
import logging


logger = logging.getLogger("ti.views")
logger.setLevel(logging.DEBUG)

mydao = None
myredis = tiredis.from_url(os.getenv("REDIS_URL"), db=REDIS_MAIN_DATABASE_NUMBER)  # main app is on DB 0

# logger.debug(u"Building reference sets")
myrefsets = None
myrefsets_histograms = None
try:
    (myrefsets, myrefsets_histograms) = collection.build_all_reference_lookups(myredis, mydao)
    logger.debug(u"Reference sets dict has %i keys" %len(myrefsets.keys()))
except (LookupError, AttributeError), e:
    logger.error(u"Exception %s: Unable to load reference sets" % (e.__repr__()))

def set_db(url, db):
    """useful for unit testing, where you want to use a local database
    """
    global mydao 
    mydao = None
    return mydao

{'fraction_non_alphanumeric': 0.06975249116039858, 'fraction_numerical': 0.0006428801028608164, 'mean_word_length': 6.154022988505747, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '728941', 'n_tokens_mistral': 920, 'n_tokens_neox': 885, 'n_words': 293}