<commit_before>from flask import json, request, redirect, abort, make_response
from flask import render_template
import os, datetime, redis, hashlib
import unicodedata, re
from werkzeug.security import generate_password_hash, check_password_hash

from totalimpact import dao, app, tiredis
from totalimpact.models import ItemFactory, CollectionFactory, MemberItems, UserFactory
from totalimpact.providers.provider import ProviderFactory
from totalimpact import default_settings
import logging

logger = logging.getLogger("ti.views")
logger.setLevel(logging.DEBUG)
redis = redis.from_url(os.getenv("REDISTOGO_URL"))

mydao = dao.Dao(os.environ["CLOUDANT_URL"], os.getenv("CLOUDANT_DB"))
myredis = tiredis.from_url(os.getenv("REDISTOGO_URL"))

# setup to remove control characters from received IDs
# from http://stackoverflow.com/questions/92438/stripping-non-printable-characters-from-a-string-in-python
control_chars = ''.join(map(unichr, range(0,32) + range(127,160)))
control_char_re = re.compile('[%s]' % re.escape(control_chars))
def clean_id(nid):
    nid = control_char_re.sub('', nid)
    nid = nid.replace(u'\u200b', "")
    nid = nid.strip()
    return(nid)
<commit_msg>fix merge problem<commit_after>from flask import json, request, redirect, abort, make_response
from flask import render_template
import os, datetime, redis, hashlib
import unicodedata, re
from werkzeug.security import generate_password_hash, check_password_hash

from totalimpact import dao, app, tiredis
from totalimpact.models import ItemFactory, CollectionFactory, MemberItems, UserFactory
from totalimpact.providers.provider import ProviderFactory, ProviderItemNotFoundError, ProviderError
from totalimpact import default_settings
import logging

logger = logging.getLogger("ti.views")
logger.setLevel(logging.DEBUG)
redis = redis.from_url(os.getenv("REDISTOGO_URL"))

mydao = dao.Dao(os.environ["CLOUDANT_URL"], os.getenv("CLOUDANT_DB"))
myredis = tiredis.from_url(os.getenv("REDISTOGO_URL"))

# setup to remove control characters from received IDs
# from http://stackoverflow.com/questions/92438/stripping-non-printable-characters-from-a-string-in-python
control_chars = ''.join(map(unichr, range(0,32) + range(127,160)))
control_char_re = re.compile('[%s]' % re.escape(control_chars))
def clean_id(nid):
    nid = control_char_re.sub('', nid)
    nid = nid.replace(u'\u200b', "")
    nid = nid.strip()
    return(nid)

{'fraction_non_alphanumeric': 0.11480865224625623, 'fraction_numerical': 0.014143094841930116, 'mean_word_length': 6.634920634920635, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7421186', 'n_tokens_mistral': 815, 'n_tokens_neox': 782, 'n_words': 174}