<commit_before>import chdb
from scripts.snippet_parser import CITATION_NEEDED_MARKER, REF_MARKER

import flask
import flask_sslify
from flask.ext.compress import Compress

import os
import contextlib
import collections
from datetime import datetime
import itertools

# the markup we're going to use for [citation needed] and <ref> tags,
# pre-marked as safe for jinja.
SUPERSCRIPT_HTML = flask.Markup('<sup class="superscript">[%s]</sup>')
CITATION_NEEDED_HTML = flask.Markup(SUPERSCRIPT_HTML % 'citation needed')

@contextlib.contextmanager
def log_time(operation):
    before = datetime.now()
    yield
    after = datetime.now()
    ms = (after - before).microseconds / 1000.
    print '[citationhunt] %s took %.2f ms' % (operation, ms)

def get_db():
    db = getattr(flask.g, '_db', None)
    if db is None:
        db = flask.g._db = chdb.init_db()
    return db

Category = collections.namedtuple('Category', ['id', 'title'])<commit_msg>Use a thin space in [citation needed].<commit_after>import chdb
from scripts.snippet_parser import CITATION_NEEDED_MARKER, REF_MARKER

import flask
import flask_sslify
from flask.ext.compress import Compress

import os
import contextlib
import collections
from datetime import datetime
import itertools

# the markup we're going to use for [citation needed] and <ref> tags,
# pre-marked as safe for jinja.
SUPERSCRIPT_HTML = flask.Markup('<sup class="superscript">[%s]</sup>')
CITATION_NEEDED_HTML = flask.Markup(SUPERSCRIPT_HTML % 'citation&thinsp;needed')

@contextlib.contextmanager
def log_time(operation):
    before = datetime.now()
    yield
    after = datetime.now()
    ms = (after - before).microseconds / 1000.
    print '[citationhunt] %s took %.2f ms' % (operation, ms)

def get_db():
    db = getattr(flask.g, '_db', None)
    if db is None:
        db = flask.g._db = chdb.init_db()
    return db

Category = collections.namedtuple('Category', ['id', 'title'])
{'fraction_non_alphanumeric': 0.11197916666666667, 'fraction_numerical': 0.005208333333333333, 'mean_word_length': 4.504297994269341, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1800465', 'n_tokens_mistral': 662, 'n_tokens_neox': 617, 'n_words': 184}