<commit_before>from flask.ext.script import Manager

from nomenklatura.core import app, db
from nomenklatura.model import *
from nomenklatura import views

manager = Manager(app)

@manager.command
def createdb():
    """ Make the database. """
    db.create_all()

@manager.command
def updb():
    """ Update the database. """
    db.engine.execute('ALTER TABLE "link" ADD COLUMN "data" TEXT')
    db.engine.execute("""UPDATE "link" SET "data" = '{}'""")
    db.session.commit()

@manager.command
def flush(dataset):
    ds = Dataset.by_name(dataset)
    for alias in Alias.all_unmatched(ds):
        db.session.delete(alias)
    db.session.commit()

@manager.command
def dedup(dataset):
    ds = Dataset.by_name(dataset)
    from time import time
    from nomenklatura.matching import match
    begin = time()
    for value in Value.all(ds).limit(20):
        matches = match(value.value, ds)
        matches = filter(lambda (c,v,s): v!=value.id, matches)
        print [value.value, '=?', matches[0][1], matches[0][2]]
    print "Time: %.2fms" % ((time() - begin)*1000)

@manager.command
def cleanup():
    """ Clean up the database. """
    for dataset in Dataset.all():
        for alias in Alias.all_unmatched(dataset):
            db.session.delete(alias)
        db.session.flush()
    db.session.commit()

if __name__ == '__main__':
    manager.run()
<commit_msg>Remove old, unneeded functions. <commit_after>from flask.ext.script import Manager

from nomenklatura.core import app, db
from nomenklatura.model import *
from nomenklatura import views

manager = Manager(app)


@manager.command
def createdb():
    """ Make the database. """
    db.create_all()


@manager.command
def flush(dataset):
    ds = Dataset.by_name(dataset)
    for alias in Alias.all_unmatched(ds):
        db.session.delete(alias)
    db.session.commit()


if __name__ == '__main__':
    manager.run()

{'fraction_non_alphanumeric': 0.1378579003181336, 'fraction_numerical': 0.005832449628844115, 'mean_word_length': 3.44, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6777618', 'n_tokens_mistral': 620, 'n_tokens_neox': 609, 'n_words': 156}