<commit_before>import click

from . import app, db
from iatidq import dqcodelists, dqdownload, dqfunctions, dqimporttests, dqindicators, dqminimal, dqorganisations, dqprocessing, dqregistry, dqruntests, dqusers
from iatidq import setup as dqsetup


@app.cli.command()
def init_db():
    """Initialize the database."""
    db.create_all()
    dqimporttests.hardcodedTests()

@app.cli.command()
def drop_db():
    """Drop the database."""
    click.echo('\nWarning! This will drop all database tables!')
    click.confirm('Are you really really sure?', abort=True)
    db.drop_all()
    click.echo('DB dropped.')

@app.cli.command()
@click.option('--minimal', is_flag=True, help='Operate on a minimal set of packages')
def setup(minimal):
    """
    Quick setup. Will init db, add tests, add codelists,
    add indicators, refresh package data from Registry
    """
    dqsetup.setup(minimal)

@app.cli.command()
@click.option('--username', prompt='Username')
@click.password_option()
def create_admin(username, password):
    """Create an admin user."""
    dqsetup.setup_admin_user(username, password)

@app.cli.command()
@click.option('--filename', required=True, help='Set filename of data to test')
@click.option('--level', type=int, default=1, help='Test level (e.g. 1 == Activity)')
def enroll_tests(filename, level):
    """Enroll a CSV file of tests."""
    result = dqimporttests.importTestsFromFile(
        filename=filename.decode(),
        level=level
    )
    if not result:
        print('Error importing')

@app.cli.command()
def clear_revisionid():
    """Clear CKAN revision ids"""
    dqfunctions.clear_revisions()

@app.cli.command()
def import_codelists():
    """Import codelists"""
    dqcodelists.importCodelists()

@app.cli.command()
def import_basic_countries():
    """Import basic list of countries"""
    filename = 'tests/countries_basic.csv'
    codelist_name = 'countriesbasic'
    codelist_description = 'Basic list of countries for running tests against'
    dqcodelists.add_manual_codelist(filename, codelist_name, codelist_description)

@app.cli.command()
@click.option('--minimal', is_flag=True, help='Operate on a minimal set of packages')
@click.option('--matching', help='Regular expression for matching packages')
def download_packages(minimal, matching):
    """Download packages"""
    if minimal:
        for package_name, _ in dqminimal.which_packages:
            dqdownload.run(package_name=package_name)
    elif matching:
        for pkg_name in dqregistry.matching_packages(matching):
            dqdownload.run(package_name=pkg_name)
    else:
        dqdownload.run()

@app.cli.command()
def update_frequency():
    """Update frequency"""
    dqorganisations.downloadOrganisationFrequency()

@app.cli.command()
@click.option('--filename', help='Set filename of data to test')
def import_indicators(filename):
    """Import indicators. Will try to assign indicators to existing tests."""
    if filename:
        dqindicators.importIndicatorsFromFile('pwyf2013', filename)
    else:
        dqindicators.importIndicators()

@app.cli.command()
@click.option('--filename', help='Set filename of data to test')
def import_organisations(filename):
    """
    Import organisations. Will try to create and assign
    organisations to existing packages.
    """
    if filename:
        dqorganisations.importOrganisationPackagesFromFile(filename)
    else:
        print('Error: please provide a filename')

@app.cli.command()
@click.option('--package-name', required=True, help='Set name of package to be tested')
@click.option('--filename', required=True, help='Set filename of data to test')
def enqueue_test(package_name, filename):
    """Set a package to be tested"""
    dqruntests.enqueue_package_for_test(filename, package_name)

@app.cli.command()<commit_msg>Remove mention of old indicator group<commit_after>import click

from . import app, db
from iatidq import dqcodelists, dqdownload, dqfunctions, dqimporttests, dqindicators, dqminimal, dqorganisations, dqprocessing, dqregistry, dqruntests, dqusers
from iatidq import setup as dqsetup


@app.cli.command()
def init_db():
    """Initialize the database."""
    db.create_all()
    dqimporttests.hardcodedTests()

@app.cli.command()
def drop_db():
    """Drop the database."""
    click.echo('\nWarning! This will drop all database tables!')
    click.confirm('Are you really really sure?', abort=True)
    db.drop_all()
    click.echo('DB dropped.')

@app.cli.command()
@click.option('--minimal', is_flag=True, help='Operate on a minimal set of packages')
def setup(minimal):
    """
    Quick setup. Will init db, add tests, add codelists,
    add indicators, refresh package data from Registry
    """
    dqsetup.setup(minimal)

@app.cli.command()
@click.option('--username', prompt='Username')
@click.password_option()
def create_admin(username, password):
    """Create an admin user."""
    dqsetup.setup_admin_user(username, password)

@app.cli.command()
@click.option('--filename', required=True, help='Set filename of data to test')
@click.option('--level', type=int, default=1, help='Test level (e.g. 1 == Activity)')
def enroll_tests(filename, level):
    """Enroll a CSV file of tests."""
    result = dqimporttests.importTestsFromFile(
        filename=filename.decode(),
        level=level
    )
    if not result:
        print('Error importing')

@app.cli.command()
def clear_revisionid():
    """Clear CKAN revision ids"""
    dqfunctions.clear_revisions()

@app.cli.command()
def import_codelists():
    """Import codelists"""
    dqcodelists.importCodelists()

@app.cli.command()
def import_basic_countries():
    """Import basic list of countries"""
    filename = 'tests/countries_basic.csv'
    codelist_name = 'countriesbasic'
    codelist_description = 'Basic list of countries for running tests against'
    dqcodelists.add_manual_codelist(filename, codelist_name, codelist_description)

@app.cli.command()
@click.option('--minimal', is_flag=True, help='Operate on a minimal set of packages')
@click.option('--matching', help='Regular expression for matching packages')
def download_packages(minimal, matching):
    """Download packages"""
    if minimal:
        for package_name, _ in dqminimal.which_packages:
            dqdownload.run(package_name=package_name)
    elif matching:
        for pkg_name in dqregistry.matching_packages(matching):
            dqdownload.run(package_name=pkg_name)
    else:
        dqdownload.run()

@app.cli.command()
def update_frequency():
    """Update frequency"""
    dqorganisations.downloadOrganisationFrequency()

@app.cli.command()
@click.option('--filename', help='Set filename of data to test')
def import_indicators(filename):
    """Import indicators. Will try to assign indicators to existing tests."""
    indicator_group_name = app.config['INDICATOR_GROUP']
    if filename:
        dqindicators.importIndicatorsFromFile(indicator_group_name, filename)
    else:
        dqindicators.importIndicators()

@app.cli.command()
@click.option('--filename', help='Set filename of data to test')
def import_organisations(filename):
    """
    Import organisations. Will try to create and assign
    organisations to existing packages.
    """
    if filename:
        dqorganisations.importOrganisationPackagesFromFile(filename)
    else:
        print('Error: please provide a filename')

@app.cli.command()
@click.option('--package-name', required=True, help='Set name of package to be tested')
@click.option('--filename', required=True, help='Set filename of data to test')
def enqueue_test(package_name, filename):
    """Set a package to be tested"""
    dqruntests.enqueue_package_for_test(filename, package_name)

@app.cli.command()
{'fraction_non_alphanumeric': 0.1268608414239482, 'fraction_numerical': 0.001035598705501618, 'mean_word_length': 4.380222841225627, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '883325', 'n_tokens_mistral': 2383, 'n_tokens_neox': 2286, 'n_words': 608}