<commit_before>import os
import sys
import subprocess
import json
from pkg_resources import require, DistributionNotFound, VersionConflict

from setproctitle import setproctitle; setproctitle('inbox')
import sqlalchemy
from alembic.config import Config as alembic_config
from alembic.script import ScriptDirectory

from inbox.config import config

from inbox.log import get_logger
log = get_logger()


def check_requirements(requirements_path):
    # Check our requirements package
    failed_deps = []
    with open(requirements_path, 'r') as f:
        for x in f:
            x = x.strip()
            if not x or x.startswith('#') or x.startswith('-e '):
                continue
            try:
                require(x)
            except (DistributionNotFound, VersionConflict):
                failed_deps.append(x)

    if failed_deps:
        raise ImportError(
            '\nPython module dependency verification failed! \n\n'
            'The following dependencies are either missing or out of '
            'date: \n\t{}\n\nYou probably need to run --> sudo pip '
            'install -r requirements.txt\n'
            .format('\n\t'.join(failed_deps)))


def check_db():
    """ Checks the database revision against the known alembic migrations. """
    from inbox.ignition import db_uri
    inbox_db_engine = sqlalchemy.create_engine(db_uri())

    # top-level, with setup.sh
    alembic_ini_filename = config.get('ALEMBIC_INI', None)
    assert alembic_ini_filename, 'Must set ALEMBIC_INI config var'
    assert os.path.isfile(alembic_ini_filename), \
        'Must have alembic.ini file at {}'.format(alembic_ini_filename)
    alembic_cfg = alembic_config(alembic_ini_filename)

    try:
        inbox_db_engine.dialect.has_table(inbox_db_engine, 'alembic_version')
    except sqlalchemy.exc.OperationalError:
        sys.exit("Databases don't exist! Run bin/create-db")

    if inbox_db_engine.dialect.has_table(inbox_db_engine, 'alembic_version'):
        res = inbox_db_engine.execute('SELECT version_num from alembic_version')
        current_revision = [r for r in res][0][0]
        assert current_revision, \
            'Need current revision in alembic_version table...'

        script = ScriptDirectory.from_config(alembic_cfg)
        head_revision = script.get_current_head()
        log.info('Head database revision: {0}'.format(head_revision))
        log.info('Current database revision: {0}'.format(current_revision))

        if current_revision != head_revision:
            raise Exception(
                'Outdated database! Migrate using `alembic upgrade head`')
        else:
            log.info('[OK] Database scheme matches latest')
    else:
        raise Exception(
            'Un-stamped database! `bin/create-db` should have done this... bailing.')


def check_sudo():
    if os.getuid() == 0:
        raise Exception("Don't run Inbox as root!")


def clean_pyc():
    # Keep it clean for development
    log.debug('Removing pyc files...')
    for root, dir, files in os.walk('./src'):
        for filename in files:
            if filename[-4:] == '.pyc':
                full_path = os.path.join(root, filename)
                log.info('removing {0}'.format(full_path))
                os.remove(full_path)
    log.debug('Not writing pyc bytecode for this exectution')
    sys.dont_write_bytecode = True<commit_msg>Fix typo<commit_after>import os
import sys
import subprocess
import json
from pkg_resources import require, DistributionNotFound, VersionConflict

from setproctitle import setproctitle; setproctitle('inbox')
import sqlalchemy
from alembic.config import Config as alembic_config
from alembic.script import ScriptDirectory

from inbox.config import config

from inbox.log import get_logger
log = get_logger()


def check_requirements(requirements_path):
    # Check our requirements package
    failed_deps = []
    with open(requirements_path, 'r') as f:
        for x in f:
            x = x.strip()
            if not x or x.startswith('#') or x.startswith('-e '):
                continue
            try:
                require(x)
            except (DistributionNotFound, VersionConflict):
                failed_deps.append(x)

    if failed_deps:
        raise ImportError(
            '\nPython module dependency verification failed! \n\n'
            'The following dependencies are either missing or out of '
            'date: \n\t{}\n\nYou probably need to run --> sudo pip '
            'install -r requirements.txt\n'
            .format('\n\t'.join(failed_deps)))


def check_db():
    """ Checks the database revision against the known alembic migrations. """
    from inbox.ignition import db_uri
    inbox_db_engine = sqlalchemy.create_engine(db_uri())

    # top-level, with setup.sh
    alembic_ini_filename = config.get('ALEMBIC_INI', None)
    assert alembic_ini_filename, 'Must set ALEMBIC_INI config var'
    assert os.path.isfile(alembic_ini_filename), \
        'Must have alembic.ini file at {}'.format(alembic_ini_filename)
    alembic_cfg = alembic_config(alembic_ini_filename)

    try:
        inbox_db_engine.dialect.has_table(inbox_db_engine, 'alembic_version')
    except sqlalchemy.exc.OperationalError:
        sys.exit("Databases don't exist! Run bin/create-db")

    if inbox_db_engine.dialect.has_table(inbox_db_engine, 'alembic_version'):
        res = inbox_db_engine.execute('SELECT version_num from alembic_version')
        current_revision = [r for r in res][0][0]
        assert current_revision, \
            'Need current revision in alembic_version table...'

        script = ScriptDirectory.from_config(alembic_cfg)
        head_revision = script.get_current_head()
        log.info('Head database revision: {0}'.format(head_revision))
        log.info('Current database revision: {0}'.format(current_revision))

        if current_revision != head_revision:
            raise Exception(
                'Outdated database! Migrate using `alembic upgrade head`')
        else:
            log.info('[OK] Database scheme matches latest')
    else:
        raise Exception(
            'Un-stamped database! `bin/create-db` should have done this... bailing.')


def check_sudo():
    if os.getuid() == 0:
        raise Exception("Don't run Inbox as root!")


def clean_pyc():
    # Keep it clean for development
    log.debug('Removing pyc files...')
    for root, dir, files in os.walk('./src'):
        for filename in files:
            if filename[-4:] == '.pyc':
                full_path = os.path.join(root, filename)
                log.info('removing {0}'.format(full_path))
                os.remove(full_path)
    log.debug('Not writing pyc bytecode for this execution')
    sys.dont_write_bytecode = True
{'fraction_non_alphanumeric': 0.09765509053131494, 'fraction_numerical': 0.0020777678836449986, 'mean_word_length': 2.7648044692737432, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7620369', 'n_tokens_mistral': 2059, 'n_tokens_neox': 1984, 'n_words': 532}