<commit_before># -*- coding: utf-8 -*-
import contextlib
import datetime
import uuid
import operator
import pkgutil
import os
import logging
from functools import reduce
from dateutil.parser import parse as parse_datetime
import json
import colorama
from importlib import import_module
from flask.ext import script


def _db_collect(configuration):
    """Collect all DB metadata from the configuration."""
    # Enumerate through all 'installed' packages.
    bases = {}
    for name in configuration['PACKAGES']:
        try:
            # Attempt to import the model file.
            models = import_module('{}.models'.format(name))

        except ImportError:
            # No installed models; move along.
            continue

        # Append the model bases.
        bases[name] = models.Base

    # Return the bases.
    return bases


def _db_flush_one(metadata, engine):
    """Delete all data from the database."""
    with contextlib.closing(engine.connect()) as connection:
        transaction = connection.begin()
        for table in reversed(metadata.sorted_tables):
            connection.execute(table.delete())
        transaction.commit()


def Database(configuration):
    # Instantiate a second-level management application for the database.
    Database = script.Manager(usage="Perform various database operations.")

    # Collect the db metadata.
    bases = _db_collect(configuration)

    @Database.command
    def init():
        """Initialize the database and create all neccessary tables."""
        for base in bases.values():
            base.metadata.create_all(configuration['DATABASE_ENGINE'])

    @Database.command
    def flush():
        """Initialize the database and create all neccessary tables."""
        for base in bases.values():
            _db_flush_one(base.metadata, configuration['DATABASE_ENGINE'])

    @Database.command
    def clear():
        """Drop all database tables."""
        for base in bases.values():
            base.metadata.drop_all(configuration['DATABASE_ENGINE'])

    # Return the database management command.
    return Database


def _make_context_one(ctx, name, base):
    items = []
    for cls in base._decl_class_registry.values():
        if isinstance(cls, type):
            items.append(cls.__name__)
            ctx[cls.__name__] = cls

    print(
        colorama.Fore.GREEN + colorama.Style.DIM +
        "from {}.models import {}".format(
            name, ', '.join(items)) + colorama.Style.RESET_ALL)


def _make_context(configuration):
    # Initialize colorama.
    colorama.init()

    # Start by adding the session instance to the user's namespace.
    namespace = {'session': configuration['DATABASE_SESSION']()}
    print(colorama.Fore.GREEN + colorama.Style.DIM +
          "from {}.db import Session".format(__package__))
    print("session = Session()")

    # Collect all the models and add those to the namespace.
    for name, base in _db_collect(configuration).items():
        _make_context_one(namespace, name, base)

    # Return the context.
    return namespace


def Shell(configuration):
    # Create the shell with a make_context method that draws from
    # all available models.
    return script.Shell(make_context=lambda: _make_context(configuration))


def Server(configuration):
    # Create a runserver command that uses settings for its host and port.
    host = configuration['SERVER_HOST']
    port = configuration['SERVER_PORT']
    return script.Server(host=host, port=port, debug=configuration['DEBUG'])


class Fixture:

    class Load(script.Command):

        option_list = (
            script.Option(dest='name'),
        )

        def __init__(self, config):
            super().__init__()

            # Store the configuration for later use.
            self.config = config

        def run(self, name):
            # Establish a database session.
            session = self.config['DATABASE_SESSION']()
<commit_msg>Add threaded.<commit_after># -*- coding: utf-8 -*-
import contextlib
import datetime
import uuid
import operator
import pkgutil
import os
import logging
from functools import reduce
from dateutil.parser import parse as parse_datetime
import json
import colorama
from importlib import import_module
from flask.ext import script


def _db_collect(configuration):
    """Collect all DB metadata from the configuration."""
    # Enumerate through all 'installed' packages.
    bases = {}
    for name in configuration['PACKAGES']:
        try:
            # Attempt to import the model file.
            models = import_module('{}.models'.format(name))

        except ImportError:
            # No installed models; move along.
            continue

        # Append the model bases.
        bases[name] = models.Base

    # Return the bases.
    return bases


def _db_flush_one(metadata, engine):
    """Delete all data from the database."""
    with contextlib.closing(engine.connect()) as connection:
        transaction = connection.begin()
        for table in reversed(metadata.sorted_tables):
            connection.execute(table.delete())
        transaction.commit()


def Database(configuration):
    # Instantiate a second-level management application for the database.
    Database = script.Manager(usage="Perform various database operations.")

    # Collect the db metadata.
    bases = _db_collect(configuration)

    @Database.command
    def init():
        """Initialize the database and create all neccessary tables."""
        for base in bases.values():
            base.metadata.create_all(configuration['DATABASE_ENGINE'])

    @Database.command
    def flush():
        """Initialize the database and create all neccessary tables."""
        for base in bases.values():
            _db_flush_one(base.metadata, configuration['DATABASE_ENGINE'])

    @Database.command
    def clear():
        """Drop all database tables."""
        for base in bases.values():
            base.metadata.drop_all(configuration['DATABASE_ENGINE'])

    # Return the database management command.
    return Database


def _make_context_one(ctx, name, base):
    items = []
    for cls in base._decl_class_registry.values():
        if isinstance(cls, type):
            items.append(cls.__name__)
            ctx[cls.__name__] = cls

    print(
        colorama.Fore.GREEN + colorama.Style.DIM +
        "from {}.models import {}".format(
            name, ', '.join(items)) + colorama.Style.RESET_ALL)


def _make_context(configuration):
    # Initialize colorama.
    colorama.init()

    # Start by adding the session instance to the user's namespace.
    namespace = {'session': configuration['DATABASE_SESSION']()}
    print(colorama.Fore.GREEN + colorama.Style.DIM +
          "from {}.db import Session".format(__package__))
    print("session = Session()")

    # Collect all the models and add those to the namespace.
    for name, base in _db_collect(configuration).items():
        _make_context_one(namespace, name, base)

    # Return the context.
    return namespace


def Shell(configuration):
    # Create the shell with a make_context method that draws from
    # all available models.
    return script.Shell(make_context=lambda: _make_context(configuration))


def Server(configuration):
    # Create a runserver command that uses settings for its host and port.
    host = configuration['SERVER_HOST']
    port = configuration['SERVER_PORT']
    return script.Server(host=host, port=port, debug=configuration['DEBUG'],
                         threaded=True)


class Fixture:

    class Load(script.Command):

        option_list = (
            script.Option(dest='name'),
        )

        def __init__(self, config):
            super().__init__()

            # Store the configuration for later use.
            self.config = config

        def run(self, name):
            # Establish a database session.
            session = self.config['DATABASE_SESSION']()

{'fraction_non_alphanumeric': 0.09753335011326454, 'fraction_numerical': 0.0002516989680342311, 'mean_word_length': 2.8225108225108224, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5970553', 'n_tokens_mistral': 2091, 'n_tokens_neox': 2035, 'n_words': 644}