<commit_before># TmServer - TissueMAPS server application.
# Copyright (C) 2016  <NAME>, University of Zurich and <NAME>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
import sys
import os
from os.path import join, dirname, abspath
import logging
from flask import Flask
from flask_sqlalchemy_session import flask_scoped_session
import gc3libs

import tmlib.models as tm
from tmlib.log import map_logging_verbosity
from tmlib.models.utils import create_db_engine, create_db_session_factory

from tmserver.extensions import jwt
from tmserver.extensions import redis_store
from tmserver.serialize import TmJSONEncoder
from tmserver import cfg
from tmlib import cfg as libcfg


logger = logging.getLogger(__name__)


def get_interrupted_tasks():
    """Gets the IDs of all tasks that are not in state ``STOPPED`` or
    ``TERMINATED``. If tasks are have one of these states at server startup,
    they have probably been interrupted by a previous shutdown and need to
    be resubmitted.

    Returns
    -------
    List[int]
    """
    with tm.utils.MainSession() as session:
        top_task_ids = session.query(tm.Submission.top_task_id).all()
        tasks = session.query(tm.Task.id).\
            filter(
                tm.Task.id.in_(top_task_ids),
                ~tm.Task.state.in_({'STOPPED', 'TERMINATED', 'TERMINATING'})
            ).\
            all()
        return [t.id for t in tasks]


def create_app(config_overrides={}, verbosity=None):
    """Creates a Flask application object that registers all the blueprints on
    which the actual routes are defined.

    Parameters
    ----------
    config_overrides : dict
        Config options to programatically override the user config.
    log_level : str
        Log level to override the one specified by the config.

    Returns
    -------
    flask.Flask
        Flask application

    """
    app = Flask('wsgi')

    if verbosity is not None:
        log_level = map_logging_verbosity(verbosity)
        cfg.log_level = log_level

    app.config.update(config_overrides)
    app.config['JWT_EXPIRATION_DELTA'] = cfg.jwt_expiration_delta

    ## Configure logging
    app.logger.setLevel(cfg.log_level)

    # Remove standard handlers
    app.logger.handlers = []

    formatter = logging.Formatter(
        fmt='%(asctime)s | %(levelname)-8s | %(name)-40s | %(message)s',
        datefmt='%Y-%m-%d %H:%M:%S'
    )

    # If production mode is activated, set a file logger
    file_handler = logging.handlers.RotatingFileHandler(
        cfg.log_file,
        maxBytes=cfg.log_max_bytes,
        backupCount=cfg.log_n_backups
    )
    file_handler.setFormatter(formatter)
    stdout_handler = logging.StreamHandler(stream=sys.stdout)
    stdout_handler.setFormatter(formatter)
    app.logger.addHandler(file_handler)
    app.logger.addHandler(stdout_handler)
    flask_jwt_logger = logging.getLogger('flask_jwt')
    flask_jwt_logger.setLevel(cfg.log_level)
    flask_jwt_logger.addHandler(file_handler)
    flask_jwt_logger.addHandler(stdout_handler)
    tmserver_logger = logging.getLogger('tmserver')
    tmserver_logger.setLevel(cfg.log_level)
    tmserver_logger.addHandler(file_handler)
    tmserver_logger.addHandler(stdout_handler)
    tmlib_logger = logging.getLogger('tmlib')
    tmlib_logger.setLevel(cfg.log_level)
    tmlib_logger.addHandler(file_handler)
    tmlib_logger.addHandler(stdout_handler)
    apscheduler_logger = logging.getLogger('apscheduler')
    apscheduler_logger.addHandler(file_handler)
    apscheduler_logger.addHandler(stdout_handler)
    wsgi_logger = logging.getLogger('wsgi')
    wsgi_logger.addHandler(file_handler)
    wsgi_logger.addHandler(stdout_handler)
    gc3pie_logger = logging.getLogger('gc3.gc3libs')
    gc3pie_logger.addHandler(file_handler)
    gc3pie_logger.addHandler(stdout_handler)
    werkzeug_logger = logging.getLogger('werkzeug')
    werkzeug_logger.addHandler(file_handler)
    werkzeug_logger.addHandler(stdout_handler)
    if verbosity > 4:
        gc3pie_logger.setLevel(logging.DEBUG)
        wsgi_logger.setLevel(logging.DEBUG)
        werkzeug_logger.setLevel(logging.DEBUG)
        apscheduler_logger.setLevel(logging.DEBUG)
    elif verbosity > 3:
        gc3pie_logger.setLevel(logging.INFO)
        wsgi_logger.setLevel(logging.INFO)
        werkzeug_logger.setLevel(logging.INFO)
        apscheduler_logger.setLevel(logging.INFO)
    else:
        gc3pie_logger.setLevel(logging.CRITICAL)
        wsgi_logger.setLevel(logging.CRITICAL)
        werkzeug_logger.setLevel(logging.CRITICAL)
        apscheduler_logger.setLevel(logging.CRITICAL)

    ## Set the JSON encoder
    app.json_encoder = TmJSONEncoder<commit_msg>Optimize logging levels for app<commit_after># TmServer - TissueMAPS server application.
# Copyright (C) 2016  <NAME>, University of Zurich and <NAME>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
import sys
import os
from os.path import join, dirname, abspath
import logging
from flask import Flask
from flask_sqlalchemy_session import flask_scoped_session
import gc3libs

import tmlib.models as tm
from tmlib.log import map_logging_verbosity
from tmlib.models.utils import create_db_engine, create_db_session_factory

from tmserver.extensions import jwt
from tmserver.extensions import redis_store
from tmserver.serialize import TmJSONEncoder
from tmserver import cfg
from tmlib import cfg as libcfg


logger = logging.getLogger(__name__)


def get_interrupted_tasks():
    """Gets the IDs of all tasks that are not in state ``STOPPED`` or
    ``TERMINATED``. If tasks are have one of these states at server startup,
    they have probably been interrupted by a previous shutdown and need to
    be resubmitted.

    Returns
    -------
    List[int]
    """
    with tm.utils.MainSession() as session:
        top_task_ids = session.query(tm.Submission.top_task_id).all()
        tasks = session.query(tm.Task.id).\
            filter(
                tm.Task.id.in_(top_task_ids),
                ~tm.Task.state.in_({'STOPPED', 'TERMINATED', 'TERMINATING'})
            ).\
            all()
        return [t.id for t in tasks]


def create_app(config_overrides={}, verbosity=None):
    """Creates a Flask application object that registers all the blueprints on
    which the actual routes are defined.

    Parameters
    ----------
    config_overrides : dict
        Config options to programatically override the user config.
    log_level : str
        Log level to override the one specified by the config.

    Returns
    -------
    flask.Flask
        Flask application

    """
    app = Flask('wsgi')

    if verbosity is not None:
        log_level = map_logging_verbosity(verbosity)
        cfg.log_level = log_level

    app.config.update(config_overrides)
    app.config['JWT_EXPIRATION_DELTA'] = cfg.jwt_expiration_delta

    ## Configure logging
    app.logger.setLevel(cfg.log_level)

    # Remove standard handlers
    app.logger.handlers = []

    formatter = logging.Formatter(
        fmt='%(asctime)s | %(levelname)-8s | %(name)-40s | %(message)s',
        datefmt='%Y-%m-%d %H:%M:%S'
    )

    # If production mode is activated, set a file logger
    file_handler = logging.handlers.RotatingFileHandler(
        cfg.log_file,
        maxBytes=cfg.log_max_bytes,
        backupCount=cfg.log_n_backups
    )
    file_handler.setFormatter(formatter)
    stdout_handler = logging.StreamHandler(stream=sys.stdout)
    stdout_handler.setFormatter(formatter)
    app.logger.addHandler(file_handler)
    app.logger.addHandler(stdout_handler)
    flask_jwt_logger = logging.getLogger('flask_jwt')
    flask_jwt_logger.setLevel(cfg.log_level)
    flask_jwt_logger.addHandler(file_handler)
    flask_jwt_logger.addHandler(stdout_handler)
    tmserver_logger = logging.getLogger('tmserver')
    tmserver_logger.setLevel(cfg.log_level)
    tmserver_logger.addHandler(file_handler)
    tmserver_logger.addHandler(stdout_handler)
    tmlib_logger = logging.getLogger('tmlib')
    tmlib_logger.setLevel(cfg.log_level)
    tmlib_logger.addHandler(file_handler)
    tmlib_logger.addHandler(stdout_handler)
    apscheduler_logger = logging.getLogger('apscheduler')
    apscheduler_logger.addHandler(file_handler)
    apscheduler_logger.addHandler(stdout_handler)
    wsgi_logger = logging.getLogger('wsgi')
    wsgi_logger.addHandler(file_handler)
    wsgi_logger.addHandler(stdout_handler)
    gc3pie_logger = logging.getLogger('gc3.gc3libs')
    gc3pie_logger.addHandler(file_handler)
    gc3pie_logger.addHandler(stdout_handler)
    werkzeug_logger = logging.getLogger('werkzeug')
    werkzeug_logger.addHandler(file_handler)
    werkzeug_logger.addHandler(stdout_handler)
    if verbosity > 5:
        gc3pie_logger.setLevel(logging.DEBUG)
        wsgi_logger.setLevel(logging.DEBUG)
        werkzeug_logger.setLevel(logging.DEBUG)
        apscheduler_logger.setLevel(logging.DEBUG)
    elif verbosity > 4:
        gc3pie_logger.setLevel(logging.INFO)
        wsgi_logger.setLevel(logging.DEBUG)
        werkzeug_logger.setLevel(logging.DEBUG)
        apscheduler_logger.setLevel(logging.DEBUG)
    elif verbosity > 3:
        gc3pie_logger.setLevel(logging.ERROR)
        wsgi_logger.setLevel(logging.INFO)
        werkzeug_logger.setLevel(logging.INFO)
        apscheduler_logger.setLevel(logging.INFO)
    else:
        gc3pie_logger.setLevel(logging.CRITICAL)
        wsgi_logger.setLevel(logging.ERROR)
        werkzeug_logger.setLevel(logging.ERROR)
        apscheduler_logger.setLevel(logging.ERROR)

    ## Set the JSON encoder
    app.json_encoder = TmJSONEncoder
{'fraction_non_alphanumeric': 0.08426132145508537, 'fraction_numerical': 0.003711952487008166, 'mean_word_length': 3.5684612123781263, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2095105', 'n_tokens_mistral': 3369, 'n_tokens_neox': 3277, 'n_words': 790}