<commit_before>#!/usr/bin/env python3
import os
import json
import connexion
import logging
from flask import redirect, jsonify
from datetime import datetime
from flask_script import Manager
from f8a_jobs.scheduler import Scheduler
import f8a_jobs.defaults as defaults
from f8a_worker.setup_celery import init_selinon

from f8a_jobs.models import create_models
from f8a_jobs.auth import oauth

logger = logging.getLogger(__name__)


class SafeJSONEncoder(json.JSONEncoder):
    """ Convert objects to JSON, safely """
    def default(self, o):
        if isinstance(o, datetime):
            return o.isoformat()
        try:
            return json.JSONEncoder.default(self, o)
        except:
            return repr(o)


def init_logging():
    """ Initialize application logging """
    # Initialize flask logging
    formatter = logging.Formatter('%(asctime)s - %(message)s')
    handler = logging.StreamHandler()
    handler.setLevel(logging.WARNING)
    handler.setFormatter(formatter)
    # Use flask App instead of Connexion's one
    application.logger.addHandler(handler)
    # API logger
    logger.setLevel(logging.DEBUG)
    # lib logger
    formatter = logging.Formatter('%(asctime)s - %(message)s')
    handler = logging.StreamHandler()
    handler.setFormatter(formatter)
    liblog = logging.getLogger('f8a_jobs')
    liblog.setLevel(logging.DEBUG)
    liblog.addHandler(handler)


app = connexion.App(__name__)
application = app.app
init_logging()
app.add_api(defaults.SWAGGER_YAML_PATH)
# Expose for uWSGI
application.json_encoder = SafeJSONEncoder
manager = Manager(application)
# Needed for sesion
application.secret_key = defaults.APP_SECRET_KEY
oauth.init_app(application)


logger.debug("Initializing Selinon")
init_selinon()
logger.debug("Selinon initialized successfully")


@app.route('/')
def base_url():
    # Be nice with user access
    return redirect('api/v1/ui')


@app.route('/api/v1')
def api_v1():
    paths = []

    for rule in application.url_map.iter_rules():
        rule = str(rule)
        if rule.startswith('/api/v1'):
            paths.append(rule)

    return jsonify({'paths': paths})


@manager.command
def initjobs():
    """ initialize default jobs """""
    logger.debug("Initializing default jobs")
    Scheduler.register_default_jobs(defaults.DEFAULT_JOB_DIR)
    logger.debug("Default jobs initialized")
    logger.debug("Initializing DB for tokens")
    create_models()
    logger.debug("DB for tokens initialized")


@manager.command
def runserver():
    """ run job service server """""
    #
    # The Flasks's runserver command was overwritten because we are using connexion.
    #
    # Make sure that you do not run the application with multiple processes since we would
    # have multiple scheduler instances. If you would like to do so, just create one scheduler
    # that would serve jobs and per-process scheduler would be in paused mode just for creating/listing jobs.
    app.run(
        port=os.environ.get('JOB_SERVICE_PORT', defaults.DEFAULT_SERVICE_PORT),
        server='flask',
        debug=True,
        use_reloader=True,
        threaded=True,
        json_encoder=SafeJSONEncoder,
        processes=1
    )

if __name__ == '__main__':
    manager.run()<commit_msg>Fixed Pylint warning in f8a-jobs<commit_after>#!/usr/bin/env python3
import os
import json
import connexion
import logging
from flask import redirect, jsonify
from datetime import datetime
from flask_script import Manager
from f8a_jobs.scheduler import Scheduler
import f8a_jobs.defaults as defaults
from f8a_worker.setup_celery import init_selinon

from f8a_jobs.models import create_models
from f8a_jobs.auth import oauth

logger = logging.getLogger(__name__)


class SafeJSONEncoder(json.JSONEncoder):
    """ Convert objects to JSON, safely """
    def default(self, o):
        if isinstance(o, datetime):
            return o.isoformat()
        try:
            return json.JSONEncoder.default(self, o)
        except:
            return repr(o)


def init_logging():
    """ Initialize application logging """
    # Initialize flask logging
    formatter = logging.Formatter('%(asctime)s - %(message)s')
    handler = logging.StreamHandler()
    handler.setLevel(logging.WARNING)
    handler.setFormatter(formatter)
    # Use flask App instead of Connexion's one
    application.logger.addHandler(handler)
    # API logger
    logger.setLevel(logging.DEBUG)
    # lib logger
    formatter = logging.Formatter('%(asctime)s - %(message)s')
    handler = logging.StreamHandler()
    handler.setFormatter(formatter)
    liblog = logging.getLogger('f8a_jobs')
    liblog.setLevel(logging.DEBUG)
    liblog.addHandler(handler)


app = connexion.App(__name__)
application = app.app
init_logging()
app.add_api(defaults.SWAGGER_YAML_PATH)
# Expose for uWSGI
application.json_encoder = SafeJSONEncoder
manager = Manager(application)
# Needed for sesion
application.secret_key = defaults.APP_SECRET_KEY
oauth.init_app(application)


logger.debug("Initializing Selinon")
init_selinon()
logger.debug("Selinon initialized successfully")


@app.route('/')
def base_url():
    # Be nice with user access
    return redirect('api/v1/ui')


@app.route('/api/v1')
def api_v1():
    paths = []

    for rule in application.url_map.iter_rules():
        rule = str(rule)
        if rule.startswith('/api/v1'):
            paths.append(rule)

    return jsonify({'paths': paths})


@manager.command
def initjobs():
    """ initialize default jobs """""
    logger.debug("Initializing default jobs")
    Scheduler.register_default_jobs(defaults.DEFAULT_JOB_DIR)
    logger.debug("Default jobs initialized")
    logger.debug("Initializing DB for tokens")
    create_models()
    logger.debug("DB for tokens initialized")


@manager.command
def runserver():
    """ run job service server """""
    #
    # The Flasks's runserver command was overwritten because we are using connexion.
    #
    # Make sure that you do not run the application with multiple processes since we would
    # have multiple scheduler instances. If you would like to do so, just create one scheduler
    # that would serve jobs and per-process scheduler would be in paused mode
    # just for creating/listing jobs.
    app.run(
        port=os.environ.get('JOB_SERVICE_PORT', defaults.DEFAULT_SERVICE_PORT),
        server='flask',
        debug=True,
        use_reloader=True,
        threaded=True,
        json_encoder=SafeJSONEncoder,
        processes=1
    )


if __name__ == '__main__':
    manager.run()
{'fraction_non_alphanumeric': 0.09874271695798835, 'fraction_numerical': 0.003833180006133088, 'mean_word_length': 3.6592857142857143, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2424469', 'n_tokens_mistral': 1962, 'n_tokens_neox': 1899, 'n_words': 513}