<commit_before>'''
Web interface to query list/download status
Manage sessions and metrics
'''

import ssl
import os

import yaml
from flask import Flask
from flask import jsonify
from flask import request
from prometheus_client import Counter
from prometheus_client import Gauge
from prometheus_client.exposition import generate_latest
import consul
import redis

from biomaj_process.message import message_pb2
from biomaj_process.process_service import ProcessService
from biomaj_core.utils import Utils

app = Flask(__name__)

process_metric = Counter("biomaj_process_total", "Bank total process execution.", ['bank'])
process_error_metric = Counter("biomaj_process_errors", "Bank total process errors.", ['bank'])
process_time_metric = Gauge("biomaj_process_time", "Bank process execution time in seconds.", ['bank'])

config_file = 'config.yml'
if 'BIOMAJ_CONFIG' in os.environ:
        config_file = os.environ['BIOMAJ_CONFIG']

config = None
with open(config_file, 'r') as ymlfile:
    config = yaml.load(ymlfile)
    Utils.service_config_override(config)


redis_client = redis.StrictRedis(
    host=config['redis']['host'],
    port=config['redis']['port'],
    db=config['redis']['db'],
    decode_responses=True
)


def consul_declare(config):
    if config['consul']['host']:
        consul_agent = consul.Consul(host=config['consul']['host'])
        consul_agent.agent.service.register('biomaj_process', service_id=config['consul']['id'], address=config['web']['hostname'], port=config['web']['port'], tags=['biomaj'])
        check = consul.Check.http(url='http://' + config['web']['hostname'] + ':' + str(config['web']['port']) + '/api/process', interval=20)
        consul_agent.agent.check.register(config['consul']['id'] + '_check', check=check, service_id=config['consul']['id'])


consul_declare(config)


@app.route('/api/process', methods=['GET'])
def ping():
    return jsonify({'msg': 'pong'})


@app.route('/api/process/metrics', methods=['GET'])
def metrics():
    return generate_latest()


@app.route('/api/process/metrics', methods=['POST'])
def add_metrics():
    '''
    Expects a JSON request with an array of {'bank': 'bank_name', 'error': 'error_message', 'execution_time': seconds_to_execute}
    '''
    procs = request.get_json()
    for proc in procs:
        if 'error' in proc and proc['error']:
            process_error_metric.labels(proc['bank']).inc()
        else:
            process_metric.labels(proc['bank']).inc()<commit_msg>modify service name to match consul rules<commit_after>'''
Web interface to query list/download status
Manage sessions and metrics
'''

import ssl
import os

import yaml
from flask import Flask
from flask import jsonify
from flask import request
from prometheus_client import Counter
from prometheus_client import Gauge
from prometheus_client.exposition import generate_latest
import consul
import redis

from biomaj_process.message import message_pb2
from biomaj_process.process_service import ProcessService
from biomaj_core.utils import Utils

app = Flask(__name__)

process_metric = Counter("biomaj_process_total", "Bank total process execution.", ['bank'])
process_error_metric = Counter("biomaj_process_errors", "Bank total process errors.", ['bank'])
process_time_metric = Gauge("biomaj_process_time", "Bank process execution time in seconds.", ['bank'])

config_file = 'config.yml'
if 'BIOMAJ_CONFIG' in os.environ:
        config_file = os.environ['BIOMAJ_CONFIG']

config = None
with open(config_file, 'r') as ymlfile:
    config = yaml.load(ymlfile)
    Utils.service_config_override(config)


redis_client = redis.StrictRedis(
    host=config['redis']['host'],
    port=config['redis']['port'],
    db=config['redis']['db'],
    decode_responses=True
)


def consul_declare(config):
    if config['consul']['host']:
        consul_agent = consul.Consul(host=config['consul']['host'])
        consul_agent.agent.service.register('biomaj-process', service_id=config['consul']['id'], address=config['web']['hostname'], port=config['web']['port'], tags=['biomaj'])
        check = consul.Check.http(url='http://' + config['web']['hostname'] + ':' + str(config['web']['port']) + '/api/process', interval=20)
        consul_agent.agent.check.register(config['consul']['id'] + '_check', check=check, service_id=config['consul']['id'])


consul_declare(config)


@app.route('/api/process', methods=['GET'])
def ping():
    return jsonify({'msg': 'pong'})


@app.route('/api/process/metrics', methods=['GET'])
def metrics():
    return generate_latest()


@app.route('/api/process/metrics', methods=['POST'])
def add_metrics():
    '''
    Expects a JSON request with an array of {'bank': 'bank_name', 'error': 'error_message', 'execution_time': seconds_to_execute}
    '''
    procs = request.get_json()
    for proc in procs:
        if 'error' in proc and proc['error']:
            process_error_metric.labels(proc['bank']).inc()
        else:
            process_metric.labels(proc['bank']).inc()
{'fraction_non_alphanumeric': 0.1536144578313253, 'fraction_numerical': 0.0012048192771084338, 'mean_word_length': 5.273299748110831, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1064024', 'n_tokens_mistral': 1573, 'n_tokens_neox': 1510, 'n_words': 353}