<commit_before>import os
import datetime
import logging
import logging.config
import yaml
import redis
import uuid
import traceback
import threading

import consul
import pika
from flask import Flask
from flask import jsonify

from biomaj_download.download.ftp import FTPDownload
from biomaj_download.download.http import HTTPDownload
from biomaj_download.download.direct import DirectFTPDownload
from biomaj_download.download.direct import DirectHttpDownload
from biomaj_download.download.localcopy import LocalDownload
from biomaj_download.message import downmessage_pb2
from biomaj_download.download.rsync import RSYNCDownload
from biomaj_core.utils import Utils
from biomaj_zipkin.zipkin import Zipkin
from biomaj_download.download.protocolirods import IRODSDownload

app = Flask(__name__)
app_log = logging.getLogger('werkzeug')
app_log.setLevel(logging.ERROR)

# Classify protocols from downmessage.proto
ALL_PROTOCOLS = [key for key, item in downmessage_pb2.DownloadFile.Protocol.items()]
DIRECT_PROTOCOLS = [
    key for key, item in downmessage_pb2.DownloadFile.Protocol.items()
    if key.startswith("DIRECT")
]


@app.route('/api/download-message')
def ping():
    return jsonify({'msg': 'pong'})


def start_web(config):
    app.run(host='0.0.0.0', port=config['web']['port'])


def consul_declare(config):
    if config['consul']['host']:
        consul_agent = consul.Consul(host=config['consul']['host'])
        consul_agent.agent.service.register(
            'biomaj-download-message',
            service_id=config['consul']['id'],<commit_msg>ALL_PROTOCOLS and DIRECT_PROTOCOLS must contain numbers not names.<commit_after>import os
import datetime
import logging
import logging.config
import yaml
import redis
import uuid
import traceback
import threading

import consul
import pika
from flask import Flask
from flask import jsonify

from biomaj_download.download.ftp import FTPDownload
from biomaj_download.download.http import HTTPDownload
from biomaj_download.download.direct import DirectFTPDownload
from biomaj_download.download.direct import DirectHttpDownload
from biomaj_download.download.localcopy import LocalDownload
from biomaj_download.message import downmessage_pb2
from biomaj_download.download.rsync import RSYNCDownload
from biomaj_core.utils import Utils
from biomaj_zipkin.zipkin import Zipkin
from biomaj_download.download.protocolirods import IRODSDownload

app = Flask(__name__)
app_log = logging.getLogger('werkzeug')
app_log.setLevel(logging.ERROR)

# Classify protocols from downmessage.proto
# Note: those lists are based on the protocol numbers, not the protocol names
ALL_PROTOCOLS = [item for key, item in downmessage_pb2.DownloadFile.Protocol.items()]
DIRECT_PROTOCOLS = [
    item for key, item in downmessage_pb2.DownloadFile.Protocol.items()
    if key.startswith("DIRECT")
]


@app.route('/api/download-message')
def ping():
    return jsonify({'msg': 'pong'})


def start_web(config):
    app.run(host='0.0.0.0', port=config['web']['port'])


def consul_declare(config):
    if config['consul']['host']:
        consul_agent = consul.Consul(host=config['consul']['host'])
        consul_agent.agent.service.register(
            'biomaj-download-message',
            service_id=config['consul']['id'],
{'fraction_non_alphanumeric': 0.09088117899907891, 'fraction_numerical': 0.004298434141848327, 'mean_word_length': 5.635437881873727, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 52, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1988924', 'n_tokens_mistral': 1004, 'n_tokens_neox': 980, 'n_words': 247}