<commit_before>import re
import os
import celery
from bottle import route, request, default_app, abort, run
from brain import braintasks, brainstorage
from config.dbconfig import SCAN_STATUS_INIT, SCAN_STATUS_LAUNCHED, SCAN_STATUS_FINISHED
from bson import ObjectId

bstorage = brainstorage.BrainStorage()
sonde_celery = celery.Celery('sondetasks')
sonde_celery.config_from_object('config.sondeconfig')

@route("/")
def svr_index():
    return "This is irma-brain:\n"

# ______________________________________________________________ API SCAN

def validid(scanid):
    return re.match(r'[0-9a-fA-F]{24}', scanid)

@route("/scan", method='POST')
def scan_new():
    """ send list of filename for scanning """
    oids = {}
    for f in request.files:
        filename = os.path.basename(f)
        upfile = request.files.get(f)
        data = upfile.file.read()
        file_oid = bstorage.store_file(data, name=filename)
        oids[file_oid] = filename
    scanid = str(ObjectId())
    celery.chord(braintasks.scan.s(scanid, oids), braintasks.scan_finished.s(scanid))()
    bstorage.update_scan_record(scanid, {'status':SCAN_STATUS_INIT, 'oids': oids, 'avlist':[]})
    return {"scanid":scanid}

@route("/scan/results/<scanid>", method='GET')
def scan_results(scanid):
    if not validid(scanid):
        return {"error": "not a valid scanid"}
    res = bstorage.get_scan_results(scanid)
    return {"result": res}

@route("/scan/progress/<scanid>", method='GET')
def scan_progress(scanid):
    if not validid(scanid):
        return {"result":"error", "info":"not a valid scanid"}
    status = bstorage.get_scan_status(scanid)
    if status == SCAN_STATUS_INIT:
        return {"result":"not ready", "info":"task not launched"}
    elif status == SCAN_STATUS_LAUNCHED:
        task_id = bstorage.get_scan_taskid(scanid)
        if not task_id:
            return {"result":"error", "info":"task_id not set"}
        job = sonde_celery.GroupResult.restore(task_id)
        if not job:
            return {"result":"error", "info":"not a valid taskid"}
        else:
            nbcompleted = nbsuccessful = 0
            for j in job:
                if j.ready(): nbcompleted += 1
                if j.successful(): nbsuccessful += 1
            return {"result": "in progress", "total":len(job), "finished":nbcompleted, "successful":nbsuccessful}
    elif status == SCAN_STATUS_FINISHED:
        return {"result":"finished"}
    elif status == SCAN_STATUS_CANCELLED:
        return {"result":"cancelled"}
    return {"result":"unknown status %d" % status}


@route("/scan/cancel/<scanid>", method='GET')
def scan_cancel(scanid):
    if not validid(scanid):
        return {"result":"error", "info":"not a valid scanid"}

    status = bstorage.get_scan_status(scanid)
    if status == SCAN_STATUS_INIT:
        return {"result":"not ready", "info":"task not launched"}
    elif status == SCAN_STATUS_LAUNCHED:
        task_id = bstorage.get_scan_taskid(scanid)
        job = sonde_celery.GroupResult.restore(task_id)
        if not job:
            return {"result":"error", "info":"not a valid taskid"}
        else:
            nbcompleted = nbcancelled = 0
            for j in job:
                if j.ready():
                    nbcompleted += 1
                else:
                    j.revoke(terminate=True)
                    nbcancelled += 1
            return {"result": "cancelling", "total":len(job), "finished":nbcompleted, "cancelled":nbcancelled}
    elif status == SCAN_STATUS_FINISHED:
        return {"result":"finished"}
    elif status == SCAN_STATUS_CANCELLED:
        return {"result":"cancelled"}
    return {"result":"unknown status %d" % status}
# ______________________________________________________________ API STATUS

@route("/status")
def status():
    return {"result": "TODO"}

# ______________________________________________________________ API EXPORT

def export(filename, oid):
    """ retrieve a file previously sent to the brain """
    return {"result": "TODO"}

# ______________________________________________________________ MAIN

application = default_app()

if __name__ == "__main__":
    run(host='192.168.130.1', port=8080)<commit_msg>update conf<commit_after>import re
import os
import celery
from bottle import route, request, default_app, abort, run
from brain import braintasks, brainstorage
from config.dbconfig import SCAN_STATUS_INIT, SCAN_STATUS_LAUNCHED, SCAN_STATUS_FINISHED
from bson import ObjectId

bstorage = brainstorage.BrainStorage()
sonde_celery = celery.Celery('sondetasks')
sonde_celery.config_from_object('config.sondeconfig')

@route("/")
def svr_index():
    return "This is irma-brain:\n"

# ______________________________________________________________ API SCAN

def validid(scanid):
    return re.match(r'[0-9a-fA-F]{24}', scanid)

@route("/scan", method='POST')
def scan_new():
    """ send list of filename for scanning """
    oids = {}
    for f in request.files:
        filename = os.path.basename(f)
        upfile = request.files.get(f)
        data = upfile.file.read()
        file_oid = bstorage.store_file(data, name=filename)
        oids[file_oid] = filename
    scanid = str(ObjectId())
    celery.chord(braintasks.scan.s(scanid, oids), braintasks.scan_finished.s(scanid))()
    bstorage.update_scan_record(scanid, {'status':SCAN_STATUS_INIT, 'oids': oids, 'avlist':[]})
    return {"scanid":scanid}

@route("/scan/results/<scanid>", method='GET')
def scan_results(scanid):
    if not validid(scanid):
        return {"error": "not a valid scanid"}
    res = bstorage.get_scan_results(scanid)
    return {"result": res}

@route("/scan/progress/<scanid>", method='GET')
def scan_progress(scanid):
    if not validid(scanid):
        return {"result":"error", "info":"not a valid scanid"}
    status = bstorage.get_scan_status(scanid)
    if status == SCAN_STATUS_INIT:
        return {"result":"not ready", "info":"task not launched"}
    elif status == SCAN_STATUS_LAUNCHED:
        task_id = bstorage.get_scan_taskid(scanid)
        if not task_id:
            return {"result":"error", "info":"task_id not set"}
        job = sonde_celery.GroupResult.restore(task_id)
        if not job:
            return {"result":"error", "info":"not a valid taskid"}
        else:
            nbcompleted = nbsuccessful = 0
            for j in job:
                if j.ready(): nbcompleted += 1
                if j.successful(): nbsuccessful += 1
            return {"result": "in progress", "total":len(job), "finished":nbcompleted, "successful":nbsuccessful}
    elif status == SCAN_STATUS_FINISHED:
        return {"result":"finished"}
    elif status == SCAN_STATUS_CANCELLED:
        return {"result":"cancelled"}
    return {"result":"unknown status %d" % status}


@route("/scan/cancel/<scanid>", method='GET')
def scan_cancel(scanid):
    if not validid(scanid):
        return {"result":"error", "info":"not a valid scanid"}

    status = bstorage.get_scan_status(scanid)
    if status == SCAN_STATUS_INIT:
        return {"result":"not ready", "info":"task not launched"}
    elif status == SCAN_STATUS_LAUNCHED:
        task_id = bstorage.get_scan_taskid(scanid)
        job = sonde_celery.GroupResult.restore(task_id)
        if not job:
            return {"result":"error", "info":"not a valid taskid"}
        else:
            nbcompleted = nbcancelled = 0
            for j in job:
                if j.ready():
                    nbcompleted += 1
                else:
                    j.revoke(terminate=True)
                    nbcancelled += 1
            return {"result": "cancelling", "total":len(job), "finished":nbcompleted, "cancelled":nbcancelled}
    elif status == SCAN_STATUS_FINISHED:
        return {"result":"finished"}
    elif status == SCAN_STATUS_CANCELLED:
        return {"result":"cancelled"}
    return {"result":"unknown status %d" % status}
# ______________________________________________________________ API STATUS

@route("/status")
def status():
    return {"result": "TODO"}

# ______________________________________________________________ API EXPORT

def export(filename, oid):
    """ retrieve a file previously sent to the brain """
    return {"result": "TODO"}

# ______________________________________________________________ MAIN

application = default_app()

if __name__ == "__main__":
    run(host='0.0.0.0', port=8080)
{'fraction_non_alphanumeric': 0.13016479579651302, 'fraction_numerical': 0.005015524241700501, 'mean_word_length': 3.39401888772298, 'pattern_counts': {'":': 68, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5063290', 'n_tokens_mistral': 2568, 'n_tokens_neox': 2490, 'n_words': 598}