<commit_before>from bottle import route
from bottle import request
from bottle import response
import os
path = os.path.abspath(os.path.join(
    os.path.dirname(os.path.abspath(__file__)), '..'))
import sys
sys.path.insert(0, path)
from Utils.Functions import jsonize
from bson.json_util import dumps
from Utils.Functions import clean_hash
from Utils.Functions import check_hashes
from Utils.Functions import change_date_to_str
from Utils.Functions import id_generator
from Utils.Functions import to_bool
from Utils.Functions import get_file_id
from Utils.Functions import add_error
from Utils.Functions import valid_hash
from PackageControl.PackageController import *
from MetaControl.MetaController import *
from VersionControl.VersionController import *
from process_hash import generic_process_hash
from virusTotalApi import get_av_result
from virusTotalApi import save_file_from_vt
from Utils.mailSender import send_mail
import datetime
from rq import Queue
from redis import Redis
from IPython import embed
import time

@route('/api/v1/task', method='OPTIONS')
def enable_cors_for_task():
    return 0

# Return true if the task has
# finished. False otherwise.
@route('/api/v1/task_finished', method='GET')
def task_finished():
    task_id = request.query.get('task_id')
    task = get_task(task_id)
    return dumps({"has_finished": task.get('date_end') is not None})

@route('/api/v1/task', method='GET')
def api_get_task():
    task_id = request.query.get('task_id')
    return dumps(get_task(task_id))

def get_task(task_id):
    mc = MetaController()
    task_report = mc.read_task(task_id)
    if task_report is not None:
        return change_date_to_str(task_report)
    else:
        return add_error({}, 8, "Task not found")

@route('/api/v1/task', method='POST')
def task():
    file_hash = request.forms.get('file_hash')
    vt_av = to_bool(request.forms.get('vt_av'))
    vt_samples = to_bool(request.forms.get('vt_samples'))
    process = to_bool(request.forms.get('process'))
    email = request.forms.get('email')
    document_name = request.forms.get('document_name')
    task_id = add_task(process, file_hash, vt_av, vt_samples, email, document_name)
    return dumps({"task_id": task_id})

def add_task(process,file_hash,vt_av,vt_samples,email,document_name):
    task_id = id_generator(40)
    response = {"requested": {
        "vt_av": vt_av,
        "vt_samples": vt_samples,
        "process": process,
        "email": email,
        "document_name": document_name,
        "file_hash": file_hash
        },
        "date_enqueued": datetime.datetime.now(),
        "task_id": task_id }
    save(response)
    if vt_samples:
        queue_name = "task_private_vt" # task needs a private VT api
    elif vt_av and not vt_samples:
        queue_name = "task_public_vt" # task needs a public VT api
    else:
        queue_name = "task_no_vt" # task doesn't need VT
    q = Queue(queue_name, connection=Redis(host=env.get('redis').get('host')))
    job = q.enqueue('Api.task.generic_task', args=(
        process, file_hash, vt_av, vt_samples, email,task_id,document_name), timeout=31536000)
    return task_id

def add_task_to_download_av_result(file_hash):
    return add_task(True,file_hash,True,False,"","[automatic-request-from-api]")

def generic_task(process, file_hash, vt_av, vt_samples, email,task_id,document_name=""):
    print "task_id="+str(task_id)
    generic_count = 0
    response = {}
    response["date_start"] = datetime.datetime.now()
    response["document_name"] = document_name
    response["task_id"] = task_id
    check_hashes_output = check_hashes(file_hash)
    errors = check_hashes_output.get('errors')
    for error in errors:
        key = error.get('error')
        value = error.get('error_message')
        print "errors (key="+str(key)+", value="+str(value)+")"
        response = add_error(response, key, value)
    hashes = check_hashes_output.get('hashes')
    remove_dups_output = remove_dups(hashes)
    # remove duplicated hashes
    hashes = remove_dups_output.get('list')
    response["duplicated_hashes"] = remove_dups_output.get('dups')
    response["hashes"] = hashes

    hash_dicts = []
    mc = MetaController()
    for x in hashes:
        x_dict = {}
        x_dict["original"] = x
        x_dict["sha1"] = get_file_id(x)
        if(x_dict["sha1"] is not None):
            doc = mc.read(x_dict["sha1"])
            if doc is not None and doc.get('hash') is not None:
                if doc.get('hash').get('md5') is not None:
                    x_dict["md5"] = doc.get('hash').get('md5')
                if doc.get('hash').get('sha2') is not None:
                    x_dict["sha2"] = doc.get('hash').get('sha2')
        hash_dicts.append(x_dict)
    response["duplicated_samples"] = []
    for x in hash_dicts:
        for y in hash_dicts:
            if x.get('original') != y.get('original') and (
                    x.get('original') == y.get('sha1') or
                    x.get('original') == y.get('md5') or
                    x.get('original') == y.get('sha2')):
                response["duplicated_samples"].append(y.get('original'))
                hash_dicts.remove(y)
    hashes=[]
    for x in hash_dicts:
        hashes.append(x.get('original'))
    response["hashes"]=hashes

    if(len(hashes) == 0):
        response = add_error(response, 6, "No valid hashes provided.")
        response["date_end"] = datetime.datetime.now()
        save(response)
        return change_date_to_str(response)

    save(response)

    response["inconsistencies"]=[]
    for hash_id in hashes:
        if fix_inconsistency(hash_id) == 1:
            response["inconsistencies"].append(hash_id)

    save(response)

    response["not_found_on_vt"] = []
    response["private_credits_spent"] = 0
    if vt_samples:
        response["downloaded"] = []
        for hash_id in hashes:
            if(get_file_id(hash_id) is None or db_inconsistency(hash_id)):
                print "task(): "+hash_id+" was not found (get_file_id returned None). "
                generic_count += 1
                if (generic_count % 20 == 0):
                    save(response)
                output = save_file_from_vt(hash_id)
                if(output.get('status') == 'out_of_credits'):
                    request_successful = False
                    while not request_successful:
                        output = save_file_from_vt(hash_id)
                        if output.get('status') != 'out_of_credits':
                            request_successful = True
                if(output.get('status') == 'added'):
                    response["downloaded"].append(hash_id)
                    response["private_credits_spent"]+=1
                elif(output.get('status') == 'inconsistency_found'):
                    response["private_credits_spent"]+=1
                    generic_process_hash(hash_id)
                elif(output.get('status') == 'not_found'):
                    response["not_found_on_vt"].append(hash_id)
                else:
                    response = add_error(response,11,"Unknown error when downloading sample from VT."+str(output))
                save(response)
    save(response)
    response["processed"] = []
    response["not_found_for_processing"] = []
    if process:
        print "process=true"<commit_msg>issue 448: it failed when hash was sha256<commit_after>from bottle import route
from bottle import request
from bottle import response
import os
path = os.path.abspath(os.path.join(
    os.path.dirname(os.path.abspath(__file__)), '..'))
import sys
sys.path.insert(0, path)
from Utils.Functions import jsonize
from bson.json_util import dumps
from Utils.Functions import clean_hash
from Utils.Functions import check_hashes
from Utils.Functions import change_date_to_str
from Utils.Functions import id_generator
from Utils.Functions import to_bool
from Utils.Functions import get_file_id
from Utils.Functions import add_error
from Utils.Functions import valid_hash
from PackageControl.PackageController import *
from MetaControl.MetaController import *
from VersionControl.VersionController import *
from process_hash import generic_process_hash
from virusTotalApi import get_av_result
from virusTotalApi import save_file_from_vt
from Utils.mailSender import send_mail
import datetime
from rq import Queue
from redis import Redis
from IPython import embed
import time

@route('/api/v1/task', method='OPTIONS')
def enable_cors_for_task():
    return 0

# Return true if the task has
# finished. False otherwise.
@route('/api/v1/task_finished', method='GET')
def task_finished():
    task_id = request.query.get('task_id')
    task = get_task(task_id)
    return dumps({"has_finished": task.get('date_end') is not None})

@route('/api/v1/task', method='GET')
def api_get_task():
    task_id = request.query.get('task_id')
    return dumps(get_task(task_id))

def get_task(task_id):
    mc = MetaController()
    task_report = mc.read_task(task_id)
    if task_report is not None:
        return change_date_to_str(task_report)
    else:
        return add_error({}, 8, "Task not found")

@route('/api/v1/task', method='POST')
def task():
    file_hash = request.forms.get('file_hash')
    vt_av = to_bool(request.forms.get('vt_av'))
    vt_samples = to_bool(request.forms.get('vt_samples'))
    process = to_bool(request.forms.get('process'))
    email = request.forms.get('email')
    document_name = request.forms.get('document_name')
    task_id = add_task(process, file_hash, vt_av, vt_samples, email, document_name)
    return dumps({"task_id": task_id})

def add_task(process,file_hash,vt_av,vt_samples,email,document_name):
    task_id = id_generator(40)
    response = {"requested": {
        "vt_av": vt_av,
        "vt_samples": vt_samples,
        "process": process,
        "email": email,
        "document_name": document_name,
        "file_hash": file_hash
        },
        "date_enqueued": datetime.datetime.now(),
        "task_id": task_id }
    save(response)
    if vt_samples:
        queue_name = "task_private_vt" # task needs a private VT api
    elif vt_av and not vt_samples:
        queue_name = "task_public_vt" # task needs a public VT api
    else:
        queue_name = "task_no_vt" # task doesn't need VT
    q = Queue(queue_name, connection=Redis(host=env.get('redis').get('host')))
    job = q.enqueue('Api.task.generic_task', args=(
        process, file_hash, vt_av, vt_samples, email,task_id,document_name), timeout=31536000)
    return task_id

def add_task_to_download_av_result(file_hash):
    return add_task(True,file_hash,True,False,"","[automatic-request-from-api]")

def generic_task(process, file_hash, vt_av, vt_samples, email,task_id,document_name=""):
    print "task_id="+str(task_id)
    generic_count = 0
    response = {}
    response["date_start"] = datetime.datetime.now()
    response["document_name"] = document_name
    response["task_id"] = task_id
    check_hashes_output = check_hashes(file_hash)
    errors = check_hashes_output.get('errors')
    for error in errors:
        key = error.get('error')
        value = error.get('error_message')
        print "errors (key="+str(key)+", value="+str(value)+")"
        response = add_error(response, key, value)
    hashes = check_hashes_output.get('hashes')
    remove_dups_output = remove_dups(hashes)
    # remove duplicated hashes
    hashes = remove_dups_output.get('list')
    response["duplicated_hashes"] = remove_dups_output.get('dups')
    response["hashes"] = hashes

    hash_dicts = []
    mc = MetaController()
    for x in hashes:
        x_dict = {}
        x_dict["original"] = x
        x_dict["sha1"] = get_file_id(x)
        if(x_dict["sha1"] is not None):
            doc = mc.read(x_dict["sha1"])
            if doc is not None and doc.get('hash') is not None:
                if doc.get('hash').get('md5') is not None:
                    x_dict["md5"] = doc.get('hash').get('md5')
                if doc.get('hash').get('sha2') is not None:
                    x_dict["sha2"] = doc.get('hash').get('sha2')
        hash_dicts.append(x_dict)
    response["duplicated_samples"] = []
    for x in hash_dicts:
        for y in hash_dicts:
            if x.get('original') != y.get('original') and (
                    x.get('original') == y.get('sha1') or
                    x.get('original') == y.get('md5') or
                    x.get('original') == y.get('sha2')):
                response["duplicated_samples"].append(y.get('original'))
                hash_dicts.remove(y)
    hashes=[]
    for x in hash_dicts:
        hashes.append(x.get('original'))
    response["hashes"]=hashes

    if(len(hashes) == 0):
        response = add_error(response, 6, "No valid hashes provided.")
        response["date_end"] = datetime.datetime.now()
        save(response)
        return change_date_to_str(response)

    save(response)

    response["inconsistencies"]=[]
    for hash_id in hashes:
        if fix_inconsistency(hash_id) == 1:
            response["inconsistencies"].append(hash_id)

    save(response)

    response["not_found_on_vt"] = []
    response["private_credits_spent"] = 0
    if vt_samples:
        response["downloaded"] = []
        for hash_id in hashes:
            if(get_file_id(hash_id) is None or db_inconsistency(hash_id)):
                print "task(): "+hash_id+" was not found (get_file_id returned None). "
                generic_count += 1
                if (generic_count % 20 == 0):
                    save(response)
                output = save_file_from_vt(hash_id)
                sha1 = output.get('hash')
                if(output.get('status') == 'out_of_credits'):
                    request_successful = False
                    while not request_successful:
                        output = save_file_from_vt(hash_id)
                        if output.get('status') != 'out_of_credits':
                            request_successful = True
                if(output.get('status') == 'added'):
                    response["downloaded"].append(hash_id)
                    # we need to process the sha1, and not the sha2 because
                    # the grid does not save the sha2.
                    generic_process_hash(sha1)
                    response["private_credits_spent"]+=1
                elif(output.get('status') == 'inconsistency_found'):
                    response["private_credits_spent"]+=1
                    generic_process_hash(sha1)
                elif(output.get('status') == 'not_found'):
                    response["not_found_on_vt"].append(hash_id)
                else:
                    response = add_error(response,11,"Unknown error when downloading sample from VT."+str(output))
                save(response)
    save(response)
    response["processed"] = []
    response["not_found_for_processing"] = []
    if process:
        print "process=true"
{'fraction_non_alphanumeric': 0.12126702905845245, 'fraction_numerical': 0.006442520636198913, 'mean_word_length': 2.878709005726184, 'pattern_counts': {'":': 22, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1496108', 'n_tokens_mistral': 4737, 'n_tokens_neox': 4603, 'n_words': 985}