<commit_before># Copyright (C) 2016 <NAME>.
# This file is part of CodexGigas - https://github.com/codexgigassys/
# See the file 'LICENSE' for copying permission.
from pymongo import MongoClient
import urllib
import ssdeep
import operator
import hashlib
import time

from PackageControl.PackageController import *
import tree_menu
from virusTotalApi import download_from_virus_total
from Sample import *
from Launcher import *
from secrets import env
from Utils.Functions import clean_hash
from Utils.ProcessDate import parse_date_range

def process_file(file_hash):
    #print "process_file("+str(file_hash)+")"
    pc=PackageController()
    res=pc.getFile(file_hash)
    if res==None:
        return None
    sam_id=file_hash
    sample=Sample()
    sample.setID(sam_id)
    sample.setBinary(res)
    sample.setStorageVersion({})
    lc=Launcher()
    lc.launchAnalysisByID(sample)

    return 0

def add_file_from_vt(hash_id):
    downloaded_file=download_from_virus_total(hash_id)
    if(downloaded_file==None):
        print "add_file_from_vt(): "+str(hash_id)+" not found in VT."
        return None

    print "add_file_from_vt(): downloaded_file is not None."+str(hash_id)
    data_bin=downloaded_file
    file_id=hashlib.sha1(data_bin).hexdigest()
    #print "file_id="+str(file_id)
    pc=PackageController()
    res=pc.searchFile(file_id)
    if(res==None): # File not found. Add it to the package.
        pc.append(file_id,data_bin,True)
        print str(hash_id)+" added to DB from VT."
        #print("Added: %s" % (file_id,))
    else:<commit_msg>issue 286: remove reapeated code<commit_after># Copyright (C) 2016 <NAME>.
# This file is part of CodexGigas - https://github.com/codexgigassys/
# See the file 'LICENSE' for copying permission.
from pymongo import MongoClient
import urllib
import ssdeep
import operator
import hashlib
import time

from PackageControl.PackageController import *
import tree_menu
from virusTotalApi import download_from_virus_total
from Sample import *
from Launcher import *
from secrets import env
from Utils.Functions import clean_hash,process_file
from Utils.ProcessDate import parse_date_range

def add_file_from_vt(hash_id):
    downloaded_file=download_from_virus_total(hash_id)
    if(downloaded_file==None):
        print "add_file_from_vt(): "+str(hash_id)+" not found in VT."
        return None

    print "add_file_from_vt(): downloaded_file is not None."+str(hash_id)
    data_bin=downloaded_file
    file_id=hashlib.sha1(data_bin).hexdigest()
    #print "file_id="+str(file_id)
    pc=PackageController()
    res=pc.searchFile(file_id)
    if(res==None): # File not found. Add it to the package.
        pc.append(file_id,data_bin,True)
        print str(hash_id)+" added to DB from VT."
        #print("Added: %s" % (file_id,))
    else:
{'fraction_non_alphanumeric': 0.09678571428571428, 'fraction_numerical': 0.005, 'mean_word_length': 4.396917148362235, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3582174', 'n_tokens_mistral': 940, 'n_tokens_neox': 909, 'n_words': 234}