<commit_before>import os
from flask import Flask
from flask import render_template
from flask import request, redirect, url_for
from werkzeug import secure_filename
import hashlib

from pymongo import Connection
from bson.code import Code

import dpkt
import sys
import socket


from celery import Celery

import requests
from requests import Request, Session

import magic
import zlib

import yara
import pyclamd

import datetime
from time import sleep

from bson.objectid import ObjectId

from zipfile import ZipFile
import redis
import json

# FIXME: move to config.py
ALLOWED_EXTENSIONS = set(['pcap'])
rules = yara.compile(filepath='yara/ekanalyzer.yar')

cd = pyclamd.ClamdAgnostic()

def create_app():
    return Flask("ekanalyzer")

app = create_app()
app.config.from_pyfile('config.py')


connection = Connection(app.config['MONGODB_SERVER'] , app.config['MONGODB_PORT'])
db = connection.ekanalyzer

memcache = redis.Redis('localhost')


app.debug = True

celery = Celery('ekanalyzer', broker=app.config['BROKER_URL'] )


@celery.task
def perform_results(pcap_id):
    try:

        pcap = {'_id' : ObjectId(pcap_id)}

        result = db.pcap.find_one(pcap)

        #if result.count() > 0:
        #    return
        #else:
        #    db.pcap.insert(pcap)

        pcap_hash = result['id']


        f = open(app.config['UPLOAD_FOLDER'] + pcap_hash, "rb")

        pcap = dpkt.pcap.Reader(f)
        for ts, buf in pcap:
            eth = dpkt.ethernet.Ethernet(buf)
            ip = eth.data
            tcp = ip.data
            if type(tcp) is str:
              continue
            # FIXME: assuming only http traffic on port 80
            if tcp.dport == 80 and len(tcp.data) > 0:
                http = dpkt.http.Request(tcp.data)
                ipaddress =  socket.inet_ntoa(ip.dst)
                
                data = { 'ip' : ipaddress,
                         'uri' : http.uri,
                         'method' : http.method,
                         'data' : http.data,
                         'headers' : http.headers,
                         'hash': pcap_hash,
                         'pcap_id' : ObjectId(pcap_id)
                       }                      
                db.requests.insert(data)                       
            #else:
            #    print "Port is " + str(tcp.dport)

    
    except dpkt.NeedData as e:
        print e
    except AttributeError as e:
        print e
    except NameError as e:
        print e
    except :
        print "Unexpected error:", sys.exc_info()
        print pcap_hash
    finally:
        status = process_requests(pcap_id)


def process_requests(pcap_id):

    request = { 'pcap_id' : ObjectId(pcap_id)}

    result = db.requests.find(request)
    nrequests = result.count()
    uas = len(app.config['USER_AGENTS'])
    nrequests*=uas
    nrequests+=1
    memcache.set(str(pcap_id) + "_tasks", str(nrequests))
    memcache.set(str(pcap_id) + "_total_tasks", str(nrequests))

    print "added %s tasks" % str(nrequests)

    for r in result:
        # Maybe hash is not necesary    
        print process_request.delay(r['ip'], r['uri'], r['method'], r['headers'], r['data'], r['hash'], r['pcap_id'])



def extract_zip(input_zip):
    input_zip=ZipFile(input_zip)
    return {name: input_zip.read(name) for name in input_zip.namelist()}

def check_vt(hash, mimetype):
        #
        # This function uses response (buffer) and fpath (path to file)
        # FIX this as soon as the "/" bug be fixed (gridfs)
        #

        vt_report = None

        try:
          vt_report_raw = memcache.get(hash)
          print "report  = " + vt_report_raw
          vt_report = json.loads(vt_report_raw)
          print "vtreport  = " + vt_report
        except Exception, e:
          print "The report cannot be loaded for %s" % hash
          vt_report = None

        if vt_report == None: 

          # Send to VT
          if mimetype == "application/octet-stream" \
              or mimetype == "application/java-archive" \
              or mimetype == "application/zip" \
              or mimetype == "application/pdf" \<commit_msg>Fixed cache bug<commit_after>import os
from flask import Flask
from flask import render_template
from flask import request, redirect, url_for
from werkzeug import secure_filename
import hashlib

from pymongo import Connection
from bson.code import Code

import dpkt
import sys
import socket


from celery import Celery

import requests
from requests import Request, Session

import magic
import zlib

import yara
import pyclamd

import datetime
from time import sleep

from bson.objectid import ObjectId

from zipfile import ZipFile
import redis
import json

# FIXME: move to config.py
ALLOWED_EXTENSIONS = set(['pcap'])
rules = yara.compile(filepath='yara/ekanalyzer.yar')

cd = pyclamd.ClamdAgnostic()

def create_app():
    return Flask("ekanalyzer")

app = create_app()
app.config.from_pyfile('config.py')


connection = Connection(app.config['MONGODB_SERVER'] , app.config['MONGODB_PORT'])
db = connection.ekanalyzer

memcache = redis.Redis('localhost')


app.debug = True

celery = Celery('ekanalyzer', broker=app.config['BROKER_URL'] )


@celery.task
def perform_results(pcap_id):
    try:

        pcap = {'_id' : ObjectId(pcap_id)}

        result = db.pcap.find_one(pcap)

        #if result.count() > 0:
        #    return
        #else:
        #    db.pcap.insert(pcap)

        pcap_hash = result['id']


        f = open(app.config['UPLOAD_FOLDER'] + pcap_hash, "rb")

        pcap = dpkt.pcap.Reader(f)
        for ts, buf in pcap:
            eth = dpkt.ethernet.Ethernet(buf)
            ip = eth.data
            tcp = ip.data
            if type(tcp) is str:
              continue
            # FIXME: assuming only http traffic on port 80
            if tcp.dport == 80 and len(tcp.data) > 0:
                http = dpkt.http.Request(tcp.data)
                ipaddress =  socket.inet_ntoa(ip.dst)
                
                data = { 'ip' : ipaddress,
                         'uri' : http.uri,
                         'method' : http.method,
                         'data' : http.data,
                         'headers' : http.headers,
                         'hash': pcap_hash,
                         'pcap_id' : ObjectId(pcap_id)
                       }                      
                db.requests.insert(data)                       
            #else:
            #    print "Port is " + str(tcp.dport)

    
    except dpkt.NeedData as e:
        print e
    except AttributeError as e:
        print e
    except NameError as e:
        print e
    except :
        print "Unexpected error:", sys.exc_info()
        print pcap_hash
    finally:
        status = process_requests(pcap_id)


def process_requests(pcap_id):

    request = { 'pcap_id' : ObjectId(pcap_id)}

    result = db.requests.find(request)
    nrequests = result.count()
    uas = len(app.config['USER_AGENTS'])
    nrequests*=uas
    nrequests+=1
    memcache.set(str(pcap_id) + "_tasks", str(nrequests))
    memcache.set(str(pcap_id) + "_total_tasks", str(nrequests))

    print "added %s tasks" % str(nrequests)

    for r in result:
        # Maybe hash is not necesary    
        print process_request.delay(r['ip'], r['uri'], r['method'], r['headers'], r['data'], r['hash'], r['pcap_id'])



def extract_zip(input_zip):
    input_zip=ZipFile(input_zip)
    return {name: input_zip.read(name) for name in input_zip.namelist()}

def check_vt(hash, mimetype):
        #
        # This function uses response (buffer) and fpath (path to file)
        # FIX this as soon as the "/" bug be fixed (gridfs)
        #

        vt_report = None

        try:
          vt_report_raw = memcache.get(hash)
          vt_report = json.loads(vt_report_raw)
        except Exception as e:
          print "The report cannot be loaded for %s: %s" % (hash, e)
          vt_report = None

        if vt_report == None: 

          # Send to VT
          if mimetype == "application/octet-stream" \
              or mimetype == "application/java-archive" \
              or mimetype == "application/zip" \
              or mimetype == "application/pdf" \
{'fraction_non_alphanumeric': 0.10328409229642291, 'fraction_numerical': 0.0017091930167256746, 'mean_word_length': 2.056716417910448, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '450994', 'n_tokens_mistral': 2622, 'n_tokens_neox': 2512, 'n_words': 655}