<commit_before>#!/usr/bin/env python

import argparse
import redis
import driller.tasks
import os
import subprocess
import signal
import sys
import multiprocessing

import driller.config as config

import logging

l = logging.getLogger("run")
l.setLevel("DEBUG")

# global list of processes so we can kill them on SIGINT
procs = [ ] 

def terminate(signal, frame):
    map(lambda p: p.terminate(), procs)
    sys.exit(0)

def start_afl(afl_path, binary, in_dir, out_dir, fuzz_id, dictionary=None, memory="8G",
                driller=None, qemu_path=None):

    args = [afl_path]

    args += ["-i", in_dir]
    args += ["-o", out_dir]
    args += ["-m", memory]
    args += ["-Q"]
    if fuzz_id == 0:
        args += ["-M", "fuzzer-master"]
        outfile = "fuzzer-master.log"
    else:
        args += ["-S", "fuzzer-%d" % fuzz_id]
        outfile = "fuzzer-%d.log" % fuzz_id

    if dictionary is not None:
        args += ["-x", dictionary]

    if driller is not None:
        args += ["-D", driller]

    args += ["--", binary]

    l.debug("execing: %s > %s" % (' '.join(args), outfile))

    fp = open(outfile, "w")
    return subprocess.Popen(args, stdout=fp)

def start_afl_master(afl_path, binary, in_dir, out_dir, dictionary=None):

    return start_afl(afl_path, binary, in_dir, out_dir, fuzz_id=0, dictionary=dictionary)

def start_afl_slave(afl_path, binary, in_dir, out_dir, fuzz_id, dictionary=None):

    return start_afl(afl_path, binary, in_dir, out_dir, fuzz_id, dictionary=dictionary)

def start_afl_driller(afl_path, binary, in_dir, out_dir, fuzz_id, driller, qemu_path, 
                dictionary=None):

    return start_afl(afl_path, binary, in_dir, out_dir, fuzz_id, dictionary=dictionary, 
                driller=driller, qemu_path=qemu_path)

def clear_redis(identifier):
    redis_inst = redis.Redis(host=config.REDIS_HOST, port=config.REDIS_PORT, db=config.REDIS_DB)

    # delete all the catalogue entries
    redis_inst.delete("%s-catalogue" % identifier)

    # delete all the traced entries
    redis_inst.delete("%s-traced" % identifier)

def listen(queue_dir, channel):
    l.info("subscring to redis channel %s" % channel)

    redis_inst = redis.Redis(host=config.REDIS_HOST, port=config.REDIS_PORT, db=config.REDIS_DB)
    p = redis_inst.pubsub()

    p.subscribe(channel)

    input_cnt = 0

    for msg in p.listen():
        if msg['type'] == 'message':
            real_msg = pickle.loads(msg['data'])
            out_filename = "driller-%d-%x-%x" % real_msg['meta']
            l.info("dumping new input to %s" % out_filename)
            afl_name = "id:%06d,src:%s" % (input_cnt, out_filename)
            out_file = os.path.join(queue_dir, afl_name)

            with open(out_file, 'wb') as ofp:
                ofp.write(real_msg['data'])

            input_cnt += 1

def start_redis_listener(identifier, out_dir):
    driller_queue_dir = os.path.join(out_dir, "driller", "queue")
    channel = "%s-generated" % identifier<commit_msg>Create output directory within the listener and pass in the qemu_path to the AFL<commit_after>#!/usr/bin/env python

import argparse
import redis
import driller.tasks
import os
import subprocess
import signal
import sys
import multiprocessing

import driller.config as config

import logging

l = logging.getLogger("run")
l.setLevel("DEBUG")

# global list of processes so we can kill them on SIGINT
procs = [ ] 

def terminate(signal, frame):
    map(lambda p: p.terminate(), procs)
    sys.exit(0)

def start_afl(afl_path, binary, in_dir, out_dir, fuzz_id, dictionary=None, memory="8G",
                driller=None, qemu_path=None):

    args = [afl_path]

    args += ["-i", in_dir]
    args += ["-o", out_dir]
    args += ["-m", memory]
    args += ["-Q"]
    if fuzz_id == 0:
        args += ["-M", "fuzzer-master"]
        outfile = "fuzzer-master.log"
    else:
        args += ["-S", "fuzzer-%d" % fuzz_id]
        outfile = "fuzzer-%d.log" % fuzz_id

    if dictionary is not None:
        args += ["-x", dictionary]

    if driller is not None:
        args += ["-D", driller]

    if qemu_path is not None:
        args += ["-q", qemu_path]

    args += ["--", binary]

    l.debug("execing: %s > %s" % (' '.join(args), outfile))

    fp = open(outfile, "w")
    return subprocess.Popen(args, stdout=fp)

def start_afl_master(afl_path, binary, in_dir, out_dir, dictionary=None):

    return start_afl(afl_path, binary, in_dir, out_dir, fuzz_id=0, dictionary=dictionary)

def start_afl_slave(afl_path, binary, in_dir, out_dir, fuzz_id, dictionary=None):

    return start_afl(afl_path, binary, in_dir, out_dir, fuzz_id, dictionary=dictionary)

def start_afl_driller(afl_path, binary, in_dir, out_dir, fuzz_id, driller, qemu_path, 
                dictionary=None):

    return start_afl(afl_path, binary, in_dir, out_dir, fuzz_id, dictionary=dictionary, 
                driller=driller, qemu_path=qemu_path)

def clear_redis(identifier):
    redis_inst = redis.Redis(host=config.REDIS_HOST, port=config.REDIS_PORT, db=config.REDIS_DB)

    # delete all the catalogue entries
    redis_inst.delete("%s-catalogue" % identifier)

    # delete all the traced entries
    redis_inst.delete("%s-traced" % identifier)

def listen(queue_dir, channel):
    l.info("subscring to redis channel %s" % channel)
    l.info("new inputs will be placed into %s" % queue_dir)

    try:
        os.makedirs(queue_dir)
    except OSError:
        l.warning("could not create output directory '%s'" % queue_dir)

    redis_inst = redis.Redis(host=config.REDIS_HOST, port=config.REDIS_PORT, db=config.REDIS_DB)
    p = redis_inst.pubsub()

    p.subscribe(channel)

    input_cnt = 0

    for msg in p.listen():
        if msg['type'] == 'message':
            real_msg = pickle.loads(msg['data'])
            out_filename = "driller-%d-%x-%x" % real_msg['meta']
            l.info("dumping new input to %s" % out_filename)
            afl_name = "id:%06d,src:%s" % (input_cnt, out_filename)
            out_file = os.path.join(queue_dir, afl_name)

            with open(out_file, 'wb') as ofp:
                ofp.write(real_msg['data'])

            input_cnt += 1

def start_redis_listener(identifier, out_dir):
    driller_queue_dir = os.path.join(out_dir, "driller", "queue")
    channel = "%s-generated" % identifier
{'fraction_non_alphanumeric': 0.13267735280114268, 'fraction_numerical': 0.0025392794794477066, 'mean_word_length': 3.212566844919786, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '687130', 'n_tokens_mistral': 2291, 'n_tokens_neox': 2216, 'n_words': 521}