<commit_before># NFQ Conductor. A tool for centralizing and visualizing logs.
# Copyright (C) 2017 <NAME>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import datetime
import json
import logging
import multiprocessing
import subprocess
from uuid import uuid4

import psutil
import tornado.httpserver
import tornado.ioloop
import tornado.options
import tornado.web
import zmq
from tornado.options import define, options

from nfq.conductor.runner import launch

UUID = str(uuid4())

define("port", default=8999, help="run on the given port", type=int)
define("interface", default='lo', help="network interface for collector connection", type=str)
define("collector", default='tcp://127.0.0.1:5555', help="Collector socket address", type=str)
define("uuid", default=UUID, help="Unique ID for the daemon", type=str)


class MainHandler(tornado.web.RequestHandler):
    def get(self):
        self.write(options.uuid)


class CpuCountHandler(tornado.web.RequestHandler):
    def get(self):
        self.write(str(psutil.cpu_count()))


class UsageHandler(tornado.web.RequestHandler):
    def get(self):
        usage_str = psutil.cpu_times_percent()
        usage = dict(
            user=usage_str.user,
            system=usage_str.system,
            nice=usage_str.nice,
            iowait=usage_str.iowait,
            irq=usage_str.irq,
            softirq=usage_str.softirq,
            steal=usage_str.steal,
            guest=usage_str.guest,
            guest_nice=usage_str.guest_nice
            )
        self.write(json.dumps(usage))


class ProcessHandler(tornado.web.RequestHandler):
    def post(self, *args, **kwargs):
        command = 'nfq-runner --command "{}" --collector {} --host {}'.format(
            self.request.body.decode(),
            options.collector,
            options.uuid
        )
        logging.info(command)
        name, command = json.loads(self.request.body.decode())
        p = multiprocessing.Process(target=launch,
                                    args=(options.collector,
                                          command,
                                          options.uuid,
                                          name)
                                    )
        p.start()
        logging.info('Launched')

        self.write(command)


class KillHandler(tornado.web.RequestHandler):
    def get(self, pid):
        wrapped, process = pid.split('-')
        subprocess.Popen('kill -9 {}'.format(int(process)), shell=True)
        subprocess.Popen('kill -2 {}'.format(int(wrapped)), shell=True)


class RunningHandler(tornado.web.RequestHandler):
    def get(self, pid):
        self.write(str(psutil.pid_exists(int(pid))))


def run():
    tornado.options.parse_command_line()

    # Fetch network information
    ip = psutil.net_if_addrs()[options.interface][0].address

    context = zmq.Context()
    socket = context.socket(zmq.PUSH)
    logging.info('Connecting to {}'.format(options.collector))

    socket.connect(options.collector)
    
    logging.info('Addr: {}:{}'.format(str(ip), str(options.port)))

    logging.info('Preparing configuration message...')
    socket.send_json({
        'source': options.uuid,
        'when': datetime.datetime.now().isoformat(),
        'message': '^^^^{{"ip": "{}", "port": {}, "uuid": "{}" }}'.format(
            ip, options.port, options.uuid
        )
    })

    socket.close()
    context.destroy()
    logging.info('Sent configuration message')
    logging.info('{{"ip": "{}", "port": {}, "uuid": "{}" }}'.format(
            ip, options.port, options.uuid
        ))

    application = tornado.web.Application([
        (r"/", MainHandler),
        (r"/cpu_count", CpuCountHandler),
        (r"/usage", UsageHandler),
        (r"/send_process", ProcessHandler),
        (r"/kill/(.+)", KillHandler),
        (r"/is_running/([0-9]+)", RunningHandler)
    ], autoreload=False)
    http_server = tornado.httpserver.HTTPServer(application)
    http_server.listen(options.port)
    tornado.ioloop.IOLoop.current().start()


if __name__ == "__main__":
    run()
<commit_msg>Added a ip option in the daemon to advertise to a different IP.<commit_after># NFQ Conductor. A tool for centralizing and visualizing logs.
# Copyright (C) 2017 <NAME>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import datetime
import json
import logging
import multiprocessing
import subprocess
from uuid import uuid4

import psutil
import tornado.httpserver
import tornado.ioloop
import tornado.options
import tornado.web
import zmq
from tornado.options import define, options

from nfq.conductor.runner import launch

UUID = str(uuid4())

define("port", default=8999, help="run on the given port", type=int)
define("interface", default='lo',
       help="network interface for collector connection", type=str)
define("collector", default='tcp://127.0.0.1:5555',
       help="Collector socket address", type=str)
define("ip", default='',
       help="Ip from which the http service can be accesed. Key"
            "if you want to put the daemon within a container.", type=str)
define("uuid", default=UUID, help="Unique ID for the daemon", type=str)


class MainHandler(tornado.web.RequestHandler):
    def get(self):
        self.write(options.uuid)


class CpuCountHandler(tornado.web.RequestHandler):
    def get(self):
        self.write(str(psutil.cpu_count()))


class UsageHandler(tornado.web.RequestHandler):
    def get(self):
        usage_str = psutil.cpu_times_percent()
        usage = dict(
            user=usage_str.user,
            system=usage_str.system,
            nice=usage_str.nice,
            iowait=usage_str.iowait,
            irq=usage_str.irq,
            softirq=usage_str.softirq,
            steal=usage_str.steal,
            guest=usage_str.guest,
            guest_nice=usage_str.guest_nice
            )
        self.write(json.dumps(usage))


class ProcessHandler(tornado.web.RequestHandler):
    def post(self, *args, **kwargs):
        command = 'nfq-runner --command "{}" --collector {} --host {}'.format(
            self.request.body.decode(),
            options.collector,
            options.uuid
        )
        logging.info(command)
        name, command = json.loads(self.request.body.decode())
        p = multiprocessing.Process(target=launch,
                                    args=(options.collector,
                                          command,
                                          options.uuid,
                                          name)
                                    )
        p.start()
        logging.info('Launched')

        self.write(command)


class KillHandler(tornado.web.RequestHandler):
    def get(self, pid):
        wrapped, process = pid.split('-')
        subprocess.Popen('kill -9 {}'.format(int(process)), shell=True)
        subprocess.Popen('kill -2 {}'.format(int(wrapped)), shell=True)


class RunningHandler(tornado.web.RequestHandler):
    def get(self, pid):
        self.write(str(psutil.pid_exists(int(pid))))


def run():
    tornado.options.parse_command_line()

    # Fetch network information
    if options.ip:
        ip = options.ip
    else:
        ip = psutil.net_if_addrs()[options.interface][0].address

    context = zmq.Context()
    socket = context.socket(zmq.PUSH)
    logging.info('Connecting to {}'.format(options.collector))

    socket.connect(options.collector)
    
    logging.info('Addr: {}:{}'.format(str(ip), str(options.port)))
    logging.info('Trying to send configuration to collector: {}'.format(
        options.collector))
    logging.info('Preparing configuration message...')
    socket.send_json({
        'source': options.uuid,
        'when': datetime.datetime.now().isoformat(),
        'message': '^^^^{{"ip": "{}", "port": {}, "uuid": "{}" }}'.format(
            ip, options.port, options.uuid
        )
    })

    socket.close()
    context.destroy()
    logging.info('Configuration message sent.')
    logging.info('{{"ip": "{}", "port": {}, "uuid": "{}" }}'.format(
            ip, options.port, options.uuid
        ))

    application = tornado.web.Application([
        (r"/", MainHandler),
        (r"/cpu_count", CpuCountHandler),
        (r"/usage", UsageHandler),
        (r"/send_process", ProcessHandler),
        (r"/kill/(.+)", KillHandler),
        (r"/is_running/([0-9]+)", RunningHandler)
    ], autoreload=False)
    http_server = tornado.httpserver.HTTPServer(application)
    http_server.listen(options.port)
    tornado.ioloop.IOLoop.current().start()


if __name__ == "__main__":
    run()

{'fraction_non_alphanumeric': 0.12790103261425212, 'fraction_numerical': 0.0053164298129025666, 'mean_word_length': 2.6955043445409896, 'pattern_counts': {'":': 14, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4787961', 'n_tokens_mistral': 2876, 'n_tokens_neox': 2786, 'n_words': 710}