<commit_before># -*- coding: utf-8 -*-
# vim: set ts=4

# Copyright 2016 <NAME>
# This file is part of ReactOBus.
#
# ReactOBus is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# ReactOBus is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with ReactOBus.  If not, see <http://www.gnu.org/licenses/>

import logging
from setproctitle import setproctitle
import multiprocessing
import re
import subprocess
import threading
import zmq
from zmq.utils.strtypes import b, u

LOG = logging.getLogger("ROB.lib.reactor")


class Matcher(object):
    def __init__(self, rule):
        self.name = rule["name"]
        self.field = rule["match"]["field"]
        self.pattern = re.compile(rule["match"]["pattern"])
        self.binary = rule["exec"]["path"]
        self.timeout = rule["exec"]["timeout"]
        self.args = rule["exec"]["args"]

    @classmethod
    def lookup(cls, name, variables, data):
        # Lookup in variables and fallback to data if the name is of the form
        # "data.key"
        if name in variables:
            return variables[name]
        elif name == "data":
            return data
        elif name.startswith("data."):
            sub_name = name[5:]
            if sub_name in data:
                return data[sub_name]
        raise KeyError(name)

    def match(self, variables, data):
        # Returne True if the field does match, overwise return False
        try:
            value = self.lookup(self.field, variables, data)
            return self.pattern.match(value) is not None
        except KeyError:
            return False

    def build_args(self, topic, uuid, datetime, username, data):
        variables = {"topic": topic,
                     "uuid": uuid,
                     "datetime": datetime,
                     "username": username}

        args = [self.binary]
        stdin = []
        # Make the substitution
        for field in self.args:
            if field.startswith("$"):
                args.append(self.lookup(field[1:], variables, data))
            elif field.startswith("stdin:"):
                if field[6:].startswith("$"):
                    stdin.append(self.lookup(field[7:], variables, data))
                else:
                    stdin.append(field[6:])
            else:
                args.append(field)
        return (args, "\n".join(stdin))

    def run(self, topic, uuid, datetime, username, data):
        try:
            (args, stdin_s) = self.build_args(topic, uuid, datetime, username,
                                              data)
        except KeyError as exc:
            LOG.error("Unable to build the argument list: %s", exc)
            return

        LOG.debug("Running: %s", args)
        try:
            out = subprocess.check_output(args, stderr=subprocess.STDOUT,
                                          universal_newlines=True,
                                          input=stdin_s, timeout=self.timeout)
        except OSError as err:
            LOG.error("Unable to run %s (%s)", args, err)
        except subprocess.TimeoutExpired:
            LOG.error("Timeout when running %s", args)
        else:
            LOG.debug("=> %s", out)


class Worker(threading.Thread):
    def __init__(self, matchers):
        super().__init__()
        self.matchers = matchers

    def run(self):
        self.context = zmq.Context.instance()
        self.sock = self.context.socket(zmq.DEALER)
        self.sock.connect("inproc://workers")
        while True:
            msg = self.sock.recv_multipart()
            matcher_index = int(msg[0])
            (topic, uuid, dt, username, data) = msg[1:]
            LOG.debug("Running matcher n°%d on %s", matcher_index, self.name)
            self.matchers[matcher_index].run(topic, uuid, dt, username, data)


class Reactor(multiprocessing.Process):
    def __init__(self, options, outbound):<commit_msg>Remove non-ascii from log output<commit_after># -*- coding: utf-8 -*-
# vim: set ts=4

# Copyright 2016 <NAME>
# This file is part of ReactOBus.
#
# ReactOBus is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# ReactOBus is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with ReactOBus.  If not, see <http://www.gnu.org/licenses/>

import logging
from setproctitle import setproctitle
import multiprocessing
import re
import subprocess
import threading
import zmq
from zmq.utils.strtypes import b, u

LOG = logging.getLogger("ROB.lib.reactor")


class Matcher(object):
    def __init__(self, rule):
        self.name = rule["name"]
        self.field = rule["match"]["field"]
        self.pattern = re.compile(rule["match"]["pattern"])
        self.binary = rule["exec"]["path"]
        self.timeout = rule["exec"]["timeout"]
        self.args = rule["exec"]["args"]

    @classmethod
    def lookup(cls, name, variables, data):
        # Lookup in variables and fallback to data if the name is of the form
        # "data.key"
        if name in variables:
            return variables[name]
        elif name == "data":
            return data
        elif name.startswith("data."):
            sub_name = name[5:]
            if sub_name in data:
                return data[sub_name]
        raise KeyError(name)

    def match(self, variables, data):
        # Returne True if the field does match, overwise return False
        try:
            value = self.lookup(self.field, variables, data)
            return self.pattern.match(value) is not None
        except KeyError:
            return False

    def build_args(self, topic, uuid, datetime, username, data):
        variables = {"topic": topic,
                     "uuid": uuid,
                     "datetime": datetime,
                     "username": username}

        args = [self.binary]
        stdin = []
        # Make the substitution
        for field in self.args:
            if field.startswith("$"):
                args.append(self.lookup(field[1:], variables, data))
            elif field.startswith("stdin:"):
                if field[6:].startswith("$"):
                    stdin.append(self.lookup(field[7:], variables, data))
                else:
                    stdin.append(field[6:])
            else:
                args.append(field)
        return (args, "\n".join(stdin))

    def run(self, topic, uuid, datetime, username, data):
        try:
            (args, stdin_s) = self.build_args(topic, uuid, datetime, username,
                                              data)
        except KeyError as exc:
            LOG.error("Unable to build the argument list: %s", exc)
            return

        LOG.debug("Running: %s", args)
        try:
            out = subprocess.check_output(args, stderr=subprocess.STDOUT,
                                          universal_newlines=True,
                                          input=stdin_s, timeout=self.timeout)
        except OSError as err:
            LOG.error("Unable to run %s (%s)", args, err)
        except subprocess.TimeoutExpired:
            LOG.error("Timeout when running %s", args)
        else:
            LOG.debug("=> %s", out)


class Worker(threading.Thread):
    def __init__(self, matchers):
        super().__init__()
        self.matchers = matchers

    def run(self):
        self.context = zmq.Context.instance()
        self.sock = self.context.socket(zmq.DEALER)
        self.sock.connect("inproc://workers")
        while True:
            msg = self.sock.recv_multipart()
            matcher_index = int(msg[0])
            (topic, uuid, dt, username, data) = msg[1:]
            LOG.debug("Running matcher num %d on %s", matcher_index, self.name)
            self.matchers[matcher_index].run(topic, uuid, dt, username, data)


class Reactor(multiprocessing.Process):
    def __init__(self, options, outbound):
{'fraction_non_alphanumeric': 0.10662989773641274, 'fraction_numerical': 0.0032172813972193496, 'mean_word_length': 2.096406972607613, 'pattern_counts': {'":': 10, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1834199', 'n_tokens_mistral': 2411, 'n_tokens_neox': 2387, 'n_words': 756}