<commit_before># Copyright 2014 <NAME>
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import datetime
import logging
import re
import sys

import eventlet
import eventlet.tpool


LOG = logging.getLogger(__name__)


class BaseInput():
    handler = None
    should_run = False
    thread = None

    def set_handler(self, handler):
        self.handler = handler

    def start(self):
        self.should_run = True
        if not self.thread:
            self.thread = eventlet.spawn(self._run)

    def stop(self):
        self.should_run = False
        thread = self.thread
        self.thread = None
        thread.kill()

    def _run(self):
        raise NotImplementedError


class Stdin(BaseInput):
    """Reads messages from stdin

    Messages are separated by newlines. Whitespace at the end of a message
    will get stripped.

    Example for ``input.yml``:

    .. code:: yaml

        - stdin: {}
    """
    def _run(self):
        while self.should_run:
            line = eventlet.tpool.execute(sys.stdin.readline)
            self.handler({"message": line.rstrip()})


SYSLOG_PRIORITIES = ['emergency', 'alert', 'critical', 'error', 'warning',
                     'notice', 'informational', 'debug']
SYSLOG_FACILITIES = ([
    'kern', 'user', 'mail', 'daemon', 'auth', 'syslog', 'lpr', 'news',
    'uucp', 'cron', 'authpriv', 'ftp', 'ntp', 'audit', 'alert', 'local', ]
    + ['local%i' % i for i in range(8)]
    + ['unknown%02i' % i for i in range(12)]
)


class Syslog(BaseInput):
    """Reads messages from syslog

    Listens for syslog messages on a TCP socket. Note that if you want to
    bind the (default) syslog port, you'll need to run logshipper as root.

    Sets the ``facility`` and ``severity`` as defined by rfc3164.

    ``bind``
        Ip address or hostname to bind to. Defaults to ``127.0.0.1``
    ``port``
        Port to bind to. Defaults to ``514``
    ``protocol``
        If set to ``rfc5424``, RFC-5424 matching will be tried,
        and non-compliant messages will silently be dropped.
        If set to ``rfc3164``, the much simpler RFC-3164 will be used.
        If set to ``auto``, RFC-5424 messages will be detected and disected,
        while non-RFC-5424 messages will be processed as RFC-3164.
    ``

    Example for ``input.yml``:

    .. code:: yaml

        - syslog:
            bind: 127.0.0.1
            port: 1514
    """

    class rfc5424_tz(datetime.tzinfo):
        def __init__(self, offset):
            self.offset = offset

        def utcoffset(self):
            return self.offset

        def dst(self, dt):
            return datetime.timedelta(0)

        def tzname(self, dt):
            return "unnamed"

    rfc3164_matcher = re.compile(r'<(?P<prival>\d{1,3})>')

    rfc5424_matcher = re.compile(r"""
        <(?P<prival>\d{1,3})>1
        \s
        (?P<timestamp>-|\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d(\.\d+)?(Z|[+-]\d\d:\d\d))
        \s
        (?P<hostname>-|\S{1,255})
        \s
        (?P<appname>-|\S{1,48})
        \s
        (?P<procid>-|\S{1,128})
        \s
        (?P<msgid>-|\S{1,32})
        \s
        (?P<sd>-|\[[^\]]+\])
        \s*
        """, re.X)  # <134>

    def __init__(self, bind="127.0.0.1", port=514, protocol='auto'):
        self.bind = bind
        self.port = int(port)

        if protocol == 'rfc5424':
            self.regexes = [Syslog.rfc5424_matcher]
        elif protocol == 'rfc3164':
            self.regexes = [Syslog.rfc3164_matcher]
        elif protocol == 'auto':
            self.regexes = [Syslog.rfc5424_matcher, Syslog.rfc3164_matcher]
        else:
            raise Exception('protocol must be either rfc3164, rfc5424 or auto')

    def _run(self):
        self.server = eventlet.listen((self.bind, self.port))
        eventlet.serve(self.server, self.handle)

    def handle(self, socket, address):
        fileobj = socket.makefile('r', 4096)

        for line in fileobj:
            self.process_message(line)

    def process_message(self, line):
        line = line.rstrip('\r\n').decode('utf-8')

        for r in self.regexes:
            match = r.match(line)
            if not match:
                continue

            message = match.groupdict()

            prival = int(message.pop('prival'))
            if prival <= 255:
                message['facility'] = SYSLOG_FACILITIES[prival / 8]
                message['severity'] = SYSLOG_PRIORITIES[prival % 8]

            structured_data = message.pop('sd', '-')
            if structured_data != '-':
                # TODO(KvdV): Parse structured data<commit_msg>make syslog python3 compatble<commit_after># Copyright 2014 <NAME>
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import datetime
import logging
import re
import sys

import eventlet
import eventlet.tpool


LOG = logging.getLogger(__name__)


class BaseInput():
    handler = None
    should_run = False
    thread = None

    def set_handler(self, handler):
        self.handler = handler

    def start(self):
        self.should_run = True
        if not self.thread:
            self.thread = eventlet.spawn(self._run)

    def stop(self):
        self.should_run = False
        thread = self.thread
        self.thread = None
        thread.kill()

    def _run(self):
        raise NotImplementedError


class Stdin(BaseInput):
    """Reads messages from stdin

    Messages are separated by newlines. Whitespace at the end of a message
    will get stripped.

    Example for ``input.yml``:

    .. code:: yaml

        - stdin: {}
    """
    def _run(self):
        while self.should_run:
            line = eventlet.tpool.execute(sys.stdin.readline)
            self.handler({"message": line.rstrip()})


SYSLOG_PRIORITIES = ['emergency', 'alert', 'critical', 'error', 'warning',
                     'notice', 'informational', 'debug']
SYSLOG_FACILITIES = ([
    'kern', 'user', 'mail', 'daemon', 'auth', 'syslog', 'lpr', 'news',
    'uucp', 'cron', 'authpriv', 'ftp', 'ntp', 'audit', 'alert', 'local', ]
    + ['local%i' % i for i in range(8)]
    + ['unknown%02i' % i for i in range(12)]
)


class Syslog(BaseInput):
    """Reads messages from syslog

    Listens for syslog messages on a TCP socket. Note that if you want to
    bind the (default) syslog port, you'll need to run logshipper as root.

    Sets the ``facility`` and ``severity`` as defined by rfc3164.

    ``bind``
        Ip address or hostname to bind to. Defaults to ``127.0.0.1``
    ``port``
        Port to bind to. Defaults to ``514``
    ``protocol``
        If set to ``rfc5424``, RFC-5424 matching will be tried,
        and non-compliant messages will silently be dropped.
        If set to ``rfc3164``, the much simpler RFC-3164 will be used.
        If set to ``auto``, RFC-5424 messages will be detected and disected,
        while non-RFC-5424 messages will be processed as RFC-3164.
    ``

    Example for ``input.yml``:

    .. code:: yaml

        - syslog:
            bind: 127.0.0.1
            port: 1514
    """

    class rfc5424_tz(datetime.tzinfo):
        def __init__(self, offset):
            self.offset = offset

        def utcoffset(self):
            return self.offset

        def dst(self, dt):
            return datetime.timedelta(0)

        def tzname(self, dt):
            return "unnamed"

    rfc3164_matcher = re.compile(r'<(?P<prival>\d{1,3})>')

    rfc5424_matcher = re.compile(r"""
        <(?P<prival>\d{1,3})>1
        \s
        (?P<timestamp>-|\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d(\.\d+)?(Z|[+-]\d\d:\d\d))
        \s
        (?P<hostname>-|\S{1,255})
        \s
        (?P<appname>-|\S{1,48})
        \s
        (?P<procid>-|\S{1,128})
        \s
        (?P<msgid>-|\S{1,32})
        \s
        (?P<sd>-|\[[^\]]+\])
        \s*
        """, re.X)  # <134>

    def __init__(self, bind="127.0.0.1", port=514, protocol='auto'):
        self.bind = bind
        self.port = int(port)

        if protocol == 'rfc5424':
            self.regexes = [Syslog.rfc5424_matcher]
        elif protocol == 'rfc3164':
            self.regexes = [Syslog.rfc3164_matcher]
        elif protocol == 'auto':
            self.regexes = [Syslog.rfc5424_matcher, Syslog.rfc3164_matcher]
        else:
            raise Exception('protocol must be either rfc3164, rfc5424 or auto')

    def _run(self):
        self.server = eventlet.listen((self.bind, self.port))
        eventlet.serve(self.server, self.handle)

    def handle(self, socket, address):
        fileobj = socket.makefile('r', 4096, encoding='utf-8')

        for line in fileobj:
            self.process_message(line)

    def process_message(self, line):
        line = line.rstrip('\r\n')

        for r in self.regexes:
            match = r.match(line)
            if not match:
                continue

            message = match.groupdict()

            prival = int(message.pop('prival'))
            if prival <= 255:
                message['facility'] = SYSLOG_FACILITIES[prival // 8]
                message['severity'] = SYSLOG_PRIORITIES[prival % 8]

            structured_data = message.pop('sd', '-')
            if structured_data != '-':
                # TODO(KvdV): Parse structured data
{'fraction_non_alphanumeric': 0.1330735509011203, 'fraction_numerical': 0.029322942036044814, 'mean_word_length': 2.159741458910434, 'pattern_counts': {'":': 2, '<': 29, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 6, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3905759', 'n_tokens_mistral': 3531, 'n_tokens_neox': 3234, 'n_words': 913}