<commit_before>import argparse, ssl, socket, sys, glob, re, os, json, time
from datetime import datetime
from dateutil import tz

TZLOCAL = tz.tzlocal()

class Syslog:
    def __init__(self, options):
        self.options = options
        self.sock = None
        self.connect()
        self.max_length = options.max_length

    def connect(self):
        if not self.options.host:
            return

        if self.sock is not None:
            self.sock.close()

        self.sock = ssl.wrap_socket(socket.socket(),
                                    keyfile=self.options.keyfile,
                                    certfile=self.options.certfile,
                                    ca_certs=self.options.ca_certs,
                                    cert_reqs=ssl.CERT_NONE if self.options.insecure else ssl.CERT_REQUIRED)
        self.sock.connect((self.options.host, self.options.port))

    def send(self, msg, timestamp, host=socket.getfqdn(), app_name="-", procid="-", msgid="-", structured_data="-"):
        payload = " ".join(["<1>1", timestamp.isoformat(), host, app_name, procid, msgid, structured_data, msg])
        if self.max_length > 0:
            payload = payload[:self.max_length]
        if self.sock is not None:
            self.sock.send(str(len(payload)) + " " + payload)
        else:
            print payload


def parse_options():
    parser = argparse.ArgumentParser(description="Tail log files and send to a syslog server")
    parser.add_argument("--host", help='hostname of syslog server')
    parser.add_argument("--port", help='port of syslog server', type=int)
    parser.add_argument("--ca-certs", help='certificate of CA to trust')
    parser.add_argument("--certfile", help='client certificate to identify as')
    parser.add_argument("--keyfile", help='key corresponding to the client certificate')
    parser.add_argument("--insecure", help='disable server certificate validation', action='store_true')
    parser.add_argument("--verbose", "-v", action='store_true')
    parser.add_argument("--statefile", help='file to save and load current log reading positions in')
    parser.add_argument("--path-regex", type=re.compile, default=r".*/(?P<app_name>[^/]+)/\d+/stdio\.(?P<date>\d\d\d\d-\d\d-\d\d)\.log")
    parser.add_argument("--line-regex", type=re.compile, default=r"(?P<time>\d\d:\d\d:\d\d\.\d\d\d) (?P<procid>[^ *]+): (?P<msg>.*)")
    parser.add_argument("--date-format", default="%Y-%m-%d")
    parser.add_argument("--time-format", default="%H:%M:%S.%f")
    parser.add_argument("--interval", "-i", default=0.0, type=float, help='interval in seconds to repeat at')
    parser.add_argument("--reset", action="store_true", help="reset state to end of all files")
    parser.add_argument("--max-length", default=8192, type=int, help='maximum message length (will truncate)')<commit_msg>Send unqualified hostnames to match other syslog behaviour<commit_after>import argparse, ssl, socket, sys, glob, re, os, json, time
from datetime import datetime
from dateutil import tz

TZLOCAL = tz.tzlocal()

class Syslog:
    def __init__(self, options):
        self.options = options
        self.sock = None
        self.connect()
        self.max_length = options.max_length

    def connect(self):
        if not self.options.host:
            return

        if self.sock is not None:
            self.sock.close()

        self.sock = ssl.wrap_socket(socket.socket(),
                                    keyfile=self.options.keyfile,
                                    certfile=self.options.certfile,
                                    ca_certs=self.options.ca_certs,
                                    cert_reqs=ssl.CERT_NONE if self.options.insecure else ssl.CERT_REQUIRED)
        self.sock.connect((self.options.host, self.options.port))

    def send(self, msg, timestamp, host=socket.getfqdn().split('.')[0], app_name="-", procid="-", msgid="-", structured_data="-"):
        payload = " ".join(["<1>1", timestamp.isoformat(), host, app_name, procid, msgid, structured_data, msg])
        if self.max_length > 0:
            payload = payload[:self.max_length]
        if self.sock is not None:
            self.sock.send(str(len(payload)) + " " + payload)
        else:
            print payload


def parse_options():
    parser = argparse.ArgumentParser(description="Tail log files and send to a syslog server")
    parser.add_argument("--host", help='hostname of syslog server')
    parser.add_argument("--port", help='port of syslog server', type=int)
    parser.add_argument("--ca-certs", help='certificate of CA to trust')
    parser.add_argument("--certfile", help='client certificate to identify as')
    parser.add_argument("--keyfile", help='key corresponding to the client certificate')
    parser.add_argument("--insecure", help='disable server certificate validation', action='store_true')
    parser.add_argument("--verbose", "-v", action='store_true')
    parser.add_argument("--statefile", help='file to save and load current log reading positions in')
    parser.add_argument("--path-regex", type=re.compile, default=r".*/(?P<app_name>[^/]+)/\d+/stdio\.(?P<date>\d\d\d\d-\d\d-\d\d)\.log")
    parser.add_argument("--line-regex", type=re.compile, default=r"(?P<time>\d\d:\d\d:\d\d\.\d\d\d) (?P<procid>[^ *]+): (?P<msg>.*)")
    parser.add_argument("--date-format", default="%Y-%m-%d")
    parser.add_argument("--time-format", default="%H:%M:%S.%f")
    parser.add_argument("--interval", "-i", default=0.0, type=float, help='interval in seconds to repeat at')
    parser.add_argument("--reset", action="store_true", help="reset state to end of all files")
    parser.add_argument("--max-length", default=8192, type=int, help='maximum message length (will truncate)')
{'fraction_non_alphanumeric': 0.15949851993731498, 'fraction_numerical': 0.003308375413546927, 'mean_word_length': 2.9504814305364513, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6924517', 'n_tokens_mistral': 1786, 'n_tokens_neox': 1738, 'n_words': 386}