<commit_before># Copyright 2014 tsuru-unit-agent authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.

import json
import logging
import Queue
import re
import os
import threading

from datetime import datetime
from logging.handlers import SysLogHandler
from socket import SOCK_DGRAM, SOCK_STREAM, gethostname

import requests


def extract_message(msg):
    # 2012-11-06 18:30:10 [13887] [INFO]
    regex = "\d+\-\d+\-\d+ \d+\:\d+\:\d+ \[\d+\] \[\w+\] "
    msgs = re.split(regex, msg)
    return [m for m in msgs if m]


class Stream(object):

    def __init__(self, **kwargs):
        self._buffer = ""
        self._max_buffer_size = kwargs.get("max_buffer_size", 10240)
        self.watcher_name = kwargs.get("watcher_name", "")
        self.timeout = kwargs.get("timeout", 2)
        self.echo = kwargs.get("echo_output")
        self.default_stream_name = kwargs.get("default_stream_name", "stdout")
        self.hostname = gethostname()
        self.start_writer()

    def start_writer(self):
        _, _, token, _, _, _, _ = self._load_envs()
        session = requests.Session()
        if token:
            session.headers.update({"Authorization": "bearer " + token})
        maxsize = int(os.environ.get("LOG_MAX_QUEUE_SIZE", 1000))
        self.queue = Queue.Queue(maxsize=maxsize)
        self.writer = TsuruLogWriter(session, self.queue)
        self.writer.start()

    def write(self, message):
        self({'data': message})
        if self.echo:
            self.echo.write(message)

    def flush(self):
        self._flush()
        if self.echo:
            self.echo.flush()

    def close(self):
        pass

    def __call__(self, data):
        (appname, host, token, syslog_server, syslog_port,
         syslog_facility, syslog_socket) = self._load_envs()
        messages = self._get_messages(data["data"])
        stream_name = data.get('name', self.default_stream_name)
        if appname and host and token:
            self._log_tsuru_api(messages, appname, host, token)
        if syslog_server and syslog_port and syslog_facility:
            self._log_syslog(messages, appname, syslog_server, syslog_port,
                             syslog_facility, syslog_socket, stream_name)

    def _log_tsuru_api(self, messages, appname, host, token):
        url = "{0}/apps/{1}/log?source={2}&unit={3}".format(host, appname,
                                                            self.watcher_name,
                                                            self.hostname)
        self.queue.put_nowait(LogEntry(url, self.timeout, messages))

    def _log_syslog(self, messages, appname, host, port, facility, socket, stream_name):
        if socket == 'tcp':
            socket_type = SOCK_STREAM
        else:
            socket_type = SOCK_DGRAM
        try:
            date_time = datetime.now().strftime("%b %d %H:%M:%S")\
                                      .lstrip("0").replace(" 0", "  ")
            logger = logging.getLogger(appname)
            logger.handlers = []
            logger.setLevel(logging.INFO)
            syslog = SysLogHandler(address=(host, int(port)),
                                   facility=facility, socktype=socket_type)
            formatter = logging.Formatter('{0} {1} %(name)s:\
                                           %(message)s'.format(date_time,<commit_msg>stream: small changes in imports<commit_after># Copyright 2014 tsuru-unit-agent authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.

import json
import logging
import Queue
import re
import socket
import os
import threading

from datetime import datetime
from logging import handlers
from socket import gethostname

import requests


def extract_message(msg):
    # 2012-11-06 18:30:10 [13887] [INFO]
    regex = "\d+\-\d+\-\d+ \d+\:\d+\:\d+ \[\d+\] \[\w+\] "
    msgs = re.split(regex, msg)
    return [m for m in msgs if m]


class Stream(object):

    def __init__(self, **kwargs):
        self._buffer = ""
        self._max_buffer_size = kwargs.get("max_buffer_size", 10240)
        self.watcher_name = kwargs.get("watcher_name", "")
        self.timeout = kwargs.get("timeout", 2)
        self.echo = kwargs.get("echo_output")
        self.default_stream_name = kwargs.get("default_stream_name", "stdout")
        self.hostname = gethostname()
        self.start_writer()

    def start_writer(self):
        _, _, token, _, _, _, _ = self._load_envs()
        session = requests.Session()
        if token:
            session.headers.update({"Authorization": "bearer " + token})
        maxsize = int(os.environ.get("LOG_MAX_QUEUE_SIZE", 1000))
        self.queue = Queue.Queue(maxsize=maxsize)
        self.writer = TsuruLogWriter(session, self.queue)
        self.writer.start()

    def write(self, message):
        self({'data': message})
        if self.echo:
            self.echo.write(message)

    def flush(self):
        self._flush()
        if self.echo:
            self.echo.flush()

    def close(self):
        pass

    def __call__(self, data):
        (appname, host, token, syslog_server, syslog_port,
         syslog_facility, syslog_socket) = self._load_envs()
        messages = self._get_messages(data["data"])
        stream_name = data.get('name', self.default_stream_name)
        if appname and host and token:
            self._log_tsuru_api(messages, appname, host, token)
        if syslog_server and syslog_port and syslog_facility:
            self._log_syslog(messages, appname, syslog_server, syslog_port,
                             syslog_facility, syslog_socket, stream_name)

    def _log_tsuru_api(self, messages, appname, host, token):
        url = "{0}/apps/{1}/log?source={2}&unit={3}".format(host, appname,
                                                            self.watcher_name,
                                                            self.hostname)
        self.queue.put_nowait(LogEntry(url, self.timeout, messages))

    def _log_syslog(self, messages, appname, host, port, facility, socktype, stream_name):
        if socktype == 'tcp':
            socket_type = socket.SOCK_STREAM
        else:
            socket_type = socket.SOCK_DGRAM
        try:
            date_time = datetime.now().strftime("%b %d %H:%M:%S")\
                                      .lstrip("0").replace(" 0", "  ")
            logger = logging.getLogger(appname)
            logger.handlers = []
            logger.setLevel(logging.INFO)
            syslog = handlers.SysLogHandler(address=(host, int(port)),
                                            facility=facility, socktype=socket_type)
            formatter = logging.Formatter('{0} {1} %(name)s:\
                                           %(message)s'.format(date_time,
{'fraction_non_alphanumeric': 0.11784365010171462, 'fraction_numerical': 0.011915140947399011, 'mean_word_length': 2.134335154826958, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 17, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2458430', 'n_tokens_mistral': 2071, 'n_tokens_neox': 2007, 'n_words': 457}