<commit_before>from __future__ import unicode_literals, division, absolute_import
from builtins import *  # noqa pylint: disable=unused-import, redefined-builtin

import logging
import random
import string
import sys
import threading

import rpyc

from flexget import terminal
from flexget.logger import capture_output
from flexget.terminal import console
from flexget.options import get_parser
from rpyc.utils.server import ThreadedServer
from terminaltables.terminal_io import terminal_size

log = logging.getLogger('ipc')

# Allow some attributes from dict interface to be called over the wire
rpyc.core.protocol.DEFAULT_CONFIG['safe_attrs'].update(['items'])
rpyc.core.protocol.DEFAULT_CONFIG['allow_pickle'] = True

IPC_VERSION = 4
AUTH_ERROR = b'authentication error'
AUTH_SUCCESS = b'authentication success'


class RemoteStream(object):
    """
    Used as a filelike to stream text to remote client. If client disconnects while this is in use, an error will be
    logged, but no exception raised.
    """

    def __init__(self, writer):
        """
        :param writer: A function which writes a line of text to remote client.
        """
        self.buffer = None
        self.writer = writer

    def write(self, data):
        # This relies on all data up to a newline being either unicode or str, not mixed
        if not self.buffer:
            self.buffer = data
        else:
            self.buffer += data
        newline = '\n' if isinstance(self.buffer, str) else b'\n'
        if newline in self.buffer:
            self.flush()

    def flush(self):
        if self.buffer is None or self.writer is None:
            return
        try:
            self.writer(self.buffer, end='')
        except EOFError:
            self.writer = None
            log.error('Client ended connection while still streaming output.')
        finally:
            self.buffer = None

<commit_msg>Simplify RemoteStream to only handle text (not bytes) [fix] Errors with some colorized non-ascii text being sent from daemon on py27 fix #1553<commit_after>from __future__ import unicode_literals, division, absolute_import
from builtins import *  # noqa pylint: disable=unused-import, redefined-builtin

import logging
import random
import string
import sys
import threading

import rpyc
from rpyc.utils.server import ThreadedServer
from terminaltables.terminal_io import terminal_size

from flexget import terminal
from flexget.logger import capture_output
from flexget.terminal import console
from flexget.options import get_parser

log = logging.getLogger('ipc')

# Allow some attributes from dict interface to be called over the wire
rpyc.core.protocol.DEFAULT_CONFIG['safe_attrs'].update(['items'])
rpyc.core.protocol.DEFAULT_CONFIG['allow_pickle'] = True

IPC_VERSION = 4
AUTH_ERROR = b'authentication error'
AUTH_SUCCESS = b'authentication success'


class RemoteStream(object):
    """
    Used as a filelike to stream text to remote client. If client disconnects while this is in use, an error will be
    logged, but no exception raised.
    """

    def __init__(self, writer):
        """
        :param writer: A function which writes a line of text to remote client.
        """
        self.buffer = ''
        self.writer = writer

    def write(self, text):
        self.buffer += text
        if '\n' in self.buffer:
            self.flush()

    def flush(self):
        if self.buffer is None or self.writer is None:
            return
        try:
            self.writer(self.buffer, end='')
        except EOFError:
            self.writer = None
            log.error('Client ended connection while still streaming output.')
        finally:
            self.buffer = ''


{'fraction_non_alphanumeric': 0.0748244192328471, 'fraction_numerical': 0.0021609940572663426, 'mean_word_length': 2.973175965665236, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7269401', 'n_tokens_mistral': 1011, 'n_tokens_neox': 989, 'n_words': 371}