<commit_before>from __future__ import with_statement

import sys
import time
from select import select

from fabric.context_managers import settings, char_buffered
from fabric.state import env, output, win32, io_sleep
from fabric.auth import get_password, set_password
import fabric.network

if win32:
    import msvcrt


def _flush(pipe, text):
    pipe.write(text)
    pipe.flush()


def _endswith(char_list, substring):
    tail = char_list[-1 * len(substring):]
    substring = list(substring)
    return tail == substring


def output_loop(chan, which, capture):
    # Obtain stdout or stderr related values
    func = getattr(chan, which)
    if which == 'recv':
        prefix = "out"
        pipe = sys.stdout
    else:
        prefix = "err"
        pipe = sys.stderr
    printing = getattr(output, 'stdout' if (which == 'recv') else 'stderr')
    # Initialize loop variables
    reprompt = False
    initial_prefix_printed = False
    while True:
        # Handle actual read/write
        byte = func(1)
        # Empty byte == EOS
        if byte == '':
            break
        # A None capture variable implies that we're in open_shell()
        if capture is None:
            # Just print directly -- no prefixes, no capturing, nada
            # And since we know we're using a pty in this mode, just go
            # straight to stdout.
            _flush(sys.stdout, byte)
        # Otherwise, we're in run/sudo and need to handle capturing and
        # prompts.
        else:
            _prefix = "[%s] %s: " % (env.host_string, prefix)
            # Allow prefix to be turned off.
            if not env.output_prefix:
                _prefix = ""
            # Print to user
            if printing:
                # Initial prefix
                if not initial_prefix_printed:
                    _flush(pipe, _prefix)
                    initial_prefix_printed = True
                # Byte itself
                _flush(pipe, byte)
                # Trailing prefix to start off next line
                if byte in ("\n", "\r"):
                    _flush(pipe, _prefix)
            # Store in capture buffer
            capture += byte
            # Handle prompts
            prompt = _endswith(capture, env.sudo_prompt)
            try_again = (_endswith(capture, env.again_prompt + '\n')
                or _endswith(capture, env.again_prompt + '\r\n'))
            if prompt:
                # Obtain cached password, if any
                password = get_password()
                # Remove the prompt itself from the capture buffer. This is
                # backwards compatible with Fabric 0.9.x behavior; the user
                # will still see the prompt on their screen (no way to avoid
                # this) but at least it won't clutter up the captured text.
                del capture[-1 * len(env.sudo_prompt):]
                # If the password we just tried was bad, prompt the user again.
                if (not password) or reprompt:
                    # Print the prompt and/or the "try again" notice if
                    # output is being hidden. In other words, since we need
                    # the user's input, they need to see why we're
                    # prompting them.
                    if not printing:<commit_msg>WIP re: linewise output<commit_after>from __future__ import with_statement

import sys
import time
from select import select

from fabric.context_managers import settings, char_buffered
from fabric.state import env, output, win32, io_sleep
from fabric.auth import get_password, set_password
import fabric.network

if win32:
    import msvcrt


def _flush(pipe, text):
    pipe.write(text)
    pipe.flush()


def _endswith(char_list, substring):
    tail = char_list[-1 * len(substring):]
    substring = list(substring)
    return tail == substring


def output_loop(chan, which, capture):
    # Obtain stdout or stderr related values
    func = getattr(chan, which)
    if which == 'recv':
        prefix = "out"
        pipe = sys.stdout
    else:
        prefix = "err"
        pipe = sys.stderr
    printing = getattr(output, 'stdout' if (which == 'recv') else 'stderr')
    # Initialize loop variables
    reprompt = False
    initial_prefix_printed = False
    linewise = True
    line = []
    while True:
        # Handle actual read/write
        byte = func(1)
        # Empty byte == EOS
        if byte == '':
            break
        # A None capture variable implies that we're in open_shell()
        if capture is None:
            # Just print directly -- no prefixes, no capturing, nada
            # And since we know we're using a pty in this mode, just go
            # straight to stdout.
            _flush(sys.stdout, byte)
        # Otherwise, we're in run/sudo and need to handle capturing and
        # prompts.
        else:
            _prefix = "[%s] %s: " % (env.host_string, prefix)
            # Allow prefix to be turned off.
            if not env.output_prefix:
                _prefix = ""
            # Print to user
            if printing:
                # Initial prefix
                if not linewise and not initial_prefix_printed:
                    _flush(pipe, _prefix)
                    initial_prefix_printed = True
                is_newline = byte in ("\n", "\r")
                # Byte itself
                if linewise:
                    line += byte
                    if is_newline:
                        _flush(pipe, _prefix)
                        _flush(pipe, "".join(line))
                        line = []
                else:
                    _flush(pipe, byte)
                    # Trailing prefix to start off next line
                    if is_newline:
                        _flush(pipe, _prefix)
            # Store in capture buffer
            capture += byte
            # Handle prompts
            prompt = _endswith(capture, env.sudo_prompt)
            try_again = (_endswith(capture, env.again_prompt + '\n')
                or _endswith(capture, env.again_prompt + '\r\n'))
            if prompt:
                # Obtain cached password, if any
                password = <PASSWORD>()
                # Remove the prompt itself from the capture buffer. This is
                # backwards compatible with Fabric 0.9.x behavior; the user
                # will still see the prompt on their screen (no way to avoid
                # this) but at least it won't clutter up the captured text.
                del capture[-1 * len(env.sudo_prompt):]
                # If the password we just tried was bad, prompt the user again.
                if (not password) or reprompt:
                    # Print the prompt and/or the "try again" notice if
                    # output is being hidden. In other words, since we need
                    # the user's input, they need to see why we're
                    # prompting them.
                    if not printing:
{'fraction_non_alphanumeric': 0.07410071942446043, 'fraction_numerical': 0.002589928057553957, 'mean_word_length': 1.625, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6044743', 'n_tokens_mistral': 1865, 'n_tokens_neox': 1741, 'n_words': 670}