<commit_before>"""Module to handle streams of text"""

import os
import sys
from StringIO import StringIO

from dotsite.platforms import get_clipboard_data


def argvs(clipboard_arg=None):
    """Yield streams of text given to a program

    If there are files in sys.argv, use them
    Otherwise if a clipboard arg was provided use a string of clipboard
    Otherwise sys.stdin
    """
    args = sys.argv[1:]
    use_clipboard = clipboard_arg in args + [True]
    if not args:
        yield sys.stdin
    else:
        files = False
        for arg in args:
            if os.path.isfile(arg):
                files = True
                yield open(arg)
        if not files and use_clipboard:
            yield StringIO(get_clipboard_data())


def first_argv(clipboard_arg=None):
    try:
        arg_stream = argvs(clipboard_arg)
        return arg_stream.next()
    except StopIteration:
        return ''


def full_lines(stream):
    for line in stream.readlines():
        stripped = line.rstrip()
        if stripped:
            yield stripped<commit_msg>Prefer ContextManagers<commit_after>"""Module to handle streams of text"""

import os
import sys
import contextlib
from io import StringIO

from dotsite.platforms import get_clipboard_data


def clipboard_stream(name=None):
    stream = StringIO(get_clipboard_data())
    stream.name = name or '<clipboard>'
    return stream


@contextlib.contextmanager
def clipin():
    yield clipboard_stream()


@contextlib.contextmanager
def files():
    """Yield streams of text given to a program

    If there are files in sys.argv, yield them
    yield sys.stdin
    """
    for arg in sys.argv[1:]:
        if os.path.isfile(arg):
            yield open(arg)


@contextlib.contextmanager
def all():
    with files() as streams:
        yield streams
    yield clipboard_stream()
    yield sys.stdin


def first_file():
    try:
        arg_stream = files()
        first, *_rest = arg_stream
        return first
    except StopIteration:
        return ''


def full_lines(stream):
    for line in stream.readlines():
        stripped = line.rstrip()
        if stripped:
            yield stripped
{'fraction_non_alphanumeric': 0.07874015748031496, 'fraction_numerical': 0.0009263547938860583, 'mean_word_length': 2.5702479338842976, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2936751', 'n_tokens_mistral': 633, 'n_tokens_neox': 617, 'n_words': 206}