<commit_before>'''
clip.py: Embeddable, composable [c]ommand [l]ine [i]nterface [p]arsing

Copyright: (c) 2015 <NAME>
License: MIT, see LICENSE for more details
'''
import itertools
import shlex
import sys
import uuid


########################################
# COMPATIBILITY & UTILITIES
########################################

PY2 = sys.version_info[0] == 2

input = raw_input if PY2 else input
text_type = basestring if PY2 else str
def is_func(e):
	return hasattr(e, '__call__')
def iteritems(d):
	return d.iteritems() if PY2 else d.items()

def get_input_fn(f=None, invisible=False):
	if f is not None:
		return f
	if not invisible:
		return input
	import getpass
	return getpass.getpass

def prompt_fn(f, s, default=None, type=None, skip=False, repeat=False):
	default = default or ''
	while True:
		try:
			ret = f(s) or (default() if is_func(default) else default)
			if skip and not ret:
				return None
			ret = type(ret) if type is not None else ret
		except (KeyboardInterrupt, EOFError):
			raise_abort()
		except ValueError:
			echo('Please provide an {}'.format(type.__name__))
		else:
			if ret or not repeat:
				return ret

def determine_type(t, default):
	if t is None:
		if default is not None:
			if isinstance(default, list):
				return type(default[0]) if len(default) > 0 else None
			return None if is_func(default) else type(default)
		return None
	return t


########################################
# GLOBAL CLASSES/METHODS
########################################

class ClipGlobals(object):

	def __init__(self):
		self._streams = {}

	def _write(self, message, stream, nl=True):
		stream.write(str(message) + ("\n" if nl else ""))
		# Custom streams may not implement a flush() method
		if hasattr(stream, 'flush'):
			stream.flush()

	def _broadcast(self, message, err=False, nl=True):
		for k, v in iteritems(self._streams):
			self._write(message, v['err' if err else 'out'], nl)

	def echo(self, message, err=False, nl=True, app=None):
		if not self._streams:
			raise AttributeError('No streams have been initialized')
		if app is None:
			self._broadcast(message, err, nl)
		else:
			self._write(message, self._streams[app]['err' if err else 'out'], nl)

	def add_streams(self, out, err, app=None):
		self._streams[app] = {
			'out': out or sys.stdout,
			'err': err or sys.stderr
		}


class ClipExit(Exception):<commit_msg>moved echo from str() to unicode() so it doesn't break on unicode messages<commit_after>'''
clip.py: Embeddable, composable [c]ommand [l]ine [i]nterface [p]arsing

Copyright: (c) 2015 <NAME>
License: MIT, see LICENSE for more details
'''
import itertools
import shlex
import sys
import uuid


########################################
# COMPATIBILITY & UTILITIES
########################################

PY2 = sys.version_info[0] == 2

input = raw_input if PY2 else input
text_type = basestring if PY2 else str
def is_func(e):
	return hasattr(e, '__call__')
def iteritems(d):
	return d.iteritems() if PY2 else d.items()

def get_input_fn(f=None, invisible=False):
	if f is not None:
		return f
	if not invisible:
		return input
	import getpass
	return getpass.getpass

def prompt_fn(f, s, default=None, type=None, skip=False, repeat=False):
	default = default or ''
	while True:
		try:
			ret = f(s) or (default() if is_func(default) else default)
			if skip and not ret:
				return None
			ret = type(ret) if type is not None else ret
		except (KeyboardInterrupt, EOFError):
			raise_abort()
		except ValueError:
			echo('Please provide an {}'.format(type.__name__))
		else:
			if ret or not repeat:
				return ret

def determine_type(t, default):
	if t is None:
		if default is not None:
			if isinstance(default, list):
				return type(default[0]) if len(default) > 0 else None
			return None if is_func(default) else type(default)
		return None
	return t


########################################
# GLOBAL CLASSES/METHODS
########################################

class ClipGlobals(object):

	def __init__(self):
		self._streams = {}

	def _write(self, message, stream, nl=True):
		stream.write(unicode(message) + ("\n" if nl else ""))
		# Custom streams may not implement a flush() method
		if hasattr(stream, 'flush'):
			stream.flush()

	def _broadcast(self, message, err=False, nl=True):
		for k, v in iteritems(self._streams):
			self._write(message, v['err' if err else 'out'], nl)

	def echo(self, message, err=False, nl=True, app=None):
		if not self._streams:
			raise AttributeError('No streams have been initialized')
		if app is None:
			self._broadcast(message, err, nl)
		else:
			self._write(message, self._streams[app]['err' if err else 'out'], nl)

	def add_streams(self, out, err, app=None):
		self._streams[app] = {
			'out': out or sys.stdout,
			'err': err or sys.stderr
		}


class ClipExit(Exception):
{'fraction_non_alphanumeric': 0.17755059461715, 'fraction_numerical': 0.005007302315877321, 'mean_word_length': 4.542196531791907, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1602725', 'n_tokens_mistral': 1725, 'n_tokens_neox': 1588, 'n_words': 521}