<commit_before># encoding: utf-8

from __future__ import unicode_literals

"""Convienent utilities."""

# ## Imports

import sys

from codecs import iterencode
from inspect import isfunction, isclass
from operator import methodcaller
from collections import deque, namedtuple, Sized, Iterable
from pkg_resources import iter_entry_points
from xml.sax.saxutils import quoteattr

try:  # pragma: no cover
	from html.parser import HTMLParser
except ImportError:  # pragma: no cover
	from HTMLParser import HTMLParser


# ## Python Cross-Compatibility
# 
# These allow us to detect relevant version differences for code generation, and overcome some of the minor
# differences in labels between Python 2 and Python 3 compatible runtimes.
# 
# The differences, in practice, are minor, and are easily overcome through a small block of version-dependant
# code.  Handily, even built-in labels are not sacrosanct; they can be easily assigned to and re-mapped.
# 

try:  # Python 2
	from types import StringTypes as stringy
	
	try:
		from cStringIO import StringIO
	except:  # pragma: no cover
		from StringIO import StringIO  # This never really happens.  Still, nice to be defensive.
	
	bytes = str
	str = unicode
	py = 2
	reduce = reduce

except:  # Python 3
	from io import StringIO
	
	stringy = str
	bytes = bytes
	str = str
	py = 3

# There are some additional complications for the Pypy runtime.

try:
	from sys import pypy_version_info
	pypy = True
except ImportError:
	pypy = False

# ## Type Definitions


# A tuple representing a single step of fancy iteration.
Iteration = namedtuple('Iteration', ['first', 'last', 'index', 'total', 'value'])


# ## Simple Utility Functions

def stream(input, encoding=None, errors='strict'):
	"""Safely iterate a template generator, ignoring ``None`` values and optionally stream encoding.
	
	Used internally by ``cinje.flatten``, this allows for easy use of a template generator as a WSGI body.
	"""
	
	input = (i for i in input if i)  # Omits `None` (empty wrappers) and empty chunks.
	
	if encoding:  # Automatically, and iteratively, encode the text if requested.
		input = iterencode(input, encoding, errors=errors)
	
	return input


def flatten(input, file=None, encoding=None, errors='strict'):
	"""Return a flattened representation of a cinje chunk stream.
	
	This has several modes of operation.  If no `file` argument is given, output will be returned as a string.
	The type of string will be determined by the presence of an `encoding`; if one is given the returned value is a
	binary string, otherwise the native unicode representation.  If a `file` is present, chunks will be written
	iteratively through repeated calls to `file.write()`, and the amount of data (characters or bytes) written
	returned.  The type of string written will be determined by `encoding`, just as the return value is when not
	writing to a file-like object.  The `errors` argument is passed through when encoding.
	
	We can highly recommend using the various stremaing IO containers available in the
	[`io`](https://docs.python.org/3/library/io.html) module, though
	[`tempfile`](https://docs.python.org/3/library/tempfile.html) classes are also quite useful.
	"""
	
	input = stream(input, encoding, errors)
	
	if file is None:  # Exit early if we're not writing to a file.
		return b''.join(input) if encoding else ''.join(input)
	
	counter = 0
	
	for chunk in input:
		file.write(chunk)
		counter += len(chunk)
	
	return counter


def fragment(string, name="anonymous", **context):
	"""Translate a template fragment into a callable function.
	
	**Note:** Use of this function is discouraged everywhere except tests, as no caching is implemented at this time.
	
	Only one function may be declared, either manually, or automatically. If automatic defintition is chosen the
	resulting function takes no arguments.  Additional keyword arguments are passed through as global variables.
	"""
	
	if ": def" in string or ":def" in string:
		code = string.encode('utf8').decode('cinje')
		name = None
	else:
		code = ": def {name}\n\n{string}\n\n: end".format(
				name = name,
				string = string,
			).encode('utf8').decode('cinje')
	
	environ = dict(context)
	
	exec(code, environ)<commit_msg>Simplify.<commit_after># encoding: utf-8

from __future__ import unicode_literals

"""Convienent utilities."""

# ## Imports

import sys

from codecs import iterencode
from inspect import isfunction, isclass
from operator import methodcaller
from collections import deque, namedtuple, Sized, Iterable
from pkg_resources import iter_entry_points
from xml.sax.saxutils import quoteattr

try:  # pragma: no cover
	from html.parser import HTMLParser
except ImportError:  # pragma: no cover
	from HTMLParser import HTMLParser


# ## Python Cross-Compatibility
# 
# These allow us to detect relevant version differences for code generation, and overcome some of the minor
# differences in labels between Python 2 and Python 3 compatible runtimes.
# 
# The differences, in practice, are minor, and are easily overcome through a small block of version-dependant
# code.  Handily, even built-in labels are not sacrosanct; they can be easily assigned to and re-mapped.
# 

try:  # Python 2
	from types import StringTypes as stringy
	
	try:
		from cStringIO import StringIO
	except:  # pragma: no cover
		from StringIO import StringIO  # This never really happens.  Still, nice to be defensive.
	
	bytes = str
	str = unicode
	py = 2
	reduce = reduce

except:  # Python 3
	from io import StringIO
	
	stringy = str
	bytes = bytes
	str = str
	py = 3

# There are some additional complications for the Pypy runtime.

try:
	from sys import pypy_version_info
	pypy = True
except ImportError:
	pypy = False

# ## Type Definitions


# A tuple representing a single step of fancy iteration.
Iteration = namedtuple('Iteration', ['first', 'last', 'index', 'total', 'value'])


# ## Simple Utility Functions

def stream(input, encoding=None, errors='strict'):
	"""Safely iterate a template generator, ignoring ``None`` values and optionally stream encoding.
	
	Used internally by ``cinje.flatten``, this allows for easy use of a template generator as a WSGI body.
	"""
	
	input = (i for i in input if i)  # Omits `None` (empty wrappers) and empty chunks.
	
	if encoding:  # Automatically, and iteratively, encode the text if requested.
		input = iterencode(input, encoding, errors=errors)
	
	return input


def flatten(input, file=None, encoding=None, errors='strict'):
	"""Return a flattened representation of a cinje chunk stream.
	
	This has several modes of operation.  If no `file` argument is given, output will be returned as a string.
	The type of string will be determined by the presence of an `encoding`; if one is given the returned value is a
	binary string, otherwise the native unicode representation.  If a `file` is present, chunks will be written
	iteratively through repeated calls to `file.write()`, and the amount of data (characters or bytes) written
	returned.  The type of string written will be determined by `encoding`, just as the return value is when not
	writing to a file-like object.  The `errors` argument is passed through when encoding.
	
	We can highly recommend using the various stremaing IO containers available in the
	[`io`](https://docs.python.org/3/library/io.html) module, though
	[`tempfile`](https://docs.python.org/3/library/tempfile.html) classes are also quite useful.
	"""
	
	input = stream(input, encoding, errors)
	
	if file is None:  # Exit early if we're not writing to a file.
		return b''.join(input) if encoding else ''.join(input)
	
	counter = 0
	
	for chunk in input:
		file.write(chunk)
		counter += len(chunk)
	
	return counter


def fragment(string, name="anonymous", **context):
	"""Translate a template fragment into a callable function.
	
	**Note:** Use of this function is discouraged everywhere except tests, as no caching is implemented at this time.
	
	Only one function may be declared, either manually, or automatically. If automatic defintition is chosen the
	resulting function takes no arguments.  Additional keyword arguments are passed through as global variables.
	"""
	
	if ": def" in string or ":def" in string:
		code = string.encode('utf8').decode('cinje')
		name = None
	else:
		code = ": def {name}\n\n{string}".format(
				name = name,
				string = string,
			).encode('utf8').decode('cinje')
	
	environ = dict(context)
	
	exec(code, environ)
{'fraction_non_alphanumeric': 0.0826897944635856, 'fraction_numerical': 0.002851372222882262, 'mean_word_length': 4.193090684762493, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '598523', 'n_tokens_mistral': 2492, 'n_tokens_neox': 2324, 'n_words': 1074}