<commit_before>import unittest2
from collections import defaultdict

from extractor import NotEnoughData
from model import BaseModel
from feature import Feature,JSONFeature
from dependency_injection import Registry
from data import *
from util import chunked

data_source = {
	'mary'   : 'mary had a little lamb little lamb little lamb',
	'humpty' : 'humpty dumpty sat on a wall humpty dumpty had a great fall',
	'numbers' : range(10),
	'cased' : 'This is a test.'
}

class TextStream(Node):
	
	def __init__(self, chunksize = 3, needs = None):
		super(TextStream,self).__init__(needs = needs)
		self._chunksize = chunksize
	
	def _process(self,data):
		try:
			flo = StringIO(data_source[data])
		except KeyError:
			flo = StringIO(data)
		
		for chunk in chunked(flo,chunksize = self._chunksize):
			print chunk
			yield chunk

class ToUpper(Node):

	def __init__(self, needs = None):<commit_msg>Dangit.  Remove print from tests<commit_after>import unittest2
from collections import defaultdict

from extractor import NotEnoughData
from model import BaseModel
from feature import Feature,JSONFeature
from dependency_injection import Registry
from data import *
from util import chunked

data_source = {
	'mary'   : 'mary had a little lamb little lamb little lamb',
	'humpty' : 'humpty dumpty sat on a wall humpty dumpty had a great fall',
	'numbers' : range(10),
	'cased' : 'This is a test.'
}

class TextStream(Node):
	
	def __init__(self, chunksize = 3, needs = None):
		super(TextStream,self).__init__(needs = needs)
		self._chunksize = chunksize
	
	def _process(self,data):
		try:
			flo = StringIO(data_source[data])
		except KeyError:
			flo = StringIO(data)
		
		for chunk in chunked(flo,chunksize = self._chunksize):
			yield chunk

class ToUpper(Node):

	def __init__(self, needs = None):
{'fraction_non_alphanumeric': 0.08843159065628477, 'fraction_numerical': 0.004449388209121246, 'mean_word_length': 4.6572327044025155, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7452816', 'n_tokens_mistral': 621, 'n_tokens_neox': 566, 'n_words': 193}