<commit_before>enc = require './encoding'
http = require('http');
b64 = enc.b64.encoding

stream_data = (src, dst) ->
	src.on 'data', (chunk) ->
		dst.write(b64.decode(new Buffer chunk, 'utf-8'));

server = http.createServer (req, res) ->
	stream_data(req, res);
	req.on 'end', ->
		res.end();

server.listen(39393);<commit_msg>Early streamer class design<commit_after>stream = require('stream');
enc = require('./encoding');

exports.Streamer = class Streamer extends stream.Duplex

	constructor : (block_len) ->
		alphabet = enc.b58.alphabet if block_len is 58
		alphabet = enc.b62.alphabet if block_len is 58
		alphabet = enc.b64.alphabet if block_len is 58
		@encoder = new Encoding(alphabet, block_len);
		super({highWaterMark: block_len});

	encode : (src) ->
		src.on('readable', () ->
			chunk = null;
			while (chunk = src.read(@highWaterMark)) != null
				@write enc.b64.encoding.encode(chunk);
			)

	decode : (src) ->
		src.on('readable', () ->
			chunk = null;
			while (chunk = src.read(@highWaterMark)) != null
				@write(enc.b64.encoding.decode(chunk));
			)
{'fraction_non_alphanumeric': 0.1685288640595903, 'fraction_numerical': 0.0260707635009311, 'mean_word_length': 3.953917050691244, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1824009', 'n_tokens_mistral': 424, 'n_tokens_neox': 373, 'n_words': 85}