<commit_before>local Sink = import("gla.textgen.Sink")
local Filter = import("gla.textgen.Filter")

const CLUSTER_SIZE = 256

enum Fragment {
	Null,
	Text,
	Newline,
	Indent,
	Begin,
	End,
	Embed,
	Command	
}

local Cluster = class {
	next = null
	types = null
	data = null
	fill = 0

	constructor(clusterSize) {
		data = array(clusterSize)
		types = array(clusterSize)
	}
}

local Printer

Printer = class {
	_firstCluster = null
	_lastCluster = null
	_clusterSize = null

	constructor(clusterSize = CLUSTER_SIZE) {
		_clusterSize = clusterSize
	}

	function clear() {
		_firstCluster = null
		_lastCluster = null
	}

	function _append(type, data = null) {
		if(_lastCluster == null) {
			_lastCluster = Cluster(_clusterSize)
			_firstCluster = _lastCluster
		}
		else if(_lastCluster.fill == _clusterSize) {
			_lastCluster.next = Cluster(_clusterSize)
			_lastCluster = _lastCluster.next
		}
		_lastCluster.types[_lastCluster.fill] = type
		_lastCluster.data[_lastCluster.fill] = data
		_lastCluster.fill++
	}

	function pn(text = null) {
		if(text != null)
			_append(Fragment.Text, text)
		_append(Fragment.Newline)
	}

	function pni(text) {
		_append(Fragment.Text, text)
		_append(Fragment.Newline)
		_append(Fragment.Indent, 1)
	}

	function ipn(text) {
		_append(Fragment.Indent, 1)
		_append(Fragment.Text, text)
		_append(Fragment.Newline)
	}

	function pnu(text) {
		_append(Fragment.Text, text)
		_append(Fragment.Newline)
		_append(Fragment.Indent, -1)
	}

	function upn(text) {
		_append(Fragment.Indent, -1)
		_append(Fragment.Text, text)
		_append(Fragment.Newline)
	}

	function upni(text) {
		_append(Fragment.Indent, -1)
		_append(Fragment.Text, text)
		_append(Fragment.Newline)
		_append(Fragment.Indent, 1)
	}

	function pr(text) {
		if(text != null)
			_append(Fragment.Text, text)
	}

	function pri(text) {
		_append(Fragment.Text, text)
		_append(Fragment.Indent, 1)
	}

	function ipr(text) {
		_append(Fragment.Indent, 1)
		_append(Fragment.Text, text)
	}

	function pru(text) {
		_append(Fragment.Text, text)
		_append(Fragment.Indent, -1)
	}

	function upr(text) {
		_append(Fragment.Indent, -1)
		_append(Fragment.Text, text)
	}

	function upri(text) {
		_append(Fragment.Indent, -1)
		_append(Fragment.Text, text)
		_append(Fragment.Indent, 1)
	}

	function indent(amount = 1) {
		_append(Fragment.Indent, amount)
	}

	function unindent(amount = 1) {
		_append(Fragment.Indent, -amount)
	}

	function embed(printer) {
		assert(printer instanceof Printer)
		_append(Fragment.Embed, printer)
	}

	function begin(filter) {
		assert(filter instanceof Filter)
		_append(Fragment.Begin, filter)
	}
<commit_msg>printer: method pt() added for printing text with indents and newlines<commit_after>local Sink = import("gla.textgen.Sink")
local Filter = import("gla.textgen.Filter")

const CLUSTER_SIZE = 256

enum Fragment {
	Null,
	Text,
	Newline,
	Indent,
	Begin,
	End,
	Embed,
	Command	
}

local Cluster = class {
	next = null
	types = null
	data = null
	fill = 0

	constructor(clusterSize) {
		data = array(clusterSize)
		types = array(clusterSize)
	}
}

local Printer

Printer = class {
	_firstCluster = null
	_lastCluster = null
	_clusterSize = null

	constructor(clusterSize = CLUSTER_SIZE) {
		_clusterSize = clusterSize
	}

	function clear() {
		_firstCluster = null
		_lastCluster = null
	}

	function _append(type, data = null) {
		if(_lastCluster == null) {
			_lastCluster = Cluster(_clusterSize)
			_firstCluster = _lastCluster
		}
		else if(_lastCluster.fill == _clusterSize) {
			_lastCluster.next = Cluster(_clusterSize)
			_lastCluster = _lastCluster.next
		}
		_lastCluster.types[_lastCluster.fill] = type
		_lastCluster.data[_lastCluster.fill] = data
		_lastCluster.fill++
	}

	function pn(text = null) {
		if(text != null)
			_append(Fragment.Text, text)
		_append(Fragment.Newline)
	}

	function pni(text) {
		_append(Fragment.Text, text)
		_append(Fragment.Newline)
		_append(Fragment.Indent, 1)
	}

	function ipn(text) {
		_append(Fragment.Indent, 1)
		_append(Fragment.Text, text)
		_append(Fragment.Newline)
	}

	function pnu(text) {
		_append(Fragment.Text, text)
		_append(Fragment.Newline)
		_append(Fragment.Indent, -1)
	}

	function upn(text) {
		_append(Fragment.Indent, -1)
		_append(Fragment.Text, text)
		_append(Fragment.Newline)
	}

	function upni(text) {
		_append(Fragment.Indent, -1)
		_append(Fragment.Text, text)
		_append(Fragment.Newline)
		_append(Fragment.Indent, 1)
	}

	function pr(text) {
		if(text != null)
			_append(Fragment.Text, text)
	}

	function pri(text) {
		_append(Fragment.Text, text)
		_append(Fragment.Indent, 1)
	}

	function ipr(text) {
		_append(Fragment.Indent, 1)
		_append(Fragment.Text, text)
	}

	function pru(text) {
		_append(Fragment.Text, text)
		_append(Fragment.Indent, -1)
	}

	function upr(text) {
		_append(Fragment.Indent, -1)
		_append(Fragment.Text, text)
	}

	function upri(text) {
		_append(Fragment.Indent, -1)
		_append(Fragment.Text, text)
		_append(Fragment.Indent, 1)
	}

	function pt(text, indentstr = "\t", nlstr = "\n") {
		local index = 0
		local indent = 0
		local curindent
		for(local next = text.find(nlstr); next != null; next = text.find(nlstr, index)) {
			curindent = 0
			while(text.find(indentstr, index) == index) {
				index += indentstr.len()
				curindent++
			}
			if(index < next) { //i.e. not an empty line
				if(curindent != indent) {
					_append(Fragment.Indent, curindent - indent)
					indent = curindent
				}
				_append(Fragment.Text, text.slice(index, next))
				_append(Fragment.Newline)
			}
			index = next + nlstr.len()
		}
		curindent = 0
		while(text.find(indentstr, index) == index) {
			index += indentstr.len()
			curindent++
		}
		if(index < text.len()) {
			if(curindent != indent) {
				_append(Fragment.Indent, curindent - indent)
				indent = curindent
			}
			_append(Fragment.Text, text.slice(index))
			_append(Fragment.Newline)
		}
		if(indent > 0)
			_append(Fragment.Indent, -indent)
	}

	function indent(amount = 1) {
		_append(Fragment.Indent, amount)
	}

	function unindent(amount = 1) {
		_append(Fragment.Indent, -amount)
	}

	function embed(printer) {
		assert(printer instanceof Printer)
		_append(Fragment.Embed, printer)
	}

	function begin(filter) {
		assert(filter instanceof Filter)
		_append(Fragment.Begin, filter)
	}

{'fraction_non_alphanumeric': 0.12472195741976486, 'fraction_numerical': 0.0065141404512233875, 'mean_word_length': 3.9026479750778815, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4414330', 'n_tokens_mistral': 2573, 'n_tokens_neox': 2305, 'n_words': 453}