-- functions for grabbing data from input strings


local function split(text, pattern)
	local t = {};
	local lp = 0;
	while (true) do
		local p = find(text, pattern, lp, true);
		if (p) then
			insert(t, sub(text, lp, p - 1));
			lp = p + 1;
		else
			insert(t, sub(text, lp));
			break;
		end;
	end;
	return t;
end;

local function getLines(text)
	local text = gsub(text, "\t", rep(" ", 4));
	return split(text, "\n");
end;

local function getWords(text, includeNewLines)
	local text = gsub(text, "\t", rep(" ", 4));
	local lines , words = split(text, "\n"), {};
	local nlines = #lines;
	for i = 1, nlines do
		local line = lines[i];
		for word in gmatch(line, " *[^%s]+ *") do
			insert(words, word);
		end;
		if (includeNewLines and i < nlines) then
			insert(words, "\n"); 
		end;
	end;
	return words;
end;
