<commit_before>local Widget = require("ui/widget/widget")
local RenderText = require("ui/rendertext")
local Screen = require("ui/screen")
local Geom = require("ui/geometry")

-- TODO: rename string:gsplit definition

--[[
A TextWidget that handles long text wrapping
--]]
local TextBoxWidget = Widget:new{
	text = nil,
	face = nil,
	bgcolor = 0.0, -- [0.0, 1.0]
	fgcolor = 1.0, -- [0.0, 1.0]
	width = 400, -- in pixels
	height = nil,
	first_line = 1,
	virtual_line = 1, -- used by scroll bar
	line_height = 0.3, -- in em
	v_list = nil,
	_bb = nil,
	_length = 0,
}

function TextBoxWidget:init()
	local v_list = nil
	if self.height then
		v_list = self:_getCurrentVerticalList()
	else
		v_list = self:_getVerticalList()
	end
	self:_render(v_list)
end

function TextBoxWidget:_wrapGreedyAlg(h_list)
	local cur_line_width = 0
	local cur_line = {}
	local v_list = {}

	for k,w in ipairs(h_list) do
		cur_line_width = cur_line_width + w.width
		if cur_line_width <= self.width then
			table.insert(cur_line, w)
		else
			-- wrap to next line
			table.insert(v_list, cur_line)
			cur_line = {}
			cur_line_width = w.width
			table.insert(cur_line, w)
		end
	end
	-- handle last line
	table.insert(v_list, cur_line)

	return v_list
end

--[[
Lua doesn't have a string.split() function and most of the time
you don't really need it because string.gmatch() is enough.
However string.gmatch() has one significant disadvantage for me:
You can't split a string while matching both the delimited
strings and the delimiters themselves without tracking positions
and substrings. The string.gsplit() function below takes care of
this problem. 
Author: <NAME>
License: MIT/X11
Source: http://snippets.luacode.org/snippets/String_splitting_130
--]]
function string:gsplit(pattern, capture)
	pattern = pattern and tostring(pattern) or '%s+'
	if (''):find(pattern) then
		error('pattern matches empty string!', 2)
	end
	return coroutine.wrap(function()
		local index = 1
		repeat
			local first, last = self:find(pattern, index)
			if first and last then
				if index < first then
					coroutine.yield(self:sub(index, first - 1))
				end
				if capture then
					coroutine.yield(self:sub(first, last))
				end
				index = last + 1
			else
				if index <= #self then
					coroutine.yield(self:sub(index))
				end
				break
			end
		until index > #self
	end)
end

function TextBoxWidget:_getVerticalList(alg)
	if self.vertical_list then
		return self.vertical_list
	end
	-- build horizontal list
	local h_list = {}
	for words in self.text:gmatch("[\32-\127\192-\255]+[\128-\191]*") do
		for word in words:gsplit("%s+", true) do
			for w in word:gsplit("%p+", true) do
				local word_box = {}
				word_box.word = w
				word_box.width = RenderText:sizeUtf8Text(0, Screen:getWidth(), self.face, w, true).x
				table.insert(h_list, word_box)
			end
		end
	end

	-- @TODO check alg here 25.04 2012 (houqp)
	-- @TODO replace greedy algorithm with K&P algorithm  25.04 2012 (houqp)
	self.vertical_list = self:_wrapGreedyAlg(h_list)
	return self.vertical_list
end
<commit_msg>process new line symbol in WordBoxWidget<commit_after>local Widget = require("ui/widget/widget")
local RenderText = require("ui/rendertext")
local Screen = require("ui/screen")
local Geom = require("ui/geometry")

-- TODO: rename string:gsplit definition

--[[
A TextWidget that handles long text wrapping
--]]
local TextBoxWidget = Widget:new{
	text = nil,
	face = nil,
	bgcolor = 0.0, -- [0.0, 1.0]
	fgcolor = 1.0, -- [0.0, 1.0]
	width = 400, -- in pixels
	height = nil,
	first_line = 1,
	virtual_line = 1, -- used by scroll bar
	line_height = 0.3, -- in em
	v_list = nil,
	_bb = nil,
	_length = 0,
}

function TextBoxWidget:init()
	local v_list = nil
	if self.height then
		v_list = self:_getCurrentVerticalList()
	else
		v_list = self:_getVerticalList()
	end
	self:_render(v_list)
end

function TextBoxWidget:_wrapGreedyAlg(h_list)
	local cur_line_width = 0
	local cur_line = {}
	local v_list = {}

	for k,w in ipairs(h_list) do
		cur_line_width = cur_line_width + w.width
		if w.word == "\n" then
			if cur_line_width > 0 then
				-- hard line break
				table.insert(v_list, cur_line)
				cur_line = {}
				cur_line_width = 0
			end
		elseif cur_line_width > self.width then
			-- wrap to next line
			table.insert(v_list, cur_line)
			cur_line = {}
			cur_line_width = w.width
			table.insert(cur_line, w)
		else
			table.insert(cur_line, w)
		end
	end
	-- handle last line
	table.insert(v_list, cur_line)

	return v_list
end

--[[
Lua doesn't have a string.split() function and most of the time
you don't really need it because string.gmatch() is enough.
However string.gmatch() has one significant disadvantage for me:
You can't split a string while matching both the delimited
strings and the delimiters themselves without tracking positions
and substrings. The string.gsplit() function below takes care of
this problem. 
Author: <NAME>
License: MIT/X11
Source: http://snippets.luacode.org/snippets/String_splitting_130
--]]
function string:gsplit(pattern, capture)
	pattern = pattern and tostring(pattern) or '%s+'
	if (''):find(pattern) then
		error('pattern matches empty string!', 2)
	end
	return coroutine.wrap(function()
		local index = 1
		repeat
			local first, last = self:find(pattern, index)
			if first and last then
				if index < first then
					coroutine.yield(self:sub(index, first - 1))
				end
				if capture then
					coroutine.yield(self:sub(first, last))
				end
				index = last + 1
			else
				if index <= #self then
					coroutine.yield(self:sub(index))
				end
				break
			end
		until index > #self
	end)
end

function TextBoxWidget:_getVerticalList(alg)
	if self.vertical_list then
		return self.vertical_list
	end
	-- build horizontal list
	local h_list = {}
	local line_count = 0
	for line in self.text:gsplit("\n", true) do
		for words in line:gmatch("[\32-\127\192-\255]+[\128-\191]*") do
			for word in words:gsplit("%s+", true) do
				for w in word:gsplit("%p+", true) do
					local word_box = {}
					word_box.word = w
					word_box.width = RenderText:sizeUtf8Text(0, Screen:getWidth(), self.face, w, true).x
					table.insert(h_list, word_box)
				end
			end
		end
		if line:sub(-1) == "\n" then table.insert(h_list, {word = '\n', width = 0}) end
	end

	-- @TODO check alg here 25.04 2012 (houqp)
	-- @TODO replace greedy algorithm with K&P algorithm  25.04 2012 (houqp)
	self.vertical_list = self:_wrapGreedyAlg(h_list)
	return self.vertical_list
end

{'fraction_non_alphanumeric': 0.11409500155231295, 'fraction_numerical': 0.020956224774914625, 'mean_word_length': 3.615329512893983, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3612454', 'n_tokens_mistral': 2548, 'n_tokens_neox': 2244, 'n_words': 678}