<commit_before>local LXSC = require 'lib/lxsc'
LXSC.Datamodel = {}; LXSC.Datamodel.__meta = {__index=LXSC.Datamodel}

setmetatable(LXSC.Datamodel,{__call=function(dm,scxml,scope)
	if not scope then scope = {} end
	function scope.In(id) return scxml:isActive(id) end
	return setmetatable({ statesInited={}, scxml=scxml, scope=scope, cache={} },dm.__meta)
end})

function LXSC.Datamodel:initAll()
	local function recurse(state)
		self:initState(state)
		for _,s in ipairs(state.reals) do recurse(s) end
	end
	recurse(self.scxml)
end

function LXSC.Datamodel:initState(state)
	if not self.statesInited[state] then
		for _,data in ipairs(state._datamodels) do
			-- TODO: support data.src
			local value = self:eval(data.expr or tostring(data._text))
			if value~=LXSC.Datamodel.EVALERROR then 
				self:set( data.id, value )
			else
				self:set( data.id, nil )
			end
		end
		self.statesInited[state] = true
	end
end

function LXSC.Datamodel:eval(expression)<commit_msg>Support <data src='file:...'><commit_after>local LXSC = require 'lib/lxsc'
LXSC.Datamodel = {}; LXSC.Datamodel.__meta = {__index=LXSC.Datamodel}

setmetatable(LXSC.Datamodel,{__call=function(dm,scxml,scope)
	if not scope then scope = {} end
	function scope.In(id) return scxml:isActive(id) end
	return setmetatable({ statesInited={}, scxml=scxml, scope=scope, cache={} },dm.__meta)
end})

function LXSC.Datamodel:initAll()
	local function recurse(state)
		self:initState(state)
		for _,s in ipairs(state.reals) do recurse(s) end
	end
	recurse(self.scxml)
end

function LXSC.Datamodel:initState(state)
	if not self.statesInited[state] then
		for _,data in ipairs(state._datamodels) do
			local value, err
			if data.src then
				local colon = data.src:find(':')
				local scheme,hierarchy = data.src:sub(1,colon-1), data.src:sub(colon+1)
				if scheme=='file' then
					local f,msg = io.open(hierarchy,"r")
					if not f then
						self.scxml:fireEvent("error.execution.invalid-file",msg)
					else
						value = self:eval(f:read("*all"))
						f:close()
					end
				else
					self.scxml:fireEvent("error.execution.invalid-data-scheme","LXSC does not support <data src='"..scheme..":...'>")
				end
			else
				value = self:eval(data.expr or tostring(data._text))
			end

			if value~=LXSC.Datamodel.EVALERROR then 
				self:set( data.id, value )
			else
				self:set( data.id, nil )
			end
		end
		self.statesInited[state] = true
	end
end

function LXSC.Datamodel:eval(expression)
{'fraction_non_alphanumeric': 0.1347488770926909, 'fraction_numerical': 0.0012249897917517355, 'mean_word_length': 4.337690631808279, 'pattern_counts': {'":': 1, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2310024', 'n_tokens_mistral': 1042, 'n_tokens_neox': 915, 'n_words': 196}