<commit_before>-- This Source Code Form is subject to the terms of the Mozilla Public
-- License, v. 2.0. If a copy of the MPL was not distributed with this
-- file, You can obtain one at http://mozilla.org/MPL/2.0/.

local rem = require("regexp/pcre")

local grammar_dg = {}

--
-- Grammar graph context
--

grammar_dg.Context = class('DGContext')

grammar_dg.ParseContext = class('DGParseContext')

grammar_dg.ParseContext.property.top = {
	get = function (self) return self._ctxs[1] end
}

grammar_dg.ParseContext.property.current = {
	get = function (self) return self._ctxs[#self._ctxs] end
}

grammar_dg.ParseContext.property.current_init = {
	get = function (self)
		if self._initctxs then
			return self._initctxs[#self._initctxs]
		end
	end
}

grammar_dg.ParseContext.property.init = {
	get = function (self) return self._initctxs ~= nil end
}

local function revalidate(self)
	local validate = self._validate
	self._validate = {}
	for f, arg in pairs(validate) do
		f(arg)
	end
end

function grammar_dg.ParseContext.method:__init(iter, topctx, init)
	self.iter = iter
	self._bitoffset = 0
	self._marks = {}
	self._ctxs = {}
	self._validate = {}
	self:push(topctx)

	if init then
		self._initctxs = { init }
		self._initctxs_count = 1
	end

	self.current.validate = revalidate
end

function grammar_dg.ParseContext.method:update(iter)
	self.iter:move_to(iter)
end

function grammar_dg.ParseContext.method:parse(entity)
	local iter = entity
	while iter do
		iter = iter:next(self)
	end
	return self._ctxs[1]
end

function grammar_dg.ParseContext.method:pop()
	self._ctxs[#self._ctxs] = nil
	if self._initctxs then
		self._initctxs[self._initctxs_count] = nil
		self._initctxs_count = self._initctxs_count-1
	end
end<commit_msg>Add lookahead to grammar context<commit_after>-- This Source Code Form is subject to the terms of the Mozilla Public
-- License, v. 2.0. If a copy of the MPL was not distributed with this
-- file, You can obtain one at http://mozilla.org/MPL/2.0/.

local rem = require("regexp/pcre")

local grammar_dg = {}

--
-- Grammar graph context
--

grammar_dg.Context = class('DGContext')

grammar_dg.ParseContext = class('DGParseContext')

grammar_dg.ParseContext.property.top = {
	get = function (self) return self._ctxs[1] end
}

grammar_dg.ParseContext.property.current = {
	get = function (self) return self._ctxs[#self._ctxs] end
}

grammar_dg.ParseContext.property.current_init = {
	get = function (self)
		if self._initctxs then
			return self._initctxs[#self._initctxs]
		end
	end
}

grammar_dg.ParseContext.property.init = {
	get = function (self) return self._initctxs ~= nil end
}

local function revalidate(self)
	local validate = self._validate
	self._validate = {}
	for f, arg in pairs(validate) do
		f(arg)
	end
end

function grammar_dg.ParseContext.method:__init(iter, topctx, init)
	self.iter = iter
	self._bitoffset = 0
	self._marks = {}
	self._ctxs = {}
	self._validate = {}
	self:push(topctx)

	if init then
		self._initctxs = { init }
		self._initctxs_count = 1
	end

	self.current.validate = revalidate
end

function grammar_dg.ParseContext.method:update(iter)
	self.iter:move_to(iter)
end

function grammar_dg.ParseContext.method:lookahead()
	local iter = self.iter:copy()
	local la = self.iter:sub(1):asnumber()
	self:update(iter)
	return la
end

function grammar_dg.ParseContext.method:parse(entity)
	local iter = entity
	while iter do
		iter = iter:next(self)
	end
	return self._ctxs[1]
end

function grammar_dg.ParseContext.method:pop()
	self._ctxs[#self._ctxs] = nil
	if self._initctxs then
		self._initctxs[self._initctxs_count] = nil
		self._initctxs_count = self._initctxs_count-1
	end
end
{'fraction_non_alphanumeric': 0.11214697011242117, 'fraction_numerical': 0.005209761447765287, 'mean_word_length': 4.271676300578035, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5033080', 'n_tokens_mistral': 1359, 'n_tokens_neox': 1283, 'n_words': 315}