<commit_before>--AST tools
local lpeg = require "lpeg"
local ansi = require "ansi"
local walker = require "backwalk"
local cyan = tostring(ansi.cyan)
local blue = tostring(ansi.blue)
local magenta = tostring(ansi.magenta)
local clear = tostring(ansi.clear)
local green = tostring(ansi.green)
local red = tostring(ansi.red)

local function select_rule (id, ast) 
-- select_rule (<String>, <Node>) -> <Index <Node>>
	local vec = {}
	local meta = {}
	if ast.isnode then
		meta = getmetatable(ast)
	else
		error "second argument must be of type <Node>" 
	end
	local function selector(id,ast)
		if ast.isnode then
			if ast.id == id then
				vec[#vec+1] = ast  
			end
		end
		for _, v in pairs(ast) do
			if type(v) == "table" and v.isnode then
				selector(id,v)
			end
		end
	end
	selector(id,ast)
	setmetatable(vec,meta)
	return vec
end

local function node_pr(node,_,depth)
	local prefix = ("  "):rep(depth-1)
	io.write(prefix,blue,node.parent().id," ",
			 magenta,node.id," ",
			 cyan,node.pos,clear,"\n")
	for i,v in ipairs(node) do
		if type(v) == "string" then
			io.write (prefix,green,'"',clear,v,green,'"',clear,"\n")
		end
	end
end

local function root(node)
	if node.parent() == node then
		return node
	else 
		return root(node.parent()) 
	end
end

local function ast_range(node)
	local root = node:root()
	local first, last, _ =  root.index(node)
	return root.index, first, last
end

local function ast_pr(ast)
	-- now we can print an AST.
	local ndx = ast.index
	for i= 1,ndx.len() do 
		node_pr(ndx(i))
	end
end

local function parse(grammar, string)
	local ast = lpeg.match(grammar,string)
	return walker.walk_ast(ast)
end

return {
	select_rule = select_rule ,
	pr = ast_pr,
	root = root,
	range= ast_range,
	copy = clone_ast,
	walk = walker.walk_ast,
	parse = parse
}<commit_msg>rewrite ast.pr to use range.<commit_after>--AST tools
local lpeg = require "lpeg"
local ansi = require "ansi"
local walker = require "backwalk"
local cyan = tostring(ansi.cyan)
local blue = tostring(ansi.blue)
local magenta = tostring(ansi.magenta)
local clear = tostring(ansi.clear)
local green = tostring(ansi.green)
local red = tostring(ansi.red)

local function root(node)
	if node.parent() == node then
		return node
	else 
		return root(node.parent()) 
	end
end

local function ast_range(node)
	local root = node:root()
	local first, last, _ =  root.index(node)
	return root.index, first, last
end

local function node_pr(node,_,depth)
	local prefix = ("  "):rep(depth-1)
	io.write(prefix,blue,node.parent().id," ",
			 magenta,node.id," ",
			 cyan,node.pos,clear,"\n")
	for i,v in ipairs(node) do
		if type(v) == "string" then
			io.write (prefix,green,'"',clear,v,green,'"',clear,"\n")
		end
	end
end

local function ast_pr(ast)
	-- now we can print an AST.
	local ndx, first, last = ast_range(ast)
	for i= first,last do 
		node_pr(ndx(i))
	end
end

local function select_rule (id, ast) 
-- select_rule (<String>, <Node>) -> <Index <Node>>
	local vec = {}
	local meta = {}
	if ast.isnode then
		meta = getmetatable(ast)
	else
		error "second argument must be of type <Node>" 
	end
	local function selector(id,ast)
		if ast.isnode then
			if ast.id == id then
				vec[#vec+1] = ast  
			end
		end
		for _, v in pairs(ast) do
			if type(v) == "table" and v.isnode then
				selector(id,v)
			end
		end
	end
	selector(id,ast)
	setmetatable(vec,meta)
	return vec
end


local function parse(grammar, string)
	local ast = lpeg.match(grammar,string)
	return walker.walk_ast(ast)
end

return {
	select_rule = select_rule ,
	pr = ast_pr,
	root = root,
	range= ast_range,
	copy = clone_ast,
	walk = walker.walk_ast,
	parse = parse
}
{'fraction_non_alphanumeric': 0.13234077750206782, 'fraction_numerical': 0.0013785497656465398, 'mean_word_length': 3.863270777479893, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4179205', 'n_tokens_mistral': 1453, 'n_tokens_neox': 1365, 'n_words': 388}