<commit_before># This file is part of NIT ( http://www.nitlanguage.org ).
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Additional features on Nit AST
#
# Most of these feature require that the precomputation `parentize_tokens`
# is run on the root node of a AST.
#
# These aditionnal features are used either to have a better association between tokens and productions
# But also to query how productions and tokens are placed in the lines of code.
module astutil

intrude import parser
import html

redef class ANode
	# Visit the AST and computes advanced AST attributes on Tokens and Prod
	# This also force a parent on the detashed tokens
	fun parentize_tokens
	do
		var v = new PTokenVisitor
		v.work(self)
	end
end

redef class Prod
	# The first token of the production in the AST
	# Computed by `parentize_tokens`
	var first_token: nullable Token = null

	# The last token of the production in the AST
	# Computed by `parentize_tokens`
	var last_token: nullable Token = null

	# Is the production contained in full block of line?
	# Computed by `parentize_tokens`
	fun is_block: Bool
	do
		if first_token == null or last_token == null then return false
		#sys.stderr.write "{self}@{location}: start={first_token.is_starting_line} {first_token.inspect}@{first_token.location} ; end={last_token.is_ending_line} {last_token.inspect}@{last_token.location}\n"
		return first_token.is_starting_line and last_token.is_ending_line
	end

	# Is the production a part of a single line (without being a block)
	# Computed by `parentize_tokens`
	fun is_span: Bool
	do
		if first_token == null or last_token == null then return false
		return not is_block and location.line_start == location.line_end
	end

	# A XML representation of the AST
	# Productions and token become elements
	# 
	# detached tokens and whitespaces are preserved in the XML
	# 
	# ~~~
	# import parser_util
	# var text = "y += foo"
	# var ast = (new ToolContext).parse_something(text)
	# assert ast isa AExpr
	# ast.parentize_tokens
	# assert ast.to_xml.write_to_string == """<ACallReassignExpr><TId>y</TId> <APlusAssignOp><TPluseq>+=</TPluseq></APlusAssignOp> <ACallExpr><TId>foo</TId></ACallExpr></ACallReassignExpr>"""
	# ~~~
	fun to_xml: HTMLTag
	do
		var res = new HTMLTag("AST")
		var stack = new Array[HTMLTag]
		var c = first_token
		while c != null do
			if c != first_token then res.append(c.blank_before)
			var sp = c.starting_prods
			if sp != null then for p in sp do
				var tag = new HTMLTag(p.class_name)
				res.add tag
				stack.add res
				res = tag
			end
			var tag = new HTMLTag(c.class_name)
			res.add tag
			tag.append c.text
			var ep = c.ending_prods<commit_msg>astutil: update nitunit related to AST<commit_after># This file is part of NIT ( http://www.nitlanguage.org ).
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Additional features on Nit AST
#
# Most of these feature require that the precomputation `parentize_tokens`
# is run on the root node of a AST.
#
# These aditionnal features are used either to have a better association between tokens and productions
# But also to query how productions and tokens are placed in the lines of code.
module astutil

intrude import parser
import html

redef class ANode
	# Visit the AST and computes advanced AST attributes on Tokens and Prod
	# This also force a parent on the detashed tokens
	fun parentize_tokens
	do
		var v = new PTokenVisitor
		v.work(self)
	end
end

redef class Prod
	# The first token of the production in the AST
	# Computed by `parentize_tokens`
	var first_token: nullable Token = null

	# The last token of the production in the AST
	# Computed by `parentize_tokens`
	var last_token: nullable Token = null

	# Is the production contained in full block of line?
	# Computed by `parentize_tokens`
	fun is_block: Bool
	do
		if first_token == null or last_token == null then return false
		#sys.stderr.write "{self}@{location}: start={first_token.is_starting_line} {first_token.inspect}@{first_token.location} ; end={last_token.is_ending_line} {last_token.inspect}@{last_token.location}\n"
		return first_token.is_starting_line and last_token.is_ending_line
	end

	# Is the production a part of a single line (without being a block)
	# Computed by `parentize_tokens`
	fun is_span: Bool
	do
		if first_token == null or last_token == null then return false
		return not is_block and location.line_start == location.line_end
	end

	# A XML representation of the AST
	# Productions and token become elements
	# 
	# detached tokens and whitespaces are preserved in the XML
	# 
	# ~~~
	# import parser_util
	# var text = "y += foo"
	# var ast = (new ToolContext).parse_something(text)
	# assert ast isa AExpr
	# ast.parentize_tokens
	# assert ast.to_xml.write_to_string == """<ACallReassignExpr><AQid><TId>y</TId></AQid> <APlusAssignOp><TPluseq>+=</TPluseq></APlusAssignOp> <ACallExpr><AQid><TId>foo</TId></AQid></ACallExpr></ACallReassignExpr>"""
	# ~~~
	fun to_xml: HTMLTag
	do
		var res = new HTMLTag("AST")
		var stack = new Array[HTMLTag]
		var c = first_token
		while c != null do
			if c != first_token then res.append(c.blank_before)
			var sp = c.starting_prods
			if sp != null then for p in sp do
				var tag = new HTMLTag(p.class_name)
				res.add tag
				stack.add res
				res = tag
			end
			var tag = new HTMLTag(c.class_name)
			res.add tag
			tag.append c.text
			var ep = c.ending_prods
{'fraction_non_alphanumeric': 0.08023237556916313, 'fraction_numerical': 0.0012560841576385617, 'mean_word_length': 4.087859424920127, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5167922', 'n_tokens_mistral': 2098, 'n_tokens_neox': 1966, 'n_words': 778}