<commit_before>package io.jenetics.incubator.grammar;

import java.util.ArrayDeque;
import java.util.Deque;
import java.util.List;
import java.util.Random;
import java.util.stream.Collectors;

import org.testng.annotations.Test;

import io.jenetics.incubator.grammar.Grammar.Symbol;
import io.jenetics.incubator.grammar.Grammar.Terminal;

import io.jenetics.ext.util.TreeFormatter;
import io.jenetics.ext.util.TreeNode;

public class SymbolListParserTest {

	@Test
	public void foo() {
		final var grammar = Grammar.parse("""
			<expr> ::= ( <expr> <op> <expr> ) | <num> | <var> |  <fun> ( <expr>, <var> )
			<fun> ::= FUN1 | FUN2
			<op> ::= + | - | * | /
			<var> ::= x | y
			<num> ::= 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9
			"""
		);

		final List<Terminal> list = grammar.generate(new Random(290234568903L)::nextInt);
		final var string = list.stream()
			.map(Symbol::toString)
			.collect(Collectors.joining());

		System.out.println(string);

		final Deque<String> expr = list.stream()
			.map(Object::toString)
			.collect(Collectors.toCollection(ArrayDeque::new));

		System.out.println(expr);

		final TreeNode<String> tree = SymbolListParser.parse(expr);
		System.out.println(TreeFormatter.TREE.format(tree));

		System.out.println(tree);
	}

}<commit_msg>#767: Experiment with 'SymbolListParser'.<commit_after>package io.jenetics.incubator.grammar;

import java.util.ArrayDeque;
import java.util.Deque;
import java.util.List;
import java.util.Random;
import java.util.stream.Collectors;

import org.testng.annotations.Test;

import io.jenetics.incubator.grammar.Grammar.Symbol;
import io.jenetics.incubator.grammar.Grammar.Terminal;

import io.jenetics.ext.util.TreeFormatter;
import io.jenetics.ext.util.TreeNode;

public class SymbolListParserTest {

	@Test
	public void foo() {
		final var grammar = Grammar.parse("""
			<expr> ::= ( <expr> <op> <expr> ) | <num> | <var> |  <fun> ( <arg>, <arg> )
			<fun> ::= FUN1 | FUN2
			<arg> ::= <expr> | <var> | <num>
			<op> ::= + | - | * | /
			<var> ::= x | y
			<num> ::= 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9
			"""
		);

		final List<Terminal> list = grammar.generate(new Random(29022156195143L)::nextInt);
		final var string = list.stream()
			.map(Symbol::toString)
			.collect(Collectors.joining());

		System.out.println(string);

		final Deque<String> expr = list.stream()
			.map(Object::toString)
			.collect(Collectors.toCollection(ArrayDeque::new));

		System.out.println(expr);

		final TreeNode<String> tree = SymbolListParser.parse(expr);
		System.out.println(TreeFormatter.TREE.format(tree));

		System.out.println(tree);
	}

}
{'fraction_non_alphanumeric': 0.1698040722243565, 'fraction_numerical': 0.020361121782558587, 'mean_word_length': 4.05631067961165, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7594281', 'n_tokens_mistral': 1073, 'n_tokens_neox': 946, 'n_words': 177}