<commit_before># Pliant
Implementation of a modified Earley parser in C# inspired by the Marpa Parser project.

## Description
Pliant is a table driven parser that implements the Earley algorithm. Two optimizations are added to handle issues with the original Earley implementation: 

1. Optimization from <NAME> to efficiently handle right recursions. 
2. Bug fix from Aycock and Horspool to handle nullable predictions

## Using the Code

### Creating Grammars

#### Using the Grammar Builder classes

```CSharp
using Pliant.Builders;
using Pliant.Grammars;
using Pliant.Automata;

public static int Main(string[] args)
{
	var digits = CreateDigitLexerRule();
	var whitespace = CreateWhitespaceLexerRule();
	
	ProductionBuilder 
		Calculator = "Calculator",
		Factor = "Factor",
		Term = "Term",
		Expression = "Expression",
		Number = "Number";
		
	Calculator.Definition 
		= Expression ;
		
	Expression.Definition
		= Expression + '+' + Term 
		| Term ;

	Term.Definition 
		= Term + '*' + Factor
		| Factor ;
		
	Factor.Definition 
		= Number ;
		
	Number.Definition
		= digits;
		
	var grammar = new GrammarBuilder(
		Calculator, 
		new[]{ Calculator, Factor, Term, Expression, Number }, 
		new []{ whitespace })
	.ToGrammar();	
	
	// TODO: Use the grammar in a parse.
}

private static BaseLexerRule CreateDigitLexerRule()
{
	var startState = new DfaState();
	var finalState = new DfaState(true);
	var digitTransition = new DfaTransition(new DigitTerminal(), finalState);
	startState.AddTransition(digitTransition);
	finalState.AddTransition(digitTransition);
	return new DfaLexerRule(startState, "[\\d]+");
}

private static ILexerRule CreateWhitespaceLexerRule()
{
	var whitespaceTerminal = new WhitespaceTerminal();
	var startState = new DfaState();
	var finalState = new DfaState(true);
	var whitespaceTransition = new DfaTransition(whitespaceTerminal, finalState);
	startState.AddTransition(whitespaceTransition);
	finalState.AddTransition(whitespaceTransition);
	return new DfaLexerRule(startState, new TokenType("[\\s]+"));	
}		
```

#### Using the Ebnf Text Interface ( *work in progress* )

```CSharp
public static int Main (string[] args)
{
	var grammarText = @"
	Calculator 
		= Expression;
		
	Expression 
		= Expression Term
		| Term;
		
	Term 
		= Term '*' Factor
		| Factor;
		
	Factor 
		= Number ;
	
	Number 
		= Digits;
		
	Digits ~ /[0-9]+/ ;
	Whitespace ~ /[\\s]+/ ;
	
	:start = Calculator;
	:ignore = Whitespace;";
	
	var compiler = new EbnfCompiler();
	var grammar = compiler.Compile(grammarText);
	
	// TODO: use the grammar in a parse.
}
```

### Using Grammars to Recognize Input

Using the calculator grammar from above, we can parse input by constructing
a parse engine and parse interface instance.

```csharp
var input = "(1 + 1) / 3 - 2";

// use the calculator grammar from above
var parseEngine = new ParseEngine(grammar);

// use the parse interface to querying the parse engine
// for state and use that state to select lexer rules.
var parseInterface = new ParseInterface(parseEngine, input);

// when a parse is recognized, the parse engine is allowed to move
// forward and continues to accept symbols. 
var recognzied = false;
var errorPosition = 0;<commit_msg>updates input to reflect grammar<commit_after># Pliant
Implementation of a modified Earley parser in C# inspired by the Marpa Parser project.

## Description
Pliant is a table driven parser that implements the Earley algorithm. Two optimizations are added to handle issues with the original Earley implementation: 

1. Optimization from Joop Leo to efficiently handle right recursions. 
2. Bug fix from Aycock and Horspool to handle nullable predictions

## Using the Code

### Creating Grammars

#### Using the Grammar Builder classes

```CSharp
using Pliant.Builders;
using Pliant.Grammars;
using Pliant.Automata;

public static int Main(string[] args)
{
	var digits = CreateDigitLexerRule();
	var whitespace = CreateWhitespaceLexerRule();
	
	ProductionBuilder 
		Calculator = "Calculator",
		Factor = "Factor",
		Term = "Term",
		Expression = "Expression",
		Number = "Number";
		
	Calculator.Definition 
		= Expression ;
		
	Expression.Definition
		= Expression + '+' + Term 
		| Term ;

	Term.Definition 
		= Term + '*' + Factor
		| Factor ;
		
	Factor.Definition 
		= Number ;
		
	Number.Definition
		= digits;
		
	var grammar = new GrammarBuilder(
		Calculator, 
		new[]{ Calculator, Factor, Term, Expression, Number }, 
		new []{ whitespace })
	.ToGrammar();	
	
	// TODO: Use the grammar in a parse.
}

private static BaseLexerRule CreateDigitLexerRule()
{
	var startState = new DfaState();
	var finalState = new DfaState(true);
	var digitTransition = new DfaTransition(new DigitTerminal(), finalState);
	startState.AddTransition(digitTransition);
	finalState.AddTransition(digitTransition);
	return new DfaLexerRule(startState, "[\\d]+");
}

private static ILexerRule CreateWhitespaceLexerRule()
{
	var whitespaceTerminal = new WhitespaceTerminal();
	var startState = new DfaState();
	var finalState = new DfaState(true);
	var whitespaceTransition = new DfaTransition(whitespaceTerminal, finalState);
	startState.AddTransition(whitespaceTransition);
	finalState.AddTransition(whitespaceTransition);
	return new DfaLexerRule(startState, new TokenType("[\\s]+"));	
}		
```

#### Using the Ebnf Text Interface ( *work in progress* )

```CSharp
public static int Main (string[] args)
{
	var grammarText = @"
	Calculator 
		= Expression;
		
	Expression 
		= Expression Term
		| Term;
		
	Term 
		= Term '*' Factor
		| Factor;
		
	Factor 
		= Number ;
	
	Number 
		= Digits;
		
	Digits ~ /[0-9]+/ ;
	Whitespace ~ /[\\s]+/ ;
	
	:start = Calculator;
	:ignore = Whitespace;";
	
	var compiler = new EbnfCompiler();
	var grammar = compiler.Compile(grammarText);
	
	// TODO: use the grammar in a parse.
}
```

### Using Grammars to Recognize Input

Using the calculator grammar from above, we can parse input by constructing
a parse engine and parse interface instance.

```csharp
var input = "(1 + 1) * 3 + 2";

// use the calculator grammar from above
var parseEngine = new ParseEngine(grammar);

// use the parse interface to querying the parse engine
// for state and use that state to select lexer rules.
var parseInterface = new ParseInterface(parseEngine, input);

// when a parse is recognized, the parse engine is allowed to move
// forward and continues to accept symbols. 
var recognzied = false;
var errorPosition = 0;
{'fraction_non_alphanumeric': 0.09948702005285248, 'fraction_numerical': 0.002798072438986476, 'mean_word_length': 4.155448717948718, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3884586', 'n_tokens_mistral': 2151, 'n_tokens_neox': 1902, 'n_words': 660}