<commit_before>namespace Boo.OMeta

import Boo.Adt
import System.Collections.Generic

data OMetaMatch(Input as OMetaInput) = SuccessfulMatch(Value as object) | FailedMatch()

callable OMetaRule(context as OMetaGrammar, input as OMetaInput) as OMetaMatch

interface OMetaGrammar:
	
	def InstallRule(ruleName as string, rule as OMetaRule)
	def Apply(context as OMetaGrammar, rule as string, input as OMetaInput) as OMetaMatch
	def SuperApply(context as OMetaGrammar, rule as string, input as OMetaInput) as OMetaMatch

class OMetaGrammarRoot(OMetaGrammar):
	
	class MemoKey:
		final _rule as string
		final _input as OMetaInput
		
		def constructor(rule as string, input as OMetaInput):
			_rule = rule
			_input = input
			
		override def Equals(o):
			other as MemoKey = o
			return _input is other._input and _rule is other._rule
			
		override def GetHashCode():
			return _rule.GetHashCode() ^ _input.GetHashCode()
	
	_rules = Dictionary[of string, OMetaRule]()
	_memo = Dictionary[of MemoKey, OMetaMatch]()
	
	def InstallRule(ruleName as string, rule as OMetaRule):
		_rules[ruleName] = rule
	
	def Apply(context as OMetaGrammar, rule as string, input as OMetaInput):
		
		key = MemoKey(rule, input)
		memo as OMetaMatch
		if _memo.TryGetValue(key, memo):
			return memo
			
		_memo[key] = FailedMatch(input)
		memo = Eval(context, rule, input)
		_memo[key] = memo
		return memo
		
	def Eval(context as OMetaGrammar, rule as string, input as OMetaInput):
		found as OMetaRule
		if _rules.TryGetValue(rule, found):
			return found(context, input)
		return RuleMissing(context, rule, input)
		
	def SuperApply(context as OMetaGrammar, rule as string, input as OMetaInput):
		return Apply(context, rule, input)
		
	virtual def RuleMissing(context as OMetaGrammar, rule as string, input as OMetaInput) as OMetaMatch:
		raise "Rule '${rule}' missing!"
		
class OMetaGrammarPrototype(OMetaGrammarRoot):
	
	def constructor():
		InstallRule("whitespace", makeRule(char.IsWhiteSpace))
		InstallRule("letter", makeRule(char.IsLetter))
		InstallRule("digit", makeRule(char.IsDigit))
		
class OMetaDelegatingGrammar(OMetaGrammarRoot):
	
	_prototype as OMetaGrammar
	
	def constructor(prototype as OMetaGrammar):
		_prototype = prototype
		
	override def RuleMissing(context as OMetaGrammar, rule as string, input as OMetaInput):
		return SuperApply(context, rule, input)
		
	override def SuperApply(context as OMetaGrammar, rule as string, input as OMetaInput):
		return _prototype.SuperApply(context, rule, input)
		
def makeRule(predicate as System.Predicate[of object]) as OMetaRule:
	def rule(context as OMetaGrammar, input as OMetaInput) as OMetaMatch:
		if not input.IsEmpty and predicate(input.Head):
			return SuccessfulMatch(input.Tail, input.Head)
		return FailedMatch(input)
	return rule
<commit_msg>any rule<commit_after>namespace Boo.OMeta

import Boo.Adt
import System.Collections.Generic

data OMetaMatch(Input as OMetaInput) = SuccessfulMatch(Value as object) | FailedMatch()

callable OMetaRule(context as OMetaGrammar, input as OMetaInput) as OMetaMatch

interface OMetaGrammar:
	
	def InstallRule(ruleName as string, rule as OMetaRule)
	def Apply(context as OMetaGrammar, rule as string, input as OMetaInput) as OMetaMatch
	def SuperApply(context as OMetaGrammar, rule as string, input as OMetaInput) as OMetaMatch

class OMetaGrammarRoot(OMetaGrammar):
	
	class MemoKey:
		final _rule as string
		final _input as OMetaInput
		
		def constructor(rule as string, input as OMetaInput):
			_rule = rule
			_input = input
			
		override def Equals(o):
			other as MemoKey = o
			return _input is other._input and _rule is other._rule
			
		override def GetHashCode():
			return _rule.GetHashCode() ^ _input.GetHashCode()
	
	_rules = Dictionary[of string, OMetaRule]()
	_memo = Dictionary[of MemoKey, OMetaMatch]()
	
	def InstallRule(ruleName as string, rule as OMetaRule):
		_rules[ruleName] = rule
	
	def Apply(context as OMetaGrammar, rule as string, input as OMetaInput):
		
		key = MemoKey(rule, input)
		memo as OMetaMatch
		if _memo.TryGetValue(key, memo):
			return memo
			
		_memo[key] = FailedMatch(input)
		memo = Eval(context, rule, input)
		_memo[key] = memo
		return memo
		
	def Eval(context as OMetaGrammar, rule as string, input as OMetaInput):
		found as OMetaRule
		if _rules.TryGetValue(rule, found):
			return found(context, input)
		return RuleMissing(context, rule, input)
		
	def SuperApply(context as OMetaGrammar, rule as string, input as OMetaInput):
		return Apply(context, rule, input)
		
	virtual def RuleMissing(context as OMetaGrammar, rule as string, input as OMetaInput) as OMetaMatch:
		raise "Rule '${rule}' missing!"
		
class OMetaGrammarPrototype(OMetaGrammarRoot):
	
	def constructor():
		InstallRule("whitespace", makeRule(char.IsWhiteSpace))
		InstallRule("letter", makeRule(char.IsLetter))
		InstallRule("digit", makeRule(char.IsDigit))
		InstallRule("_", makeRule({ o | return true }))
		
class OMetaDelegatingGrammar(OMetaGrammarRoot):
	
	_prototype as OMetaGrammar
	
	def constructor(prototype as OMetaGrammar):
		_prototype = prototype
		
	override def RuleMissing(context as OMetaGrammar, rule as string, input as OMetaInput):
		return SuperApply(context, rule, input)
		
	override def SuperApply(context as OMetaGrammar, rule as string, input as OMetaInput):
		return _prototype.SuperApply(context, rule, input)
		
def makeRule(predicate as System.Predicate[of object]) as OMetaRule:
	def rule(context as OMetaGrammar, input as OMetaInput) as OMetaMatch:
		if not input.IsEmpty and predicate(input.Head):
			return SuccessfulMatch(input.Tail, input.Head)
		return FailedMatch(input)
	return rule

{'fraction_non_alphanumeric': 0.07927800389311626, 'fraction_numerical': 0.0, 'mean_word_length': 4.930745015739769, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4926798', 'n_tokens_mistral': 1981, 'n_tokens_neox': 1791, 'n_words': 568}