<commit_before>package expreduce

import (
	"bytes"
	"sort"
	"strings"
)

type PDManager struct {
	patternDefined map[string]Ex
}

func EmptyPD() *PDManager {
	return &PDManager{make(map[string]Ex)}
}

func CopyPD(orig *PDManager) (dest *PDManager) {
	dest = EmptyPD()
	// We do not care that this iterates in a random order.
	for k, v := range (*orig).patternDefined {
		(*dest).patternDefined[k] = v.DeepCopy()
	}
	return
}

func (this *PDManager) Update(toAdd *PDManager) {
	// We do not care that this iterates in a random order.
	for k, v := range (*toAdd).patternDefined {
		(*this).patternDefined[k] = v
	}
}

func (this *PDManager) Len() int {
	return len(this.patternDefined)
}

func (this *PDManager) String() string {
	var buffer bytes.Buffer
	buffer.WriteString("{")
	// We sort the keys here such that converting identical PDManagers always
	// produces the same string.
	keys := []string{}
	for k := range this.patternDefined {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for _, k := range keys {
		v := this.patternDefined[k]
		buffer.WriteString(k)
		buffer.WriteString("_: ")
		buffer.WriteString(v.String())
		buffer.WriteString(", ")
	}
	if strings.HasSuffix(buffer.String(), ", ") {
		buffer.Truncate(buffer.Len() - 2)
	}
	buffer.WriteString("}")
	return buffer.String()
}

func (this *PDManager) Expression() Ex {
	res := NewExpression([]Ex{&Symbol{"List"}})
	// We sort the keys here such that converting identical PDManagers always
	// produces the same string.
	keys := []string{}
	for k := range this.patternDefined {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for _, k := range keys {
		v := this.patternDefined[k]
		res.appendEx(NewExpression([]Ex{
			&Symbol{"Rule"},
			&String{k},
			v,
		}))
	}
	return res
}

func DefineSequence(lhs parsedForm, sequence []Ex, pm *PDManager, sequenceHead string, es *EvalState) bool {
	pat, isPat := HeadAssertion(lhs.origForm, "Pattern")
	//optional, isOptional := HeadAssertion(lhs.origForm, "Optional")
	//if isOptional {
		//pat, isPat = HeadAssertion(optional.Parts[1], "Pattern")
	//}
	if !isPat {
		return true
	}
	sAsSymbol, sAsSymbolOk := pat.Parts[1].(*Symbol)
	var attemptDefine Ex = nil
	if sAsSymbolOk {
		sequenceHeadSym := &Symbol{sequenceHead}
		oneIdent := sequenceHeadSym.Attrs(&es.defined).OneIdentity
		if len(sequence) == 1 && (lhs.isBlank || oneIdent) {
			attemptDefine = sequence[0]
		} else if len(sequence) == 0 && lhs.isOptional && lhs.defaultExpr != nil {
			attemptDefine = lhs.defaultExpr
		} else if lhs.isImpliedBs {
			attemptDefine = NewExpression(append([]Ex{sequenceHeadSym}, sequence...))
		} else {
			head := &Symbol{"Sequence"}
			attemptDefine = NewExpression(append([]Ex{head}, sequence...))
		}

		defined, ispd := pm.patternDefined[sAsSymbol.Name]
		if ispd && !IsSameQ(defined, attemptDefine, &es.CASLogger) {
			es.Debugf("patterns do not match! continuing.")
			return false
		}
		pm.patternDefined[sAsSymbol.Name] = attemptDefine
	}
	return true
}
<commit_msg>Fix Optional.<commit_after>package expreduce

import (
	"bytes"
	"sort"
	"strings"
)

type PDManager struct {
	patternDefined map[string]Ex
}

func EmptyPD() *PDManager {
	return &PDManager{make(map[string]Ex)}
}

func CopyPD(orig *PDManager) (dest *PDManager) {
	dest = EmptyPD()
	// We do not care that this iterates in a random order.
	for k, v := range (*orig).patternDefined {
		(*dest).patternDefined[k] = v.DeepCopy()
	}
	return
}

func (this *PDManager) Update(toAdd *PDManager) {
	// We do not care that this iterates in a random order.
	for k, v := range (*toAdd).patternDefined {
		(*this).patternDefined[k] = v
	}
}

func (this *PDManager) Len() int {
	return len(this.patternDefined)
}

func (this *PDManager) String() string {
	var buffer bytes.Buffer
	buffer.WriteString("{")
	// We sort the keys here such that converting identical PDManagers always
	// produces the same string.
	keys := []string{}
	for k := range this.patternDefined {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for _, k := range keys {
		v := this.patternDefined[k]
		buffer.WriteString(k)
		buffer.WriteString("_: ")
		buffer.WriteString(v.String())
		buffer.WriteString(", ")
	}
	if strings.HasSuffix(buffer.String(), ", ") {
		buffer.Truncate(buffer.Len() - 2)
	}
	buffer.WriteString("}")
	return buffer.String()
}

func (this *PDManager) Expression() Ex {
	res := NewExpression([]Ex{&Symbol{"List"}})
	// We sort the keys here such that converting identical PDManagers always
	// produces the same string.
	keys := []string{}
	for k := range this.patternDefined {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for _, k := range keys {
		v := this.patternDefined[k]
		res.appendEx(NewExpression([]Ex{
			&Symbol{"Rule"},
			&String{k},
			v,
		}))
	}
	return res
}

func DefineSequence(lhs parsedForm, sequence []Ex, pm *PDManager, sequenceHead string, es *EvalState) bool {
	pat, isPat := HeadAssertion(lhs.origForm, "Pattern")
	optional, isOptional := HeadAssertion(lhs.origForm, "Optional")
	if isOptional {
		pat, isPat = HeadAssertion(optional.Parts[1], "Pattern")
	}
	if !isPat {
		return true
	}
	sAsSymbol, sAsSymbolOk := pat.Parts[1].(*Symbol)
	var attemptDefine Ex = nil
	if sAsSymbolOk {
		sequenceHeadSym := &Symbol{sequenceHead}
		oneIdent := sequenceHeadSym.Attrs(&es.defined).OneIdentity
		if len(sequence) == 1 && (lhs.isBlank || oneIdent || lhs.isOptional) {
			attemptDefine = sequence[0]
		} else if len(sequence) == 0 && lhs.isOptional && lhs.defaultExpr != nil {
			attemptDefine = lhs.defaultExpr
		} else if lhs.isImpliedBs {
			attemptDefine = NewExpression(append([]Ex{sequenceHeadSym}, sequence...))
		} else {
			head := &Symbol{"Sequence"}
			attemptDefine = NewExpression(append([]Ex{head}, sequence...))
		}

		defined, ispd := pm.patternDefined[sAsSymbol.Name]
		if ispd && !IsSameQ(defined, attemptDefine, &es.CASLogger) {
			es.Debugf("patterns do not match! continuing.")
			return false
		}
		pm.patternDefined[sAsSymbol.Name] = attemptDefine
	}
	return true
}

{'fraction_non_alphanumeric': 0.1482101037136166, 'fraction_numerical': 0.0020073603211776514, 'mean_word_length': 4.208188153310105, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5734239', 'n_tokens_mistral': 2186, 'n_tokens_neox': 2049, 'n_words': 516}