func TokensMatch(tokens []*TokenNode, patterns [][]*TokenNode, matchPrefixes bool, variables map[string]*Variable) (bool, []VariableValue, []*TokenNode, error) {
	for i := range patterns {
		if match, values, tokensContinue, err := TokensSingleMatch(tokens, patterns[i], matchPrefixes, variables); err != nil {
			return false, nil, nil, err
		} else if match {
			return true, values, tokensContinue, nil
		}
	}
	return false, nil, nil, nil
}
func ManyTokensMatch(tokens [][]*TokenNode, patterns [][]*TokenNode, matchPrefixes bool, variables map[string]*Variable) (bool, []VariableValue, []*TokenNode, error) {
for i := range tokens {
if match, values, tokensContinue, err := TokensMatch(tokens[i], patterns, matchPrefixes, variables); err != nil {
return false, nil, nil, err
} else if match {
return true, values, tokensContinue, nil
}
}
return false, nil, nil, nil
}
package search
import (
	"container/list"
	"fmt"
	"regexp"
	"sort"
	"strings"
	"sync"
	"time"
	"github.com/pkg/errors"
)
var debugSpan = 0
type TokensCache struct {
	entries map[string]map[string]*list.Element
	order   *list.List
	mux     *sync.Mutex
	limit   int
}
func MakeTokensCache(size int) *TokensCache {
	return &TokensCache{
		entries: make(map[string]map[string]*list.Element),
		order:   list.New(),
		mux:     &sync.Mutex{},
		limit:   size,
	}
}
type TokensCacheElement struct {
	Phrase string
	Lang   string
	Tokens []*TokenNode
}
func (tc *TokensCache) Has(phrase string, lang string) bool {
	tc.mux.Lock()
	defer tc.mux.Unlock()
	if _, ok := tc.entries[phrase]; ok {
		_, okLang := tc.entries[phrase][lang]
		return okLang
	}
	return false
}
func (tc *TokensCache) Get(phrase string, lang string) []*TokenNode {
	tc.mux.Lock()
	defer tc.mux.Unlock()
	if _, ok := tc.entries[phrase]; ok {
		if element, okLang := tc.entries[phrase][lang]; okLang {
			tc.order.MoveToFront(element)
			return element.Value.(*TokensCacheElement).Tokens
		}
	}
	return nil
}
func (tc *TokensCache) Set(phrase string, lang string, tokens []*TokenNode) {
	tc.mux.Lock()
	defer tc.mux.Unlock()
	exist := false
	if _, exist = tc.entries[phrase]; exist {
		if element, exist := tc.entries[phrase][lang]; exist {
			tc.order.MoveToFront(element)
			element.Value = &TokensCacheElement{phrase, lang, tokens}
		}
	}
	if !exist {
		if len(tc.entries) >= tc.limit {
			element := tc.order.Remove(tc.order.Back()).(*TokensCacheElement)
			phraseEntries := tc.entries[element.Phrase]
			delete(phraseEntries, element.Lang)
			if len(phraseEntries) == 0 {
				delete(tc.entries, element.Phrase)
			}
		}
		if _, ok := tc.entries[phrase]; !ok {
			tc.entries[phrase] = make(map[string]*list.Element)
		}
		tc.entries[phrase][lang] = tc.order.PushFront(&TokensCacheElement{phrase, lang, tokens})
	}
}
func TokenNodesToString(root []*TokenNode, variables map[string]*Variable) string {
	printPhrases := TokenNodesToPhrases(root, variables, true  )
	parts := []string{}
	for i := range printPhrases {
		parts = append(parts, fmt.Sprintf("[%d]: %s", i, printPhrases[i].ToString()))
	}
	return strings.Join(parts, "\n")
}
func VariablesMapToString(variablesMap map[string][]string) string {
	parts := []string(nil)
	for k, v := range variablesMap {
		parts = append(parts, fmt.Sprintf("%s=[%s]", k, strings.Join(v, ",")))
	}
	sort.Strings(parts)
	return strings.Join(parts, ",")
}
func MakeVariablePhraseWithOrigin(variable, prefix, suffix string) *PhrasesWithOrigin {
	return &PhrasesWithOrigin{[]VariableValue{MakeVariableValue(variable, prefix, variable, suffix, variable, variable)}, nil, "", "", ""}
}
func MakePhrasesWithOrigin(prefix, phrase, suffix, token string) *PhrasesWithOrigin {
	return &PhrasesWithOrigin{[]VariableValue{MakeTextVariableValue(prefix, phrase, suffix, token)}, nil, "", "", ""}
}
func MakeTextVariableValue(prefix, phrase, suffix, token string) VariableValue {
	return MakeVariableValue("$Text", prefix, phrase, suffix, token, fmt.Sprintf("%s%s%s", prefix, phrase, suffix))
}
func MakeVariableValue(variable, prefix, phrase, suffix, token, value string) VariableValue {
	ret := VariableValue{
		Name:       variable,                                       
		Value:      value,                                          
		Tokenized:  []string{token},                                
		Origin:     phrase,                                         
		OriginFull: fmt.Sprintf("%s%s%s", prefix, phrase, suffix),  
	}
	return ret
}
type PhrasesWithOrigin struct {
	VariableValues []VariableValue
	variablesMap       map[string][]string
	variablesMapString string
	joinSpace          string
	originalJoin       string
}
func (p *PhrasesWithOrigin) Invalidate() {
	p.variablesMap = nil
	p.variablesMapString = ""
	p.joinSpace = ""
	p.originalJoin = ""
}
func CopyPhrasesWithOrigin(p []*PhrasesWithOrigin) []*PhrasesWithOrigin {
	ret := []*PhrasesWithOrigin(nil)
	for i := range p {
		pCopy := *p[i]
		pCopy.Invalidate()
		ret = append(ret, &pCopy)
	}
	return ret
}
var countVM = 0
var countVMFirst = 0
func (pwo *PhrasesWithOrigin) VariablesMap() (map[string][]string, string) {
	countVM++
	if len(pwo.variablesMap) == 0 {
		countVMFirst++
		pwo.variablesMap = make(map[string][]string)
		for i := range pwo.VariableValues {
			if pwo.VariableValues[i].Name != "$Text" {
				pwo.variablesMap[pwo.VariableValues[i].Name] = append(pwo.variablesMap[pwo.VariableValues[i].Name], pwo.VariableValues[i].Value)
			}
		}
		pwo.variablesMapString = VariablesMapToString(pwo.variablesMap)
	}
	return pwo.variablesMap, pwo.variablesMapString
}
func (p *PhrasesWithOrigin) Reduce(variableName, variableValue, prefix, suffix string) {
	tokenized := []string(nil)
	for i := range p.VariableValues {
		tokenized = append(tokenized, p.VariableValues[i].Tokenized...)
	}
	p.VariableValues = []VariableValue{VariableValue{
		Name:       variableName,
		Value:      variableValue,
		Tokenized:  tokenized,
		Origin:     p.OriginalJoin(),
		OriginFull: fmt.Sprintf("%s%s%s", prefix, p.OriginalJoin(), suffix),
	}}
	p.Invalidate()
}
func (p *PhrasesWithOrigin) Join(s string) string {
	if s == " " && p.joinSpace != "" {
		return p.joinSpace
	}
	parts := []string{}
	for i := range p.VariableValues {
		parts = append(parts, strings.Join(p.VariableValues[i].Tokenized, s))
	}
	ret := strings.Join(parts, s)
	if s == " " && p.joinSpace == "" {
		p.joinSpace = ret
	}
	return ret
}
func (p *PhrasesWithOrigin) OriginalJoin() string {
	if p.originalJoin == "" {
		parts := []string{}
		for i := range p.VariableValues {
			parts = append(parts, p.VariableValues[i].OriginFull)
		}
		p.originalJoin = strings.Join(parts, "")
	}
	return p.originalJoin
}
func (p *PhrasesWithOrigin) VariableValuesJoin() string {
	parts := []string{}
	for i := range p.VariableValues {
		parts = append(parts, fmt.Sprintf("%s:%s", p.VariableValues[i].Name, p.VariableValues[i].Value))
	}
	return strings.Join(parts, ",")
}
func (p *PhrasesWithOrigin) ToString() string {
	return fmt.Sprintf("[P:%s|O:%s|V:%s]", p.Join(" "), p.OriginalJoin(), p.VariableValuesJoin())
}
func TokenNodesToPhrases(root []*TokenNode, variables map[string]*Variable, reduceVariables bool) []*PhrasesWithOrigin {
	ret := []*PhrasesWithOrigin(nil)
	for i := range root {
		ret = append(ret, tokenNodeToPhrases(root[i], variables, reduceVariables)...)
	}
	return ret
}
func tokenNodeToPhrases(root *TokenNode, variables map[string]*Variable, reduceVariables bool) []*PhrasesWithOrigin {
	if len(root.Phrases) == 0 {
		root.Phrases = OriginalTokenNodesToPhrases(root.OriginalTokenNodes, variables, reduceVariables)
	}
	return root.Phrases
}
func OriginalTokenNodesToPhrases(otns []*OriginalTokenNode, variables map[string]*Variable, reduceVariables bool) []*PhrasesWithOrigin {
	ret := []*PhrasesWithOrigin(nil)
	for i := range otns {
		currentPhrases := []*PhrasesWithOrigin(nil)
		if variable, ok := variables[otns[i].OriginalVariableToString()]; ok {
			if reduceVariables {
				currentPhrases = (*variable).VariableToPhrases(otns[i].OriginalVariablePrefixToString(), otns[i].SkippedSuffixToString(), variables)
			} else {
				currentPhrases = append(currentPhrases, MakeVariablePhraseWithOrigin(otns[i].OriginalVariableToString(), otns[i].SkippedPrefixToString(), otns[i].SkippedSuffixToString()))
			}
		} else {
			currentPhrases = append(currentPhrases, MakePhrasesWithOrigin(otns[i].SkippedPrefixToString(), otns[i].OriginalPhrase, otns[i].SkippedSuffixToString(), otns[i].TokenNode.Token.Token))
		}
		phrases := OriginalTokenNodesToPhrases(otns[i].Children, variables, reduceVariables)
		for j := range phrases {
			for k := range currentPhrases {
				phrasesCopy := *phrases[j]
				phrasesCopy.VariableValues = append(currentPhrases[k].VariableValues, phrasesCopy.VariableValues...)
				ret = append(ret, &phrasesCopy)
			}
		}
		if otns[i].TokenNode.IsEnd || len(phrases) == 0 {
			ret = append(ret, currentPhrases...)
		}
	}
	return ret
}