<commit_before>// Copyright (c) 2012-2019 <NAME>
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package htrie

import (
	"fmt"
)

const globChar byte = 1

// A globPathNode represents a path checker that supports globbing comparisons
type globPathNode struct {
	subtrees map[byte]*globPathNode
	// used to avoid map lookup when there is only one subtree candidate
	oneShot *globPathNode
	// is this path component a glob
	isGlob bool
	// determines whether a node can be a match even if it isn't a leaf node;
	// this becomes necessary due to the possibility of longer and shorter
	// paths overlapping
	canMatch bool
	// optimization to avoid an extra map lookup on every char
	hasGlobChild bool
	// char for this node
	nodeChar byte
	// is this a case insensitive comparison tree?
	icase bool
}

func (gpn *globPathNode) addPath(s string) error {
	if gpn.subtrees == nil {
		return fmt.Errorf("got nil <gpn>.subtrees in receiver")
	}

	curnode := gpn
	prevnode := curnode
	mlen := len(s)
	//for _, part := range s {
	for i := 0; i < mlen; i++ {
		part := s[i]

		// if icase, use lowercase letters for comparisons
		if gpn.icase && 'A' <= part && part <= 'Z' {
			part = part + 32
		}

		var c byte
		if part == '*' {
			c = globChar
		} else {
			c = part
		}

		subt := curnode.subtrees
		if subt[c] == nil {
			subt[c] = newGlobPathNode(gpn.icase)
		}

		subt[c].nodeChar = part

		// setup oneshot as an optimizaiton if there is only one subcandidate...
		if len(subt) == 1 {
			curnode.oneShot = subt[c]
		} else {
			curnode.oneShot = nil
		}

		prevnode = curnode
		curnode = subt[c]
		if part == '*' {
			prevnode.hasGlobChild = true
			curnode.isGlob = true
		}
	}

	// this is the end of the path, so this node can be a match, even if future
	// urls add children to it (a longer url).
	curnode.canMatch = true
	return nil
}

func (gpn *globPathNode) globConsume(s string, index, mlen int) bool {
	// we have a glob and no follow-on chars, so we can consume
	// till the end and then match. early return
	if gpn.canMatch {
		return true
	}

	oneShotLookahead := false
	oneShotStep := false
	// optimize common single char after * globbing
	// eg. .../*/...
	if gpn.oneShot != nil {
		oneShotLookahead = true
		oneShotStep = true
	}

	// otherwise we have some work to do...
	curnode := gpn
	// don't need to iter runes since we have ascii
	for i := index; i < mlen; i++ {
		part := s[i]
		// if icase, use lowercase letters for comparisons
		if gpn.icase && 'A' <= part && part <= 'Z' {
			part = part + 32
		}

		// we know the glob has one one subcandidate (next char), so consume until
		// we hit one of those
		if oneShotStep {
			if part != curnode.oneShot.nodeChar {
				continue
			}
			// got the oneshot expected char finally, so unset oneshot
			// stepping, and proceed<commit_msg>add a newline for eathetics<commit_after>// Copyright (c) 2012-2019 <NAME>
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package htrie

import (
	"fmt"
)

const globChar byte = 1

// A globPathNode represents a path checker that supports globbing comparisons
type globPathNode struct {
	subtrees map[byte]*globPathNode
	// used to avoid map lookup when there is only one subtree candidate
	oneShot *globPathNode
	// is this path component a glob
	isGlob bool
	// determines whether a node can be a match even if it isn't a leaf node;
	// this becomes necessary due to the possibility of longer and shorter
	// paths overlapping
	canMatch bool
	// optimization to avoid an extra map lookup on every char
	hasGlobChild bool
	// char for this node
	nodeChar byte
	// is this a case insensitive comparison tree?
	icase bool
}

func (gpn *globPathNode) addPath(s string) error {
	if gpn.subtrees == nil {
		return fmt.Errorf("got nil <gpn>.subtrees in receiver")
	}

	curnode := gpn
	prevnode := curnode
	mlen := len(s)
	//for _, part := range s {
	for i := 0; i < mlen; i++ {
		part := s[i]

		// if icase, use lowercase letters for comparisons
		if gpn.icase && 'A' <= part && part <= 'Z' {
			part = part + 32
		}

		var c byte
		if part == '*' {
			c = globChar
		} else {
			c = part
		}

		subt := curnode.subtrees
		if subt[c] == nil {
			subt[c] = newGlobPathNode(gpn.icase)
		}

		subt[c].nodeChar = part

		// setup oneshot as an optimizaiton if there is only one subcandidate...
		if len(subt) == 1 {
			curnode.oneShot = subt[c]
		} else {
			curnode.oneShot = nil
		}

		prevnode = curnode
		curnode = subt[c]
		if part == '*' {
			prevnode.hasGlobChild = true
			curnode.isGlob = true
		}
	}

	// this is the end of the path, so this node can be a match, even if future
	// urls add children to it (a longer url).
	curnode.canMatch = true
	return nil
}

func (gpn *globPathNode) globConsume(s string, index, mlen int) bool {
	// we have a glob and no follow-on chars, so we can consume
	// till the end and then match. early return
	if gpn.canMatch {
		return true
	}

	oneShotLookahead := false
	oneShotStep := false
	// optimize common single char after * globbing
	// eg. .../*/...
	if gpn.oneShot != nil {
		oneShotLookahead = true
		oneShotStep = true
	}

	// otherwise we have some work to do...
	curnode := gpn
	// don't need to iter runes since we have ascii
	for i := index; i < mlen; i++ {
		part := s[i]

		// if icase, use lowercase letters for comparisons
		if gpn.icase && 'A' <= part && part <= 'Z' {
			part = part + 32
		}

		// we know the glob has one one subcandidate (next char), so consume until
		// we hit one of those
		if oneShotStep {
			if part != curnode.oneShot.nodeChar {
				continue
			}
			// got the oneshot expected char finally, so unset oneshot
			// stepping, and proceed
{'fraction_non_alphanumeric': 0.10308201288525161, 'fraction_numerical': 0.005223750652968832, 'mean_word_length': 2.803973509933775, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6482747', 'n_tokens_mistral': 2165, 'n_tokens_neox': 1953, 'n_words': 765}