<commit_before>package kagome

import (
	"fmt"
	"io"
	"unicode"
	"unicode/utf8"
)

const (
	maximumCost              = 1<<31 - 1
	maximumUnknownWordLength = 1024
	searchModeKanjiLength    = 2
	searchModeKanjiPenalty   = 3000
	searchModeOtherLength    = 7
	searchModeOtherPenalty   = 1700
)

type lattice struct {
	input  string
	list   [][]*node
	output []*node
	pool   *nodePool
	dic    *Dic
	udic   *UserDic
}

func newLattice() (la *lattice) {
	la = new(lattice)
	la.dic = NewSysDic()
	return
}

func (la *lattice) setNodePool(size int) {
	la.pool = newNodePool(size)
}

func (la *lattice) setDic(dic *Dic) {
	if dic != nil {
		la.dic = dic
	}
}

func (la *lattice) setUserDic(udic *UserDic) {
	la.udic = udic
}

func (la *lattice) addNode(pos, id, start int, class NodeClass, surface string) {
	var m Morph
	switch class {
	case DUMMY:
		//use default cost
	case KNOWN:
		m = la.dic.Morphs[id]
	case UNKNOWN:
		m = la.dic.UnkMorphs[id]
	case USER:
		// use default cost
	}
	n := la.pool.get()
	n.id = id
	n.start = start
	n.class = class
	n.left, n.right, n.weight = int32(m.LeftId), int32(m.RightId), int32(m.Weight)
	n.surface = surface
	n.prev = nil
	p := pos + utf8.RuneCountInString(surface)
	la.list[p] = append(la.list[p], n)
}

func (la *lattice) build(input string) {
	rc := utf8.RuneCountInString(input)
	la.pool.clear()
	la.input = input
	la.list = make([][]*node, rc+2)

	la.addNode(0, BosEosId, 0, DUMMY, input[0:0])
	la.addNode(rc+1, BosEosId, rc, DUMMY, input[rc:rc])

	runePos := -1
	for pos, ch := range input {
		runePos++
		anyMatches := false

		// (1) TODO: USER DIC
		if la.udic != nil {
			ids, lens := la.udic.Index.CommonPrefixSearchString(input[pos:])
			for i := range ids {
				la.addNode(runePos, ids[i], runePos, USER, input[pos:pos+lens[i]])
			}
			anyMatches = (len(ids) > 0)
		}
		if anyMatches {
			continue
		}

		// (2) KNOWN DIC
		if ids, lens := la.dic.Index.CommonPrefixSearchString(input[pos:]); len(ids) > 0 {
			anyMatches = true
			for i, id := range ids {
				dup, ok := la.dic.IndexDup[id]
				if !ok {
					dup = 1
				}
				for x := 0; x < dup; x++ {
					la.addNode(runePos, id+x, runePos, KNOWN, input[pos:pos+lens[i]])
				}
			}
		}
		// (3) UNKNOWN DIC
		class := la.dic.CharCategory[ch]
		if !anyMatches || la.dic.InvokeList[class] {
			endPos := pos + utf8.RuneLen(ch)
			unkWordLen := 1
			for i, w, size := endPos, 1, len(input); i < size; i += w {
				var c rune
				c, w = utf8.DecodeRuneInString(input[i:])
				if la.dic.CharCategory[c] != class {
					break
				}
				endPos += w
				unkWordLen++
				if unkWordLen >= maximumUnknownWordLength {
					break
				}
			}
			id := la.dic.UnkIndex[int(class)]
			for i, w := pos, 0; i < endPos; i += w {
				_, w = utf8.DecodeRuneInString(input[i:])
				end := i + w
				dup, ok := la.dic.UnkIndexDup[int(class)]
				if !ok {
					dup = 1
				}
				for x := 0; x < dup; x++ {
					la.addNode(runePos, id+x, runePos, UNKNOWN, input[pos:end])
				}
			}
		}
	}
	return
}

// String returns a debug string of a lattice.
func (la *lattice) String() string {
	str := ""
	for i, nodes := range la.list {
		str += fmt.Sprintf("[%v] :\n", i)
		for _, node := range nodes {
			str += fmt.Sprintf("%v\n", node)<commit_msg>fixed bugs of unknown words<commit_after>package kagome

import (
	"fmt"
	"io"
	"unicode"
	"unicode/utf8"
)

const (
	maximumCost              = 1<<31 - 1
	maximumUnknownWordLength = 1024
	searchModeKanjiLength    = 2
	searchModeKanjiPenalty   = 3000
	searchModeOtherLength    = 7
	searchModeOtherPenalty   = 1700
)

type lattice struct {
	input  string
	list   [][]*node
	output []*node
	pool   *nodePool
	dic    *Dic
	udic   *UserDic
}

func newLattice() (la *lattice) {
	la = new(lattice)
	la.dic = NewSysDic()
	return
}

func (la *lattice) setNodePool(size int) {
	la.pool = newNodePool(size)
}

func (la *lattice) setDic(dic *Dic) {
	if dic != nil {
		la.dic = dic
	}
}

func (la *lattice) setUserDic(udic *UserDic) {
	la.udic = udic
}

func (la *lattice) addNode(pos, id, start int, class NodeClass, surface string) {
	var m Morph
	switch class {
	case DUMMY:
		//use default cost
	case KNOWN:
		m = la.dic.Morphs[id]
	case UNKNOWN:
		m = la.dic.UnkMorphs[id]
	case USER:
		// use default cost
	}
	n := la.pool.get()
	n.id = id
	n.start = start
	n.class = class
	n.left, n.right, n.weight = int32(m.LeftId), int32(m.RightId), int32(m.Weight)
	n.surface = surface
	n.prev = nil
	p := pos + utf8.RuneCountInString(surface)
	la.list[p] = append(la.list[p], n)
}

func (la *lattice) build(input string) {
	rc := utf8.RuneCountInString(input)
	la.pool.clear()
	la.input = input
	la.list = make([][]*node, rc+2)

	la.addNode(0, BosEosId, 0, DUMMY, input[0:0])
	la.addNode(rc+1, BosEosId, rc, DUMMY, input[rc:rc])

	runePos := -1
	for pos, ch := range input {
		runePos++
		anyMatches := false

		// (1) TODO: USER DIC
		if la.udic != nil {
			ids, lens := la.udic.Index.CommonPrefixSearchString(input[pos:])
			for i := range ids {
				la.addNode(runePos, ids[i], runePos, USER, input[pos:pos+lens[i]])
			}
			anyMatches = (len(ids) > 0)
		}
		if anyMatches {
			continue
		}

		// (2) KNOWN DIC
		if ids, lens := la.dic.Index.CommonPrefixSearchString(input[pos:]); len(ids) > 0 {
			anyMatches = true
			for i, id := range ids {
				dup, ok := la.dic.IndexDup[id]
				if !ok {
					dup = 1
				}
				for x := 0; x < dup; x++ {
					la.addNode(runePos, id+x, runePos, KNOWN, input[pos:pos+lens[i]])
				}
			}
		}
		// (3) UNKNOWN DIC
		class := la.dic.CharCategory[ch]
		if !anyMatches || la.dic.InvokeList[class] {
			endPos := pos + utf8.RuneLen(ch)
			unkWordLen := 1
			if la.dic.GroupList[class] {
				for i, w, size := endPos, 1, len(input); i < size; i += w {
					var c rune
					c, w = utf8.DecodeRuneInString(input[i:])
					if la.dic.CharCategory[c] != class {
						break
					}
					endPos += w
					unkWordLen++
					if unkWordLen >= maximumUnknownWordLength {
						break
					}
				}
			}
			id := la.dic.UnkIndex[int(class)]
			for i, w := pos, 0; i < endPos; i += w {
				_, w = utf8.DecodeRuneInString(input[i:])
				end := i + w
				dup, ok := la.dic.UnkIndexDup[int(class)]
				if !ok {
					dup = 1
				}
				for x := 0; x < dup; x++ {
					la.addNode(runePos, id+x, runePos, UNKNOWN, input[pos:end])
				}
			}
		}
	}
	return
}

// String returns a debug string of a lattice.
func (la *lattice) String() string {
	str := ""
	for i, nodes := range la.list {
		str += fmt.Sprintf("[%v] :\n", i)
		for _, node := range nodes {
			str += fmt.Sprintf("%v\n", node)
{'fraction_non_alphanumeric': 0.15833973866256726, 'fraction_numerical': 0.01506533435818601, 'mean_word_length': 2.8158357771261, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1584698', 'n_tokens_mistral': 3114, 'n_tokens_neox': 2734, 'n_words': 655}