func BuildUnknowns(sentence []rune, toks []Token) []Token {
	/*
	 * We will iterate through the sentence
	 * Till the positions of each token is reached we will itearte through the sentence and build each word
	 * Upon reaching a position, it will add the word to the words list
	 * Once all the unknown words are recorded, we will split each word based on space
	 * The resultant words are consolidated as tokens
	 */
	result := []Token{}
	words := [][]rune{}
	tok := 0
	word := []rune{}
	tokenMap := map[string]Token{}

	//iterating through the sentence
	for p := 0; p < len(sentence); p++ {

		//checking whether the rune is outside the token
		if (len(toks) > tok && p < toks[tok].Pos) || len(toks) <= tok {
			word = append(word, sentence[p])
		} else if len(toks) > tok && toks[tok].Pos == p {

			//need to append only if there exist a word
			if len(word) > 0 {
				words = append(words, word)
				word = []rune{}
			}
			tokenMap[string(toks[tok].Word)] = toks[tok]
			words = append(words, toks[tok].Word)
			p = toks[tok].Pos + len(toks[tok].Word)
			tok++
		}
	}

	//if any pending word, copy it to the words list
	if len(word) > 0 {
		words = append(words, word)
	}

	//split the words with space
	for i, w := range words {
		// ws := strings.Split(string(w), " ")
		// for i, wr := range ws {
		//no need to take empty strings
		if len(w) == 0 {
			continue
		}
		tok := Token{
			Word: []rune(w),
		}
		oTok, ok := tokenMap[string(w)]
		if ok {
			tok.Nodes = oTok.Nodes
		} else {
			tok.Nodes = []Node{&UnknownNode{UID: fmt.Sprint("U", i), Word: []rune(w)}}
		}
		result = append(result, tok)
		// }
	}

	return result
}
//BuildUnknowns build unknown nodes.
//It return the tokens with unidentified words in the sentence transfomed to tokens with unknowns
BuildUnknowns build unknown nodes.
It return the tokens with unidentified words in the sentence transfomed to tokens with unknowns
// BuildUnknowns build unknown nodes . // It return the tokens with unidentified words in the sentence transfomed to tokens with unknowns
func BuildUnknowns(sentence []rune, toks []Token) []Token {
	result := []Token{}
	words := [][]rune{}
	tok := 0
	word := []rune{}
	tokenMap := map[string]Token{}
	for p := 0; p < len(sentence); p++ {
		if (len(toks) > tok && p < toks[tok].Pos) || len(toks) <= tok {
			word = append(word, sentence[p])
		} else if len(toks) > tok && toks[tok].Pos == p {
			if len(word) > 0 {
				words = append(words, word)
				word = []rune{}
			}
			tokenMap[string(toks[tok].Word)] = toks[tok]
			words = append(words, toks[tok].Word)
			p = toks[tok].Pos + len(toks[tok].Word)
			tok++
		}
	}
	if len(word) > 0 {
		words = append(words, word)
	}
	for i, w := range words {
		if len(w) == 0 {
			continue
		}
		tok := Token{
			Word: []rune(w),
		}
		oTok, ok := tokenMap[string(w)]
		if ok {
			tok.Nodes = oTok.Nodes
		} else {
			tok.Nodes = []Node{&UnknownNode{UID: fmt.Sprint("U", i), Word: []rune(w)}}
		}
		result = append(result, tok)
	}
	return result
}
func BuildUnknowns ( sentence [ ] rune , toks [ ] Token ) [ ] Token { result := [ ] Token { } words := [ ] [ ] rune { } tok := 0 word := [ ] rune { } tokenMap := map [ string ] Token { } for p := 0 ; p < len ( sentence ) ; p ++ { if ( len ( toks ) > tok && p < toks [ tok ] . Pos ) || len ( toks ) <= tok { word = append ( word , sentence [ p ] ) } else if len ( toks ) > tok && toks [ tok ] . Pos == p { if len ( word ) > 0 { words = append ( words , word ) word = [ ] rune { } } tokenMap [ string ( toks [ tok ] . Word ) ] = toks [ tok ] words = append ( words , toks [ tok ] . Word ) p = toks [ tok ] . Pos + len ( toks [ tok ] . Word ) tok ++ } } if len ( word ) > 0 { words = append ( words , word ) } for i , w := range words { if len ( w ) == 0 { continue } tok := Token { Word : [ ] rune ( w ) , } oTok , ok := tokenMap [ string ( w ) ] if ok { tok . Nodes = oTok . Nodes } else { tok . Nodes = [ ] Node { & UnknownNode { UID : fmt . Sprint ( " " , i ) , Word : [ ] rune ( w ) } } } result = append ( result , tok ) } return result }
BuildUnknowns build unknown nodes.
BuildUnknowns build unknown nodes .
/*
	 * We will iterate through the sentence
	 * Till the positions of each token is reached we will itearte through the sentence and build each word
	 * Upon reaching a position, it will add the word to the words list
	 * Once all the unknown words are recorded, we will split each word based on space
	 * The resultant words are consolidated as tokens
	 */ //iterating through the sentence //checking whether the rune is outside the token //need to append only if there exist a word //if any pending word, copy it to the words list //split the words with space // ws := strings.Split(string(w), " ") // for i, wr := range ws { //no need to take empty strings // }