<commit_before>// Copyright 2015 <NAME>. All rights reserved.
// Use of this source code is governed by a MIT license
// that can be found in the LICENSE file.

package builder

import (
	"bytes"
	"github.com/achtern/kluver/lexer"
)

func Build(tokens <-chan lexer.Token) string {
	var buffer bytes.Buffer

	for token := range tokens {
		buffer.WriteString(token.String())
		buffer.WriteString("\n")
	}

	return buffer.String()
}<commit_msg>builder stores all non void tokens in a slice<commit_after>// Copyright 2015 <NAME>. All rights reserved.
// Use of this source code is governed by a MIT license
// that can be found in the LICENSE file.

package builder

import (
	"bytes"
	"github.com/achtern/kluver/lexer"
)

func Build(tokenStream <-chan lexer.Token) string {
	tokens := make([]lexer.Token, 0)

	var buffer bytes.Buffer

	for token := range tokenStream {
		if token.Typ == lexer.TokenVoid {
			continue
		}
		tokens = append(tokens, token)
	}

	for _, token := range tokens {
		buffer.WriteString(token.String())
		buffer.WriteString("\n")
	}

	return buffer.String()
}
{'fraction_non_alphanumeric': 0.11574074074074074, 'fraction_numerical': 0.008333333333333333, 'mean_word_length': 3.6796536796536796, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5168729', 'n_tokens_mistral': 366, 'n_tokens_neox': 336, 'n_words': 116}