<commit_before>// Copyright 2015 <NAME>. All rights reserved.
// Use of this source code is governed by a MIT license
// that can be found in the LICENSE file.

package build

import (
	"fmt"
	"github.com/achtern/kluver/lexer"
)

type Shader struct {
	version  int
	vertex   GLSL
	fragment GLSL
	global   GLSL
}

type GLSL struct {
	tokens Tokens
}

type Tokens []lexer.Token

func (s *Shader) String() string {
	return fmt.Sprintf("Shader(%d, vertex=%q, fragment=%q, global=%q)", s.version, s.vertex.tokens, s.fragment.tokens, s.global.tokens)
}

func Build(tokenStream <-chan lexer.Token) string {
	global := make(Tokens, 0)
	vertex := make(Tokens, 0)
	fragment := make(Tokens, 0)

	shader := Shader{}

	// phase 0 : global
	// phase 1 : vertex
	// phase 2 : fragment
	phase := 0
	for token := range tokenStream {
		switch token.Typ {
		case lexer.TokenVertex:
			phase = 1
		case lexer.TokenFragment:
			phase = 2
		case lexer.TokenEnd:
			phase = 0
		}

		switch phase {
		case 0:
			global = append(global, token)
		case 1:
			vertex = append(vertex, token)
		case 2:
			fragment = append(fragment, token)
		default:
			panic("unknow phase")
		}

	}

	shader.global = GLSL{global}
	shader.vertex = GLSL{vertex}
	shader.fragment = GLSL{fragment}

	return shader.String()
}
<commit_msg>Ignoring void tokens<commit_after>// Copyright 2015 <NAME>. All rights reserved.
// Use of this source code is governed by a MIT license
// that can be found in the LICENSE file.

package build

import (
	"fmt"
	"github.com/achtern/kluver/lexer"
)

type Shader struct {
	version  int
	vertex   GLSL
	fragment GLSL
	global   GLSL
}

type GLSL struct {
	tokens Tokens
}

type Tokens []lexer.Token

func (s *Shader) String() string {
	return fmt.Sprintf("Shader(%d, vertex=%q, fragment=%q, global=%q)", s.version, s.vertex.tokens, s.fragment.tokens, s.global.tokens)
}

func Build(tokenStream <-chan lexer.Token) string {
	global := make(Tokens, 0)
	vertex := make(Tokens, 0)
	fragment := make(Tokens, 0)

	shader := Shader{}

	// phase 0 : global
	// phase 1 : vertex
	// phase 2 : fragment
	phase := 0
	for token := range tokenStream {
		switch token.Typ {
		case lexer.TokenVertex:
			phase = 1
		case lexer.TokenFragment:
			phase = 2
		case lexer.TokenEnd:
			phase = 0
		case lexer.TokenVoid:
			continue
		}

		switch phase {
		case 0:
			global = append(global, token)
		case 1:
			vertex = append(vertex, token)
		case 2:
			fragment = append(fragment, token)
		default:
			panic("unknow phase")
		}
	}

	shader.global = GLSL{global}
	shader.vertex = GLSL{vertex}
	shader.fragment = GLSL{fragment}

	return shader.String()
}

{'fraction_non_alphanumeric': 0.11984732824427481, 'fraction_numerical': 0.012977099236641221, 'mean_word_length': 3.332231404958678, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2042232', 'n_tokens_mistral': 1005, 'n_tokens_neox': 908, 'n_words': 279}