<commit_before>package nstcc

import (
	"reflect"
	"testing"
)

func TestParser(t *testing.T) {
	// Map identifier strings like "main" to (negative) placeholder token
	// values.
	t2s := map[token]string{}
	s2t := map[string]token{}
	ident := func(s string) token {
		x, ok := s2t[s]
		if !ok {
			x = token(^len(s2t))
			s2t[s] = x
			t2s[x] = s
		}
		return x
	}

	testCases := []struct {
		desc string
		src  string
		want []token
	}{{
		"hello world",
		// TODO: s/XXX/42/
		"int main(int argc, char** argv) { return XXX; }",
		[]token{
			tokInt,
			ident("main"),
			'(',
			tokInt,
			ident("argc"),
			',',
			tokChar,
			'*',
			'*',
			ident("argv"),
			')',
			'{',
			tokReturn,
			ident("XXX"),
			';',
			'}',
		},
	}, {
		"slash star",
		"int /*foo*/ x;",
		[]token{
			tokInt,
			ident("x"),
			';',
		},
	}, {
		"slash slash",
		"int x;\n// int y;\nint z;\n",
		[]token{
			tokInt,
			ident("x"),
			';',
			tokInt,
			ident("z"),
			';',
		},
	}}

	for _, tc := range testCases {
		c := newCompiler(nil, nil, []byte(tc.src))
		var got []token
		for {
			if err := c.next(); err != nil {
				t.Fatal(err)
			}
			if c.tok == tokEOF {
				break
			}
			got = append(got, c.tok)
		}

		// Replace the placeholder identifier tokens with their real values.
		for i, x := range tc.want {
			if x >= 0 {
				continue
			}
			ts, err := c.idents.byStr([]byte(t2s[x]))
			if err != nil {
				t.Fatal(err)
			}
			tc.want[i] = ts.tok
		}

		if !reflect.DeepEqual(got, tc.want) {
			t.Fatalf("parsing %q:\ngot  %v\nwant %v", tc.desc, got, tc.want)
		}
	}
}<commit_msg>Make TestParser allow arbitrary 'want' token types.<commit_after>package nstcc

import (
	"reflect"
	"testing"
)

func TestParser(t *testing.T) {
	testCases := []struct {
		desc string
		src  string
		want []interface{}
	}{{
		"hello world",
		// TODO: s/XXX/42/
		"int main(int argc, char** argv) { return XXX; }",
		[]interface{}{
			tokInt,
			"main",
			'(',
			tokInt,
			"argc",
			',',
			tokChar,
			'*',
			'*',
			"argv",
			')',
			'{',
			tokReturn,
			"XXX",
			';',
			'}',
		},
	}, {
		"slash star",
		"int /*foo*/ x;",
		[]interface{}{
			tokInt,
			"x",
			';',
		},
	}, {
		"slash slash",
		"int x;\n// int y;\nint z;\n",
		[]interface{}{
			tokInt,
			"x",
			';',
			tokInt,
			"z",
			';',
		},
	}}

	for _, tc := range testCases {
		c := newCompiler(nil, nil, []byte(tc.src))
		var got []token
		for {
			if err := c.next(); err != nil {
				t.Fatal(err)
			}
			if c.tok == tokEOF {
				break
			}
			got = append(got, c.tok)
		}

		// Replace the placeholder tokens with their real values.
		want := make([]token, len(tc.want))
		for i, x := range tc.want {
			switch x := x.(type) {
			case rune:
				want[i] = token(x)
			case string:
				ts, err := c.idents.byStr([]byte(x))
				if err != nil {
					t.Fatal(err)
				}
				want[i] = ts.tok
			case token:
				want[i] = x
			default:
				t.Fatalf("invalid type %T", x)
			}
		}

		if !reflect.DeepEqual(got, want) {
			t.Errorf("parsing %q:\ngot  %v\nwant %v", tc.desc, got, want)
		}
	}
}
{'fraction_non_alphanumeric': 0.21019736842105263, 'fraction_numerical': 0.003947368421052632, 'mean_word_length': 2.2663802363050483, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7609135', 'n_tokens_mistral': 1534, 'n_tokens_neox': 1274, 'n_words': 289}