<commit_before>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package ast

import (
	"bytes"
	"io/ioutil"
	"path/filepath"
	"reflect"
	"strings"
	"testing"
)

func TestParseAll(t *testing.T) {
	dir := filepath.Join("..", "..", "sys")
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		t.Fatalf("failed to read sys dir: %v", err)
	}
	for _, file := range files {
		if file.IsDir() || !strings.HasSuffix(file.Name(), ".txt") {
			continue
		}
		data, err := ioutil.ReadFile(filepath.Join(dir, file.Name()))
		if err != nil {
			t.Fatalf("failed to read file: %v", err)
		}
		t.Run(file.Name(), func(t *testing.T) {
			eh := func(pos Pos, msg string) {
				t.Fatalf("%v: %v", pos, msg)
			}
			desc := Parse(data, file.Name(), eh)
			if desc == nil {
				t.Fatalf("parsing failed, but no error produced")
			}
			data2 := Format(desc)
			desc2 := Parse(data2, file.Name(), eh)
			if desc2 == nil {
				t.Fatalf("parsing failed, but no error produced")
			}
			if len(desc.Nodes) != len(desc2.Nodes) {
				t.Fatalf("formatting number of top level decls: %v/%v",
					len(desc.Nodes), len(desc2.Nodes))<commit_msg>pkg/ast: fix TestParseAll<commit_after>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package ast

import (
	"bytes"
	"io/ioutil"
	"path/filepath"
	"reflect"
	"strings"
	"testing"
)

func TestParseAll(t *testing.T) {
	files, err := filepath.Glob(filepath.Join("..", "..", "sys", "linux", "*.txt"))
	if err != nil || len(files) == 0 {
		t.Fatalf("failed to read sys dir: %v", err)
	}
	for _, file := range files {
		data, err := ioutil.ReadFile(file)
		if err != nil {
			t.Fatalf("failed to read file: %v", err)
		}
		t.Run(file, func(t *testing.T) {
			eh := func(pos Pos, msg string) {
				t.Fatalf("%v: %v", pos, msg)
			}
			desc := Parse(data, file, eh)
			if desc == nil {
				t.Fatalf("parsing failed, but no error produced")
			}
			data2 := Format(desc)
			desc2 := Parse(data2, file, eh)
			if desc2 == nil {
				t.Fatalf("parsing failed, but no error produced")
			}
			if len(desc.Nodes) != len(desc2.Nodes) {
				t.Fatalf("formatting number of top level decls: %v/%v",
					len(desc.Nodes), len(desc2.Nodes))
{'fraction_non_alphanumeric': 0.16800337695230055, 'fraction_numerical': 0.009708737864077669, 'mean_word_length': 3.247311827956989, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7283033', 'n_tokens_mistral': 965, 'n_tokens_neox': 869, 'n_words': 257}