<commit_before>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package fstest implements support for testing implementations and users of file systems.
package fstest

import (
	"errors"
	"fmt"
	"io"
	"io/fs"
	"io/ioutil"
	"path"
	"reflect"
	"sort"
	"strings"
	"testing/iotest"
)

// TestFS tests a file system implementation.
// It walks the entire tree of files in fsys,
// opening and checking that each file behaves correctly.
// It also checks that the file system contains at least the expected files.
// As a special case, if no expected files are listed, fsys must be empty.
// Otherwise, fsys must only contain at least the listed files: it can also contain others.
//
// If TestFS finds any misbehaviors, it returns an error reporting all of them.
// The error text spans multiple lines, one per detected misbehavior.
//
// Typical usage inside a test is:
//
//	if err := fstest.TestFS(myFS, "file/that/should/be/present"); err != nil {
//		t.Fatal(err)
//	}
//
func TestFS(fsys fs.FS, expected ...string) error {
	if err := testFS(fsys, expected...); err != nil {
		return err
	}
	for _, name := range expected {
		if i := strings.Index(name, "/"); i >= 0 {
			dir, dirSlash := name[:i], name[:i+1]
			var subExpected []string
			for _, name := range expected {
				if strings.HasPrefix(name, dirSlash) {
					subExpected = append(subExpected, name[len(dirSlash):])
				}
			}
			sub, err := fs.Sub(fsys, dir)
			if err != nil {
				return err
			}
			if err := testFS(sub, subExpected...); err != nil {
				return fmt.Errorf("testing fs.Sub(fsys, %s): %v", dir, err)
			}
			break // one sub-test is enough
		}
	}
	return nil
}

func testFS(fsys fs.FS, expected ...string) error {
	t := fsTester{fsys: fsys}
	t.checkDir(".")
	t.checkOpen(".")
	found := make(map[string]bool)
	for _, dir := range t.dirs {
		found[dir] = true
	}
	for _, file := range t.files {
		found[file] = true
	}
	delete(found, ".")
	if len(expected) == 0 && len(found) > 0 {
		var list []string
		for k := range found {
			if k != "." {
				list = append(list, k)
			}
		}
		sort.Strings(list)
		if len(list) > 15 {
			list = append(list[:10], "...")
		}
		t.errorf("expected empty file system but found files:\n%s", strings.Join(list, "\n"))
	}
	for _, name := range expected {
		if !found[name] {
			t.errorf("expected but not found: %s", name)
		}
	}
	if len(t.errText) == 0 {
		return nil
	}
	return errors.New("TestFS found errors:\n" + string(t.errText))
}

// An fsTester holds state for running the test.
type fsTester struct {
	fsys    fs.FS
	errText []byte
	dirs    []string
	files   []string
}

// errorf adds an error line to errText.
func (t *fsTester) errorf(format string, args ...interface{}) {
	if len(t.errText) > 0 {
		t.errText = append(t.errText, '\n')
	}
	t.errText = append(t.errText, fmt.Sprintf(format, args...)...)
}

func (t *fsTester) openDir(dir string) fs.ReadDirFile {
	f, err := t.fsys.Open(dir)
	if err != nil {
		t.errorf("%s: Open: %v", dir, err)
		return nil
	}
	d, ok := f.(fs.ReadDirFile)
	if !ok {
		f.Close()
		t.errorf("%s: Open returned File type %T, not a io.ReadDirFile", dir, f)
		return nil
	}
	return d
}

// checkDir checks the directory dir, which is expected to exist
// (it is either the root or was found in a directory listing with IsDir true).
func (t *fsTester) checkDir(dir string) {
	// Read entire directory.
	t.dirs = append(t.dirs, dir)
	d := t.openDir(dir)
	if d == nil {
		return
	}
	list, err := d.ReadDir(-1)<commit_msg>testing/fstest: fix typo in error message<commit_after>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package fstest implements support for testing implementations and users of file systems.
package fstest

import (
	"errors"
	"fmt"
	"io"
	"io/fs"
	"io/ioutil"
	"path"
	"reflect"
	"sort"
	"strings"
	"testing/iotest"
)

// TestFS tests a file system implementation.
// It walks the entire tree of files in fsys,
// opening and checking that each file behaves correctly.
// It also checks that the file system contains at least the expected files.
// As a special case, if no expected files are listed, fsys must be empty.
// Otherwise, fsys must only contain at least the listed files: it can also contain others.
//
// If TestFS finds any misbehaviors, it returns an error reporting all of them.
// The error text spans multiple lines, one per detected misbehavior.
//
// Typical usage inside a test is:
//
//	if err := fstest.TestFS(myFS, "file/that/should/be/present"); err != nil {
//		t.Fatal(err)
//	}
//
func TestFS(fsys fs.FS, expected ...string) error {
	if err := testFS(fsys, expected...); err != nil {
		return err
	}
	for _, name := range expected {
		if i := strings.Index(name, "/"); i >= 0 {
			dir, dirSlash := name[:i], name[:i+1]
			var subExpected []string
			for _, name := range expected {
				if strings.HasPrefix(name, dirSlash) {
					subExpected = append(subExpected, name[len(dirSlash):])
				}
			}
			sub, err := fs.Sub(fsys, dir)
			if err != nil {
				return err
			}
			if err := testFS(sub, subExpected...); err != nil {
				return fmt.Errorf("testing fs.Sub(fsys, %s): %v", dir, err)
			}
			break // one sub-test is enough
		}
	}
	return nil
}

func testFS(fsys fs.FS, expected ...string) error {
	t := fsTester{fsys: fsys}
	t.checkDir(".")
	t.checkOpen(".")
	found := make(map[string]bool)
	for _, dir := range t.dirs {
		found[dir] = true
	}
	for _, file := range t.files {
		found[file] = true
	}
	delete(found, ".")
	if len(expected) == 0 && len(found) > 0 {
		var list []string
		for k := range found {
			if k != "." {
				list = append(list, k)
			}
		}
		sort.Strings(list)
		if len(list) > 15 {
			list = append(list[:10], "...")
		}
		t.errorf("expected empty file system but found files:\n%s", strings.Join(list, "\n"))
	}
	for _, name := range expected {
		if !found[name] {
			t.errorf("expected but not found: %s", name)
		}
	}
	if len(t.errText) == 0 {
		return nil
	}
	return errors.New("TestFS found errors:\n" + string(t.errText))
}

// An fsTester holds state for running the test.
type fsTester struct {
	fsys    fs.FS
	errText []byte
	dirs    []string
	files   []string
}

// errorf adds an error line to errText.
func (t *fsTester) errorf(format string, args ...interface{}) {
	if len(t.errText) > 0 {
		t.errText = append(t.errText, '\n')
	}
	t.errText = append(t.errText, fmt.Sprintf(format, args...)...)
}

func (t *fsTester) openDir(dir string) fs.ReadDirFile {
	f, err := t.fsys.Open(dir)
	if err != nil {
		t.errorf("%s: Open: %v", dir, err)
		return nil
	}
	d, ok := f.(fs.ReadDirFile)
	if !ok {
		f.Close()
		t.errorf("%s: Open returned File type %T, not a fs.ReadDirFile", dir, f)
		return nil
	}
	return d
}

// checkDir checks the directory dir, which is expected to exist
// (it is either the root or was found in a directory listing with IsDir true).
func (t *fsTester) checkDir(dir string) {
	// Read entire directory.
	t.dirs = append(t.dirs, dir)
	d := t.openDir(dir)
	if d == nil {
		return
	}
	list, err := d.ReadDir(-1)
{'fraction_non_alphanumeric': 0.14817903808729496, 'fraction_numerical': 0.004170141784820684, 'mean_word_length': 3.030812324929972, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5974951', 'n_tokens_mistral': 2722, 'n_tokens_neox': 2546, 'n_words': 866}