<commit_before>// Package rwvfs augments vfs to support write operations.
package rwvfs

import (
	"io"
	"os"
	"path/filepath"
	"syscall"

	"github.com/kr/fs"

	"golang.org/x/tools/godoc/vfs"
)

type FileSystem interface {
	vfs.FileSystem

	// Create creates the named file, truncating it if it already exists.
	Create(path string) (io.WriteCloser, error)

	// Mkdir creates a new directory. If name is already a directory, Mkdir
	// returns an error (that can be detected using os.IsExist).
	Mkdir(name string) error

	// Remove removes the named file or directory.
	Remove(name string) error
}

// MkdirAll creates a directory named path, along with any necessary parents. If
// path is already a directory, MkdirAll does nothing and returns nil.
func MkdirAll(fs FileSystem, path string) error {
	// adapted from os/MkdirAll

	dir, err := fs.Stat(path)
	if err == nil {
		if dir.IsDir() {
			return nil
		}
		return &os.PathError{"mkdir", path, syscall.ENOTDIR}
	}

	i := len(path)
	for i > 0 && os.IsPathSeparator(path[i-1]) {
		i--
	}

	j := i
	for j > 0 && !os.IsPathSeparator(path[j-1]) {
		j--
	}

	if j > 1 {
		err = MkdirAll(fs, path[0:j-1])
		if err != nil {
			return err
		}
	}

	err = fs.Mkdir(path)
	if err != nil {
		dir, err1 := fs.Lstat(path)
		if err1 == nil && dir.IsDir() {
			return nil
		}
		return err
	}
	return nil
}

// Glob returns the names of all files under prefix matching pattern or nil if
// there is no matching file. The syntax of patterns is the same as in
// path/filepath.Match.
func Glob(wfs WalkableFileSystem, prefix, pattern string) (matches []string, err error) {
	walker := fs.WalkFS(filepath.Clean(prefix), wfs)
	for walker.Step() {
		path := walker.Path()
		matched, err := filepath.Match(pattern, path)
		if err != nil {
			return nil, err
		}
		if matched {
			matches = append(matches, path)
		}
	}
	return
}

type WalkableFileSystem interface {
	FileSystem
	Join(elem ...string) string
}

type walkableFileSystem struct{ FileSystem }

func (_ walkableFileSystem) Join(elem ...string) string { return filepath.Join(elem...) }<commit_msg>add method for accessing a walkable fs's underlying fs<commit_after>// Package rwvfs augments vfs to support write operations.
package rwvfs

import (
	"io"
	"os"
	"path/filepath"
	"syscall"

	"github.com/kr/fs"

	"golang.org/x/tools/godoc/vfs"
)

type FileSystem interface {
	vfs.FileSystem

	// Create creates the named file, truncating it if it already exists.
	Create(path string) (io.WriteCloser, error)

	// Mkdir creates a new directory. If name is already a directory, Mkdir
	// returns an error (that can be detected using os.IsExist).
	Mkdir(name string) error

	// Remove removes the named file or directory.
	Remove(name string) error
}

// MkdirAll creates a directory named path, along with any necessary parents. If
// path is already a directory, MkdirAll does nothing and returns nil.
func MkdirAll(fs FileSystem, path string) error {
	// adapted from os/MkdirAll

	dir, err := fs.Stat(path)
	if err == nil {
		if dir.IsDir() {
			return nil
		}
		return &os.PathError{"mkdir", path, syscall.ENOTDIR}
	}

	i := len(path)
	for i > 0 && os.IsPathSeparator(path[i-1]) {
		i--
	}

	j := i
	for j > 0 && !os.IsPathSeparator(path[j-1]) {
		j--
	}

	if j > 1 {
		err = MkdirAll(fs, path[0:j-1])
		if err != nil {
			return err
		}
	}

	err = fs.Mkdir(path)
	if err != nil {
		dir, err1 := fs.Lstat(path)
		if err1 == nil && dir.IsDir() {
			return nil
		}
		return err
	}
	return nil
}

// Glob returns the names of all files under prefix matching pattern or nil if
// there is no matching file. The syntax of patterns is the same as in
// path/filepath.Match.
func Glob(wfs WalkableFileSystem, prefix, pattern string) (matches []string, err error) {
	walker := fs.WalkFS(filepath.Clean(prefix), wfs)
	for walker.Step() {
		path := walker.Path()
		matched, err := filepath.Match(pattern, path)
		if err != nil {
			return nil, err
		}
		if matched {
			matches = append(matches, path)
		}
	}
	return
}

type WalkableFileSystem interface {
	FileSystem
	Join(elem ...string) string
}

type walkableFileSystem struct{ FileSystem }

func (_ walkableFileSystem) Join(elem ...string) string { return filepath.Join(elem...) }

func (fs walkableFileSystem) Underlying() FileSystem { return fs.FileSystem }
{'fraction_non_alphanumeric': 0.1192361434559851, 'fraction_numerical': 0.004191895668374476, 'mean_word_length': 3.3209255533199196, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2151187', 'n_tokens_mistral': 1593, 'n_tokens_neox': 1494, 'n_words': 495}