<commit_before>// Package fs provides filesystem-related functions.
package fs

import (
	"io/ioutil"
	"os"
	"path/filepath"
)

// Walker provides a convenient interface for iterating over the
// descendents of a filesystem path.
// Successive calls to the Step method will step through each
// file or directory in the tree, including the root. The files
// are walked in lexical order, which makes the output deterministic
// but means that for very large directories Walker can be inefficient.
// Walker does not follow symbolic links.
type Walker struct {
	cur     item
	stack   []item
	descend bool
}

type item struct {
	path string
	info os.FileInfo
	err  error
}

// Walk returns a new Walker rooted at root.
func Walk(root string) *Walker {
	info, err := os.Lstat(root)
	return &Walker{stack: []item{{root, info, err}}}
}

// Step advances the Walker to the next file or directory,<commit_msg>fix spelling mistake<commit_after>// Package fs provides filesystem-related functions.
package fs

import (
	"io/ioutil"
	"os"
	"path/filepath"
)

// Walker provides a convenient interface for iterating over the
// descendants of a filesystem path.
// Successive calls to the Step method will step through each
// file or directory in the tree, including the root. The files
// are walked in lexical order, which makes the output deterministic
// but means that for very large directories Walker can be inefficient.
// Walker does not follow symbolic links.
type Walker struct {
	cur     item
	stack   []item
	descend bool
}

type item struct {
	path string
	info os.FileInfo
	err  error
}

// Walk returns a new Walker rooted at root.
func Walk(root string) *Walker {
	info, err := os.Lstat(root)
	return &Walker{stack: []item{{root, info, err}}}
}

// Step advances the Walker to the next file or directory,
{'fraction_non_alphanumeric': 0.08061844284925455, 'fraction_numerical': 0.0, 'mean_word_length': 3.658097686375321, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5148240', 'n_tokens_mistral': 524, 'n_tokens_neox': 501, 'n_words': 245}