<commit_before>package fs

import (
	"os"
	"path/filepath"
	"strings"

	"go.polydawn.net/repeatr/lib/treewalk"
)

type WalkFunc func(filenode *FilewalkNode) error

/*
	Walks a filesystem.

	This is much like the standard library's `path/filepath.Walk`, except
	it's based on `treewalk`, which means it supports both pre- and post-order
	traversals.

	All paths begin in `./`, and directory names are slash-suffixed.
	E.g. you'll see a series like `{"./", "./a/", "./a/b"}`, etc.
	This matches the behaviors described by `Normalize` in the `lib/fshash`.

	Symlinks are not followed.

	The traversal order of siblings is *not* guaranteed, and is *not* necessarily
	stable.

	Caveat: calling `node.NextChild()` during your walk results in undefined behavior.
*/
func Walk(basePath string, preVisit WalkFunc, postVisit WalkFunc) error {
	return treewalk.Walk(
		newFileWalkNode(basePath, "./"),
		func(node treewalk.Node) error {
			filenode := node.(*FilewalkNode)
			if preVisit != nil {
				if err := preVisit(filenode); err != nil {
					return err
				}
			}
			return filenode.prepareChildren(basePath)
		},
		func(node treewalk.Node) error {
			filenode := node.(*FilewalkNode)
			var err error
			if postVisit != nil {
				err = postVisit(filenode)
			}
			filenode.forgetChildren()
			return err
		},
	)
}

var _ treewalk.Node = &FilewalkNode{}

type FilewalkNode struct {
	Path string // relative to start
	Info os.FileInfo
	Err  error

	children []*FilewalkNode // note we didn't sort this
	itrIndex int             // next child offset
}

func (t *FilewalkNode) NextChild() treewalk.Node {
	if t.itrIndex >= len(t.children) {
		return nil
	}
	t.itrIndex++
	return t.children[t.itrIndex-1]
}

func newFileWalkNode(basePath, path string) (filenode *FilewalkNode) {
	filenode = &FilewalkNode{Path: path}
	filenode.Info, filenode.Err = os.Lstat(filepath.Join(basePath, path))
	if filenode.Err == nil && filenode.Info.IsDir() {
		if !strings.HasSuffix(filenode.Path, "/") {
			filenode.Path += "/"
		}
	}
	// don't expand the children until the previsit function
	// we don't want them all crashing into memory at once
	return
}

/*
	Expand next subtree.  Used in the pre-order visit step so we don't walk
	every dir up front.  `Walk()` wraps the user-defined pre-visit function
	to do this at the end.
*/
func (t *FilewalkNode) prepareChildren(basePath string) error {
	if !t.Info.IsDir() {
		return nil
	}
	f, err := os.Open(filepath.Join(basePath, t.Path))
	if err != nil {
		return err
	}
	names, err := f.Readdirnames(-1)
	f.Close()<commit_msg>Attempt to handle outputting a tar containing a single file.<commit_after>package fs

import (
	"os"
	"path/filepath"
	"strings"

	"go.polydawn.net/repeatr/lib/treewalk"
)

type WalkFunc func(filenode *FilewalkNode) error

/*
	Walks a filesystem.

	This is much like the standard library's `path/filepath.Walk`, except
	it's based on `treewalk`, which means it supports both pre- and post-order
	traversals.

	All paths begin in `./`, and directory names are slash-suffixed.
	E.g. you'll see a series like `{"./", "./a/", "./a/b"}`, etc.
	This matches the behaviors described by `Normalize` in the `lib/fshash`.

	If walking directories, implicitly the first path will always be `./`;
	if the basePath is a file however, the first (and only) path with be
	`./thefile` (where 'thefile' is effectively `filepath.Base` of the basePath).
	Note the subtle difference in the amount of information shown to visit funcs:
	when basePath is a directory, no information about the path names leading up
	to *and including* basePath's name are sent to the visit funcs;
	when basePath is a file, part of basePath *is* sent to the visit funcs.
	Typically this does what you mean (after all, what can we do with a file
	with no name?), but the distinction may be important to note if the caller's
	intention is to have no effects from basePath's location.

	Symlinks are not followed.

	The traversal order of siblings is *not* guaranteed, and is *not* necessarily
	stable.

	Caveat: calling `node.NextChild()` during your walk results in undefined behavior.
*/
func Walk(basePath string, preVisit WalkFunc, postVisit WalkFunc) error {
	return treewalk.Walk(
		newFileWalkNode(basePath, "./"),
		func(node treewalk.Node) error {
			filenode := node.(*FilewalkNode)
			if preVisit != nil {
				if err := preVisit(filenode); err != nil {
					return err
				}
			}
			return filenode.prepareChildren(basePath)
		},
		func(node treewalk.Node) error {
			filenode := node.(*FilewalkNode)
			var err error
			if postVisit != nil {
				err = postVisit(filenode)
			}
			filenode.forgetChildren()
			return err
		},
	)
}

var _ treewalk.Node = &FilewalkNode{}

type FilewalkNode struct {
	Path string // relative to start
	Info os.FileInfo
	Err  error

	children []*FilewalkNode // note we didn't sort this
	itrIndex int             // next child offset
}

func (t *FilewalkNode) NextChild() treewalk.Node {
	if t.itrIndex >= len(t.children) {
		return nil
	}
	t.itrIndex++
	return t.children[t.itrIndex-1]
}

func newFileWalkNode(basePath, path string) (filenode *FilewalkNode) {
	// Mostly: fill in attributes from os.Lstat.
	filenode = &FilewalkNode{Path: path}
	filenode.Info, filenode.Err = os.Lstat(filepath.Join(basePath, path))
	// Normalize the reported path of dirs to include trailing slash.
	if filenode.Err == nil && filenode.Info.IsDir() {
		if !strings.HasSuffix(filenode.Path, "/") {
			filenode.Path += "/"
		}
	}
	// Handle boundary condition for a basepath that is a file.
	if path == "./" && !filenode.Info.IsDir() {
		filenode.Path = "./" + filenode.Info.Name()
	}
	// don't expand the children until the previsit function
	// we don't want them all crashing into memory at once
	return
}

/*
	Expand next subtree.  Used in the pre-order visit step so we don't walk
	every dir up front.  `Walk()` wraps the user-defined pre-visit function
	to do this at the end.
*/
func (t *FilewalkNode) prepareChildren(basePath string) error {
	if !t.Info.IsDir() {
		return nil
	}
	f, err := os.Open(filepath.Join(basePath, t.Path))
	if err != nil {
		return err
	}
	names, err := f.Readdirnames(-1)
	f.Close()
{'fraction_non_alphanumeric': 0.11403936879778753, 'fraction_numerical': 0.0006507239303725395, 'mean_word_length': 3.557449962935508, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2431278', 'n_tokens_mistral': 2196, 'n_tokens_neox': 2048, 'n_words': 695}