<commit_before>package lazycache

import (
	"fmt"
	"github.com/amarburg/go-lazyfs"
	"os"
	"path"
)

//====

type LocalFS struct {
	Path string
}

func OpenLocalFS(path string) (*LocalFS, error) {
	fs := &LocalFS{Path: path} //Uri: uri}
	return fs, nil
}

func (fs *LocalFS) PathType(path string) int {

	fmt.Println("Finding pathType of ", path)

	// Pure heuristics right now
	if trailingSlash.MatchString(path) {
		return Directory
	}

	return File
}

func (fs *LocalFS) OriginalPath(p string) string {
	return path.Join(fs.Path, p)
}

func (fs *LocalFS) LazyFile(p string) (lazyfs.FileSource, error) {
	return lazyfs.OpenLocalFileSource(fs.Path, p)
}

func (fs *LocalFS) ReadDir(p string) (*DirListing, error) {

	listing := &DirListing{Path: p,
		Files:       make([]string, 0),
		Directories: make([]string, 0),
	}

	fullPath := path.Join(fs.Path, p)

	fmt.Printf("Querying directory: %s\n", fullPath)

	dir, err := os.Open(fullPath)
	if err != nil {
		return listing, err
	}

	files, err := dir.Readdir(0)

	if err != nil {
		return listing, err
	}

	for _, f := range files {
		if f.IsDir() {
			listing.Directories = append(listing.Directories, f.Name())
		} else if f.Mode()&os.ModeType == 0 {
			// os.ModeType includes all of the "special types" e.g. pipes, directories, etc.
			listing.Files = append(listing.Files, f.Name())
		}
	}

	return listing, err
}<commit_msg>Allow Lazycache to follow symlinks.<commit_after>package lazycache

import (
	"fmt"
	"github.com/amarburg/go-lazyfs"
	"os"
	"path"
)

//====

type LocalFS struct {
	Path string
}

func OpenLocalFS(path string) (*LocalFS, error) {
	fs := &LocalFS{Path: path} //Uri: uri}
	return fs, nil
}

func (fs *LocalFS) PathType(path string) int {

	fmt.Println("Finding pathType of ", path)

	// Pure heuristics right now
	if trailingSlash.MatchString(path) {
		return Directory
	}

	return File
}

func (fs *LocalFS) OriginalPath(p string) string {
	return path.Join(fs.Path, p)
}

func (fs *LocalFS) LazyFile(p string) (lazyfs.FileSource, error) {
	return lazyfs.OpenLocalFileSource(fs.Path, p)
}

func (fs *LocalFS) ReadDir(p string) (*DirListing, error) {

	listing := &DirListing{Path: p,
		Files:       make([]string, 0),
		Directories: make([]string, 0),
	}

	fullPath := path.Join(fs.Path, p)

	fmt.Printf("Querying directory: %s\n", fullPath)

	dir, err := os.Open(fullPath)
	if err != nil {
		return listing, err
	}

	files, err := dir.Readdir(0)

	if err != nil {
		return listing, err
	}

	for _, f := range files {
		if f.IsDir() {
			listing.Directories = append(listing.Directories, f.Name())
		} else if f.Mode()&(os.ModeDir | os.ModeNamedPipe | os.ModeSocket | os.ModeDevice) == 0 {
			// os.ModeType includes all of the "special types" e.g. pipes, directories, etc., but we allow symlinks...
			listing.Files = append(listing.Files, f.Name())
		}
	}

	return listing, err
}
{'fraction_non_alphanumeric': 0.1513986013986014, 'fraction_numerical': 0.002797202797202797, 'mean_word_length': 3.721122112211221, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3332607', 'n_tokens_mistral': 1133, 'n_tokens_neox': 1057, 'n_words': 281}