<commit_before>package unionfs

import (
	"fmt"
	"github.com/hanwen/go-fuse/fuse"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"syscall"
	"time"
)

type knownFs struct {
	*UnionFs
	*fuse.PathNodeFs
}

// Creates unions for all files under a given directory,
// walking the tree and looking for directories D which have a
// D/READONLY symlink.
//
// A union for A/B/C will placed under directory A-B-C.
type AutoUnionFs struct {
	fuse.DefaultFileSystem

	lock             sync.RWMutex
	knownFileSystems map[string]knownFs
	nameRootMap      map[string]string
	root             string

	nodeFs  *fuse.PathNodeFs
	options *AutoUnionFsOptions

	mountState *fuse.MountState
}

type AutoUnionFsOptions struct {
	UnionFsOptions
	fuse.FileSystemOptions
	fuse.PathNodeFsOptions

	// If set, run updateKnownFses() after mounting.
	UpdateOnMount bool

	// If set hides the _READONLY file.
	HideReadonly bool
}

const (
	_READONLY    = "READONLY"
	_STATUS      = "status"
	_CONFIG      = "config"
	_DEBUG       = "debug"
	_ROOT        = "root"
	_VERSION     = "gounionfs_version"
	_SCAN_CONFIG = ".scan_config"
)

func NewAutoUnionFs(directory string, options AutoUnionFsOptions) *AutoUnionFs {
	if options.HideReadonly {
		options.HiddenFiles = append(options.HiddenFiles, _READONLY)
	}
	a := new(AutoUnionFs)
	a.knownFileSystems = make(map[string]knownFs)
	a.nameRootMap = make(map[string]string)
	a.options = &options
	directory, err := filepath.Abs(directory)
	if err != nil {
		panic("filepath.Abs returned err")
	}
	a.root = directory
	return a
}

func (me *AutoUnionFs) String() string {
	return fmt.Sprintf("AutoUnionFs(%s)", me.root)
}

func (me *AutoUnionFs) OnMount(nodeFs *fuse.PathNodeFs) {
	me.nodeFs = nodeFs
	if me.options.UpdateOnMount {
		time.AfterFunc(0.1e9, func() { me.updateKnownFses() })
	}
}

func (me *AutoUnionFs) addAutomaticFs(roots []string) {
	relative := strings.TrimLeft(strings.Replace(roots[0], me.root, "", -1), "/")
	name := strings.Replace(relative, "/", "-", -1)

	if me.getUnionFs(name) == nil {
		me.addFs(name, roots)
	}
}

func (me *AutoUnionFs) createFs(name string, roots []string) fuse.Status {
	me.lock.Lock()
	defer me.lock.Unlock()

	for workspace, root := range me.nameRootMap {
		if root == roots[0] && workspace != name {
			log.Printf("Already have a union FS for directory %s in workspace %s",
				roots[0], workspace)
			return fuse.EBUSY
		}
	}

	known := me.knownFileSystems[name]
	if known.UnionFs != nil {
		log.Println("Already have a workspace:", name)
		return fuse.EBUSY
	}

	ufs, err := NewUnionFsFromRoots(roots, &me.options.UnionFsOptions, true)
	if err != nil {
		log.Println("Could not create UnionFs:", err)
		return fuse.EPERM
	}

	log.Printf("Adding workspace %v for roots %v", name, ufs.String())
	nfs := fuse.NewPathNodeFs(ufs, &me.options.PathNodeFsOptions)
	code := me.nodeFs.Mount(name, nfs, &me.options.FileSystemOptions)
	if code.Ok() {
		me.knownFileSystems[name] = knownFs{
			ufs,
			nfs,
		}
		me.nameRootMap[name] = roots[0]
	}
	return code
}

func (me *AutoUnionFs) rmFs(name string) (code fuse.Status) {
	me.lock.Lock()
	defer me.lock.Unlock()

	known := me.knownFileSystems[name]
	if known.UnionFs == nil {
		return fuse.ENOENT
	}

	code = me.nodeFs.Unmount(name)
	if code.Ok() {
		delete(me.knownFileSystems, name)
		delete(me.nameRootMap, name)
	} else {
		log.Printf("Unmount failed for %s.  Code %v", name, code)
	}

	return code
}

func (me *AutoUnionFs) addFs(name string, roots []string) (code fuse.Status) {
	if name == _CONFIG || name == _STATUS || name == _SCAN_CONFIG {
		log.Println("Illegal name for overlay", roots)
		return fuse.EINVAL
	}
	return me.createFs(name, roots)
}

func (me *AutoUnionFs) getRoots(path string) []string {
	ro := filepath.Join(path, _READONLY)
	fi, err := os.Lstat(ro)
	fiDir, errDir := os.Stat(ro)
	if err != nil || errDir != nil {
		return nil
	}

	if fi.Mode()&os.ModeSymlink != 0 && fiDir.IsDir() {
		// TODO - should recurse and chain all READONLYs
		// together.
		return []string{path, ro}
	}
	return nil
}

func (me *AutoUnionFs) visit(path string, fi os.FileInfo, err error) error {
	if fi.IsDir() {
		roots := me.getRoots(path)
		if roots != nil {
			me.addAutomaticFs(roots)
		}
	}
	return nil
}

func (me *AutoUnionFs) updateKnownFses() {
	log.Println("Looking for new filesystems")
	// We unroll the first level of entries in the root manually in order
	// to allow symbolic links on that level.
	directoryEntries, err := ioutil.ReadDir(me.root)
	if err == nil {
		for _, dir := range directoryEntries {
			if dir.IsDir() || dir.Mode()&os.ModeSymlink != 0 {
				path := filepath.Join(me.root, dir.Name())
				dir, _ = os.Stat(path)
				me.visit(path, dir, nil)
				filepath.Walk(path,
					func(path string, fi os.FileInfo, err error) error {
						return me.visit(path, fi, err)
					})
			}
		}
	}
	log.Println("Done looking")
}

func (me *AutoUnionFs) Readlink(path string, context *fuse.Context) (out string, code fuse.Status) {<commit_msg>Fix a nil-pointer dereference in AutoUnionFs.<commit_after>package unionfs

import (
	"fmt"
	"github.com/hanwen/go-fuse/fuse"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"syscall"
	"time"
)

type knownFs struct {
	*UnionFs
	*fuse.PathNodeFs
}

// Creates unions for all files under a given directory,
// walking the tree and looking for directories D which have a
// D/READONLY symlink.
//
// A union for A/B/C will placed under directory A-B-C.
type AutoUnionFs struct {
	fuse.DefaultFileSystem

	lock             sync.RWMutex
	knownFileSystems map[string]knownFs
	nameRootMap      map[string]string
	root             string

	nodeFs  *fuse.PathNodeFs
	options *AutoUnionFsOptions

	mountState *fuse.MountState
}

type AutoUnionFsOptions struct {
	UnionFsOptions
	fuse.FileSystemOptions
	fuse.PathNodeFsOptions

	// If set, run updateKnownFses() after mounting.
	UpdateOnMount bool

	// If set hides the _READONLY file.
	HideReadonly bool
}

const (
	_READONLY    = "READONLY"
	_STATUS      = "status"
	_CONFIG      = "config"
	_DEBUG       = "debug"
	_ROOT        = "root"
	_VERSION     = "gounionfs_version"
	_SCAN_CONFIG = ".scan_config"
)

func NewAutoUnionFs(directory string, options AutoUnionFsOptions) *AutoUnionFs {
	if options.HideReadonly {
		options.HiddenFiles = append(options.HiddenFiles, _READONLY)
	}
	a := new(AutoUnionFs)
	a.knownFileSystems = make(map[string]knownFs)
	a.nameRootMap = make(map[string]string)
	a.options = &options
	directory, err := filepath.Abs(directory)
	if err != nil {
		panic("filepath.Abs returned err")
	}
	a.root = directory
	return a
}

func (me *AutoUnionFs) String() string {
	return fmt.Sprintf("AutoUnionFs(%s)", me.root)
}

func (me *AutoUnionFs) OnMount(nodeFs *fuse.PathNodeFs) {
	me.nodeFs = nodeFs
	if me.options.UpdateOnMount {
		time.AfterFunc(0.1e9, func() { me.updateKnownFses() })
	}
}

func (me *AutoUnionFs) addAutomaticFs(roots []string) {
	relative := strings.TrimLeft(strings.Replace(roots[0], me.root, "", -1), "/")
	name := strings.Replace(relative, "/", "-", -1)

	if me.getUnionFs(name) == nil {
		me.addFs(name, roots)
	}
}

func (me *AutoUnionFs) createFs(name string, roots []string) fuse.Status {
	me.lock.Lock()
	defer me.lock.Unlock()

	for workspace, root := range me.nameRootMap {
		if root == roots[0] && workspace != name {
			log.Printf("Already have a union FS for directory %s in workspace %s",
				roots[0], workspace)
			return fuse.EBUSY
		}
	}

	known := me.knownFileSystems[name]
	if known.UnionFs != nil {
		log.Println("Already have a workspace:", name)
		return fuse.EBUSY
	}

	ufs, err := NewUnionFsFromRoots(roots, &me.options.UnionFsOptions, true)
	if err != nil {
		log.Println("Could not create UnionFs:", err)
		return fuse.EPERM
	}

	log.Printf("Adding workspace %v for roots %v", name, ufs.String())
	nfs := fuse.NewPathNodeFs(ufs, &me.options.PathNodeFsOptions)
	code := me.nodeFs.Mount(name, nfs, &me.options.FileSystemOptions)
	if code.Ok() {
		me.knownFileSystems[name] = knownFs{
			ufs,
			nfs,
		}
		me.nameRootMap[name] = roots[0]
	}
	return code
}

func (me *AutoUnionFs) rmFs(name string) (code fuse.Status) {
	me.lock.Lock()
	defer me.lock.Unlock()

	known := me.knownFileSystems[name]
	if known.UnionFs == nil {
		return fuse.ENOENT
	}

	code = me.nodeFs.Unmount(name)
	if code.Ok() {
		delete(me.knownFileSystems, name)
		delete(me.nameRootMap, name)
	} else {
		log.Printf("Unmount failed for %s.  Code %v", name, code)
	}

	return code
}

func (me *AutoUnionFs) addFs(name string, roots []string) (code fuse.Status) {
	if name == _CONFIG || name == _STATUS || name == _SCAN_CONFIG {
		log.Println("Illegal name for overlay", roots)
		return fuse.EINVAL
	}
	return me.createFs(name, roots)
}

func (me *AutoUnionFs) getRoots(path string) []string {
	ro := filepath.Join(path, _READONLY)
	fi, err := os.Lstat(ro)
	fiDir, errDir := os.Stat(ro)
	if err != nil || errDir != nil {
		return nil
	}

	if fi.Mode()&os.ModeSymlink != 0 && fiDir.IsDir() {
		// TODO - should recurse and chain all READONLYs
		// together.
		return []string{path, ro}
	}
	return nil
}

func (me *AutoUnionFs) visit(path string, fi os.FileInfo, err error) error {
	if fi != nil && fi.IsDir() {
		roots := me.getRoots(path)
		if roots != nil {
			me.addAutomaticFs(roots)
		}
	}
	return nil
}

func (me *AutoUnionFs) updateKnownFses() {
	log.Println("Looking for new filesystems")
	// We unroll the first level of entries in the root manually in order
	// to allow symbolic links on that level.
	directoryEntries, err := ioutil.ReadDir(me.root)
	if err == nil {
		for _, dir := range directoryEntries {
			if dir.IsDir() || dir.Mode()&os.ModeSymlink != 0 {
				path := filepath.Join(me.root, dir.Name())
				dir, _ = os.Stat(path)
				me.visit(path, dir, nil)
				filepath.Walk(path,
					func(path string, fi os.FileInfo, err error) error {
						return me.visit(path, fi, err)
					})
			}
		}
	}
	log.Println("Done looking")
}

func (me *AutoUnionFs) Readlink(path string, context *fuse.Context) (out string, code fuse.Status) {
{'fraction_non_alphanumeric': 0.1356879177890851, 'fraction_numerical': 0.0021949516112940238, 'mean_word_length': 3.6928838951310863, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5070758', 'n_tokens_mistral': 3984, 'n_tokens_neox': 3656, 'n_words': 921}