<commit_before>package vfs

import (
	"fmt"
	"github.com/Sirupsen/logrus"
	"github.com/rancher/convoy/objectstore"
	"github.com/rancher/convoy/util"
	"io"
	"net/url"
	"os"
	"path/filepath"
	"strings"
)

var (
	log = logrus.WithFields(logrus.Fields{"pkg": "vfs"})
)

type VfsObjectStoreDriver struct {
	destURL string
	path    string
}

const (
	KIND = "vfs"

	VFS_PATH = "vfs.path"

	MAX_CLEANUP_LEVEL = 10
)

func init() {
	objectstore.RegisterDriver(KIND, initFunc)
}

func initFunc(destURL string) (objectstore.ObjectStoreDriver, error) {
	b := &VfsObjectStoreDriver{}
	u, err := url.Parse(destURL)
	if err != nil {
		return nil, err
	}

	if u.Scheme != KIND {
		return nil, fmt.Errorf("BUG: Why dispatch %v to %v?", u.Scheme, KIND)
	}

	if u.Host != "" {
		return nil, fmt.Errorf("VFS path must follow: vfs:///path/ format")
	}

	b.path = u.Path

	if b.path == "" {
		return nil, fmt.Errorf("Cannot find vfs path")
	}
	if _, err := b.List(""); err != nil {
		return nil, fmt.Errorf("VFS path %v doesn't exist or is not a directory", b.path)
	}

	b.destURL = KIND + "://" + b.path
	log.Debug("Loaded driver for %v", b.destURL)
	return b, nil
}

func (v *VfsObjectStoreDriver) updatePath(path string) string {
	return filepath.Join(v.path, path)
}

func (v *VfsObjectStoreDriver) preparePath(file string) error {
	if err := os.MkdirAll(filepath.Dir(v.updatePath(file)), os.ModeDir|0700); err != nil {
		return err
	}
	return nil
}

func (v *VfsObjectStoreDriver) Kind() string {
	return KIND
}

func (v *VfsObjectStoreDriver) GetURL() string {
	return v.destURL
}

func (v *VfsObjectStoreDriver) FileSize(filePath string) int64 {
	file := v.updatePath(filePath)
	st, err := os.Stat(file)
	if err != nil || st.IsDir() {
		return -1
	}
	return st.Size()
}

func (v *VfsObjectStoreDriver) FileExists(filePath string) bool {
	return v.FileSize(filePath) >= 0
}

func (v *VfsObjectStoreDriver) Remove(names ...string) error {
	for _, name := range names {
		if err := os.RemoveAll(v.updatePath(name)); err != nil {
			return err
		}
		//Also automatically cleanup upper level directories
		dir := v.updatePath(name)
		for i := 0; i < MAX_CLEANUP_LEVEL; i++ {
			dir = filepath.Dir(dir)
			// If directory is not empty, then we don't need to continue
			if err := os.Remove(dir); err != nil {<commit_msg>vfs: objectstore: Don't auto clean above OBEJCTSTORE_BASE<commit_after>package vfs

import (
	"fmt"
	"github.com/Sirupsen/logrus"
	"github.com/rancher/convoy/objectstore"
	"github.com/rancher/convoy/util"
	"io"
	"net/url"
	"os"
	"path/filepath"
	"strings"
)

var (
	log = logrus.WithFields(logrus.Fields{"pkg": "vfs"})
)

type VfsObjectStoreDriver struct {
	destURL string
	path    string
}

const (
	KIND = "vfs"

	VFS_PATH = "vfs.path"

	MAX_CLEANUP_LEVEL = 10
)

func init() {
	objectstore.RegisterDriver(KIND, initFunc)
}

func initFunc(destURL string) (objectstore.ObjectStoreDriver, error) {
	b := &VfsObjectStoreDriver{}
	u, err := url.Parse(destURL)
	if err != nil {
		return nil, err
	}

	if u.Scheme != KIND {
		return nil, fmt.Errorf("BUG: Why dispatch %v to %v?", u.Scheme, KIND)
	}

	if u.Host != "" {
		return nil, fmt.Errorf("VFS path must follow: vfs:///path/ format")
	}

	b.path = u.Path

	if b.path == "" {
		return nil, fmt.Errorf("Cannot find vfs path")
	}
	if _, err := b.List(""); err != nil {
		return nil, fmt.Errorf("VFS path %v doesn't exist or is not a directory", b.path)
	}

	b.destURL = KIND + "://" + b.path
	log.Debug("Loaded driver for %v", b.destURL)
	return b, nil
}

func (v *VfsObjectStoreDriver) updatePath(path string) string {
	return filepath.Join(v.path, path)
}

func (v *VfsObjectStoreDriver) preparePath(file string) error {
	if err := os.MkdirAll(filepath.Dir(v.updatePath(file)), os.ModeDir|0700); err != nil {
		return err
	}
	return nil
}

func (v *VfsObjectStoreDriver) Kind() string {
	return KIND
}

func (v *VfsObjectStoreDriver) GetURL() string {
	return v.destURL
}

func (v *VfsObjectStoreDriver) FileSize(filePath string) int64 {
	file := v.updatePath(filePath)
	st, err := os.Stat(file)
	if err != nil || st.IsDir() {
		return -1
	}
	return st.Size()
}

func (v *VfsObjectStoreDriver) FileExists(filePath string) bool {
	return v.FileSize(filePath) >= 0
}

func (v *VfsObjectStoreDriver) Remove(names ...string) error {
	for _, name := range names {
		if err := os.RemoveAll(v.updatePath(name)); err != nil {
			return err
		}
		//Also automatically cleanup upper level directories
		dir := v.updatePath(name)
		for i := 0; i < MAX_CLEANUP_LEVEL; i++ {
			dir = filepath.Dir(dir)
			// Don't clean above OBJECTSTORE_BASE
			if strings.HasSuffix(dir, objectstore.OBJECTSTORE_BASE) {
				break
			}
			// If directory is not empty, then we don't need to continue
			if err := os.Remove(dir); err != nil {
{'fraction_non_alphanumeric': 0.14681789540012602, 'fraction_numerical': 0.004620877966813695, 'mean_word_length': 3.795568982880161, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7283656', 'n_tokens_mistral': 1884, 'n_tokens_neox': 1794, 'n_words': 470}