<commit_before>package filetree // import "a4.io/blobstash/pkg/filetree"

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"github.com/gorilla/mux"
	log "github.com/inconshreveable/log15"
	"golang.org/x/net/context"

	"a4.io/blobstash/pkg/blob"
	"a4.io/blobstash/pkg/blobstore"
	"a4.io/blobstash/pkg/cache"
	"a4.io/blobstash/pkg/client/clientutil"
	"a4.io/blobstash/pkg/config"
	rnode "a4.io/blobstash/pkg/filetree/filetreeutil/node"
	"a4.io/blobstash/pkg/filetree/imginfo"
	"a4.io/blobstash/pkg/filetree/reader/filereader"
	"a4.io/blobstash/pkg/filetree/writer"
	"a4.io/blobstash/pkg/httputil"
	"a4.io/blobstash/pkg/httputil/bewit"
	"a4.io/blobstash/pkg/httputil/resize"
	"a4.io/blobstash/pkg/hub"
	"a4.io/blobstash/pkg/kvstore"
	"a4.io/blobstash/pkg/vkv"
)

var (
	// FIXME(tsileo): add a way to set a custom fmt key life for Blobs CLI as we don't care about the FS?
	indexFile = "index.html"
	FSKeyFmt  = "blobfs:root:%s"
	// FSKeyFmt = "_:filetree:fs:%s"
	// PermName     = "filetree"
	// PermTreeName = "filetree:root:"
	// PermWrite    = "write"
	// PermRead     = "read"

	MaxUploadSize int64 = 512 << 20
)

// TODO(tsileo): rename to FileTree
type FileTreeExt struct {
	kvStore       *kvstore.KvStore
	blobStore     *blobstore.BlobStore
	conf          *config.Config
	hub           *hub.Hub
	sharingCred   *bewit.Cred
	authFunc      func(*http.Request) bool
	shareTTL      time.Duration
	thumbCache    *cache.Cache
	metadataCache *cache.Cache

	log log.Logger
}

func (ft *FileTreeExt) SharingCred() *bewit.Cred {
	return ft.sharingCred
}

func (ft *FileTreeExt) ShareTTL() time.Duration {
	return ft.shareTTL
}

// BlobStore is the interface to be compatible with both the server and the BlobStore client
type BlobStore struct {
	blobStore *blobstore.BlobStore
}

func (bs *BlobStore) Get(hash string) ([]byte, error) {
	return bs.blobStore.Get(context.TODO(), hash)
}

func (bs *BlobStore) Stat(hash string) (bool, error) {
	return bs.blobStore.Stat(context.TODO(), hash)
}

func (bs *BlobStore) Put(hash string, data []byte) error {
	return bs.blobStore.Put(context.TODO(), &blob.Blob{Hash: hash, Data: data})
}

type FS struct {
	Name string `json:"-"`
	Ref  string `json:"ref"`

	ft *FileTreeExt `json:"-"`
}

// New initializes the `DocStoreExt`
func New(logger log.Logger, conf *config.Config, authFunc func(*http.Request) bool, kvStore *kvstore.KvStore, blobStore *blobstore.BlobStore, chub *hub.Hub) (*FileTreeExt, error) {
	logger.Debug("init")
	// FIXME(tsileo): make the number of thumbnails to keep in memory a config item
	thumbscache, err := cache.New(conf, "filetree_thumbs.cache", 512<<20)
	if err != nil {
		return nil, err
	}
	metacache, err := cache.New(conf, "filetree_info.cache", 256<<20)
	if err != nil {
		return nil, err
	}
	return &FileTreeExt{
		conf:      conf,
		kvStore:   kvStore,
		blobStore: blobStore,
		sharingCred: &bewit.Cred{
			Key: []byte(conf.SharingKey),
			ID:  "filetree",
		},
		thumbCache:    thumbscache,
		metadataCache: metacache,
		authFunc:      authFunc,
		shareTTL:      1 * time.Hour,
		hub:           chub,
		log:           logger,
	}, nil
}

// Close closes all the open DB files.
func (ft *FileTreeExt) Close() error {
	ft.thumbCache.Close()
	ft.metadataCache.Close()
	return nil
}

// RegisterRoute registers all the HTTP handlers for the extension
func (ft *FileTreeExt) Register(r *mux.Router, root *mux.Router, basicAuth func(http.Handler) http.Handler) {
	// Raw node endpoint
	r.Handle("/node/{ref}", basicAuth(http.HandlerFunc(ft.nodeHandler())))

	// Public/semi-private handler
	dirHandler := http.HandlerFunc(ft.dirHandler())
	fileHandler := http.HandlerFunc(ft.fileHandler())

	r.Handle("/fs/root", basicAuth(http.HandlerFunc(ft.fsRootHandler())))
	r.Handle("/fs/{type}/{name}/", basicAuth(http.HandlerFunc(ft.fsHandler())))
	r.Handle("/fs/{type}/{name}/{path:.+}", basicAuth(http.HandlerFunc(ft.fsHandler())))
	// r.Handle("/fs", http.HandlerFunc(ft.fsHandler()))
	// r.Handle("/fs/{name}", http.HandlerFunc(ft.fsByNameHandler()))

	r.Handle("/upload", basicAuth(http.HandlerFunc(ft.uploadHandler())))

	// Hook the standard endpint
	r.Handle("/dir/{ref}", dirHandler)
	r.Handle("/file/{ref}", fileHandler)

	r.Handle("/index/{ref}", basicAuth(http.HandlerFunc(ft.indexHandler())))

	// Enable shortcut path from the root
	root.Handle("/d/{ref}", dirHandler)
	root.Handle("/f/{ref}", fileHandler)
}

// Node holds the data about the file node (either file/dir), analog to a Meta
type Node struct {
	Name     string  `json:"name"`
	Type     string  `json:"type"`
	Size     int     `json:"size"`
	Hash     string  `json:"ref"`
	Children []*Node `json:"children,omitempty"`

	// FIXME(ts): rename to Metadata
	Data map[string]interface{} `json:"metadata,omitempty"`
	Info *Info                  `json:"info,omitempty"`

	Meta   *rnode.RawNode `json:"-"`
	parent *Node          `json:"-"`
	fs     *FS            `json:"-"`

	URL string `json:"url",omitempty`
}

// Update the given node with the given meta
func (ft *FileTreeExt) Update(n *Node, m *rnode.RawNode) (*Node, error) {
	newNode, err := metaToNode(m)
	if err != nil {
		return nil, err
	}
	newNode.fs = n.fs
	newNode.parent = n.parent
	// fmt.Printf("\n\n\n###Update: n=%+v\nn.meta=%+v\nn.parent=%+v\nm=%+v\nnewNode=%+v\n\n\n###", n, n.meta, n.parent, m, newNode)
	// if n.parent != nil {
	// 	fmt.Printf("n.parent.meta=%+v\n", n.parent.meta)

	// }
	if n.parent == nil {
		n.fs.Ref = newNode.Hash
		js, err := json.Marshal(n.fs)
		if err != nil {
			return nil, err
		}
		if ft.kvStore.Put(context.TODO(), fmt.Sprintf(FSKeyFmt, n.fs.Name), "", js, -1); err != nil {
			return nil, err
		}
		return newNode, nil
	}

	newRefs := []interface{}{newNode.Hash}
	newChildren := []*Node{newNode}

	for _, c := range newNode.parent.Children {
		if c.Hash != n.Hash {
			newRefs = append(newRefs, c.Hash)
			newChildren = append(newChildren, c)
		}
	}

	newNode.parent.Meta.Refs = newRefs
	newNode.parent.Children = newChildren

	// parentMeta := n.parent.meta
	// parentMeta.Refs = newRefs
	// n.parent.Children = newChildren

	newRef, data := newNode.parent.Meta.Encode()
	newNode.parent.Hash = newRef
	newNode.parent.Meta.Hash = newRef
	if err := ft.blobStore.Put(context.TODO(), &blob.Blob{Hash: newRef, Data: data}); err != nil {
		return nil, err
	}
	// n.parent.Hash = newRef
	// parentMeta.Hash = newRef
	// Propagate the change to the parents
	if _, err := ft.Update(newNode.parent, newNode.parent.Meta); err != nil {
		return nil, err
	}
	return newNode, nil
}

func (n *Node) Close() error {
	return nil
}

type byName []*Node

func (s byName) Len() int           { return len(s) }
func (s byName) Less(i, j int) bool { return s[i].Name < s[j].Name }
func (s byName) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }

func metaToNode(m *rnode.RawNode) (*Node, error) {
	n := &Node{
		Name: m.Name,
		Type: m.Type,
		Size: m.Size,
		Data: m.Metadata,
		Hash: m.Hash,
		Meta: m,
	}
	return n, nil
}

// fetchDir recursively fetch dir children
func (ft *FileTreeExt) fetchDir(n *Node, depth, maxDepth int) error {
	if depth > maxDepth {
		return nil
	}
	if n.Type == "dir" {
		n.Children = []*Node{}
		for _, ref := range n.Meta.Refs {
			cn, err := ft.nodeByRef(ref.(string))
			if err != nil {
				return err
			}
			n.Children = append(n.Children, cn)
			if err := ft.fetchDir(cn, depth+1, maxDepth); err != nil {
				return err
			}
		}
	}
	// n.meta.Close()
	return nil
}

// FS fetch the FileSystem by name, returns an empty one if not found
func (ft *FileTreeExt) FS(name string, newState bool) (*FS, error) {
	fs := &FS{}
	if !newState {
		kv, err := ft.kvStore.Get(context.TODO(), fmt.Sprintf(FSKeyFmt, name), -1)
		if err != nil && err != vkv.ErrNotFound {
			return nil, err
		}
		switch err {
		case nil:
			if err := json.Unmarshal([]byte(kv.Data), fs); err != nil {
				return nil, err
			}
		case vkv.ErrNotFound:
			// XXX(tsileo): should the `ErrNotFound` be returned here?
		default:
			return nil, err
		}
	}
	fs.Name = name
	fs.ft = ft
	return fs, nil
}

// Root fetch the FS root, and creates a new one if `create` is set to true (but it won't be savec automatically in the BlobStore
func (fs *FS) Root(create bool) (*Node, error) {
	fs.ft.log.Info("Root", "fs", fs)
	node, err := fs.ft.nodeByRef(fs.Ref)
	switch err {
	case clientutil.ErrBlobNotFound:
		if !create {
			return nil, err
		}
		meta := &rnode.RawNode{
			Type: "dir",
			Name: "_root",
		}
		node, err = metaToNode(meta)
		if err != nil {<commit_msg>filetree: bugfixes, catch up with latest internal API<commit_after>package filetree // import "a4.io/blobstash/pkg/filetree"

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"github.com/gorilla/mux"
	log "github.com/inconshreveable/log15"
	"golang.org/x/net/context"

	"a4.io/blobsfile"
	"a4.io/blobstash/pkg/blob"
	"a4.io/blobstash/pkg/blobstore"
	"a4.io/blobstash/pkg/cache"
	"a4.io/blobstash/pkg/client/clientutil"
	"a4.io/blobstash/pkg/config"
	rnode "a4.io/blobstash/pkg/filetree/filetreeutil/node"
	"a4.io/blobstash/pkg/filetree/imginfo"
	"a4.io/blobstash/pkg/filetree/reader/filereader"
	"a4.io/blobstash/pkg/filetree/writer"
	"a4.io/blobstash/pkg/httputil"
	"a4.io/blobstash/pkg/httputil/bewit"
	"a4.io/blobstash/pkg/httputil/resize"
	"a4.io/blobstash/pkg/hub"
	"a4.io/blobstash/pkg/kvstore"
	"a4.io/blobstash/pkg/vkv"
)

var (
	// FIXME(tsileo): add a way to set a custom fmt key life for Blobs CLI as we don't care about the FS?
	indexFile = "index.html"
	FSKeyFmt  = "_filetree:fs:%s"
	// FSKeyFmt = "_:filetree:fs:%s"
	// PermName     = "filetree"
	// PermTreeName = "filetree:root:"
	// PermWrite    = "write"
	// PermRead     = "read"

	MaxUploadSize int64 = 512 << 20
)

// TODO(tsileo): rename to FileTree
type FileTreeExt struct {
	kvStore       *kvstore.KvStore
	blobStore     *blobstore.BlobStore
	conf          *config.Config
	hub           *hub.Hub
	sharingCred   *bewit.Cred
	authFunc      func(*http.Request) bool
	shareTTL      time.Duration
	thumbCache    *cache.Cache
	metadataCache *cache.Cache

	log log.Logger
}

func (ft *FileTreeExt) SharingCred() *bewit.Cred {
	return ft.sharingCred
}

func (ft *FileTreeExt) ShareTTL() time.Duration {
	return ft.shareTTL
}

// BlobStore is the interface to be compatible with both the server and the BlobStore client
type BlobStore struct {
	blobStore *blobstore.BlobStore
}

func (bs *BlobStore) Get(hash string) ([]byte, error) {
	return bs.blobStore.Get(context.TODO(), hash)
}

func (bs *BlobStore) Stat(hash string) (bool, error) {
	return bs.blobStore.Stat(context.TODO(), hash)
}

func (bs *BlobStore) Put(hash string, data []byte) error {
	return bs.blobStore.Put(context.TODO(), &blob.Blob{Hash: hash, Data: data})
}

type FS struct {
	Name string `json:"-"`
	Ref  string `json:"ref"`

	ft *FileTreeExt `json:"-"`
}

// New initializes the `DocStoreExt`
func New(logger log.Logger, conf *config.Config, authFunc func(*http.Request) bool, kvStore *kvstore.KvStore, blobStore *blobstore.BlobStore, chub *hub.Hub) (*FileTreeExt, error) {
	logger.Debug("init")
	// FIXME(tsileo): make the number of thumbnails to keep in memory a config item
	thumbscache, err := cache.New(conf, "filetree_thumbs.cache", 512<<20)
	if err != nil {
		return nil, err
	}
	metacache, err := cache.New(conf, "filetree_info.cache", 256<<20)
	if err != nil {
		return nil, err
	}
	return &FileTreeExt{
		conf:      conf,
		kvStore:   kvStore,
		blobStore: blobStore,
		sharingCred: &bewit.Cred{
			Key: []byte(conf.SharingKey),
			ID:  "filetree",
		},
		thumbCache:    thumbscache,
		metadataCache: metacache,
		authFunc:      authFunc,
		shareTTL:      1 * time.Hour,
		hub:           chub,
		log:           logger,
	}, nil
}

// Close closes all the open DB files.
func (ft *FileTreeExt) Close() error {
	ft.thumbCache.Close()
	ft.metadataCache.Close()
	return nil
}

// RegisterRoute registers all the HTTP handlers for the extension
func (ft *FileTreeExt) Register(r *mux.Router, root *mux.Router, basicAuth func(http.Handler) http.Handler) {
	// Raw node endpoint
	r.Handle("/node/{ref}", basicAuth(http.HandlerFunc(ft.nodeHandler())))

	// Public/semi-private handler
	dirHandler := http.HandlerFunc(ft.dirHandler())
	fileHandler := http.HandlerFunc(ft.fileHandler())

	r.Handle("/fs/root", basicAuth(http.HandlerFunc(ft.fsRootHandler())))
	r.Handle("/fs/{type}/{name}/", basicAuth(http.HandlerFunc(ft.fsHandler())))
	r.Handle("/fs/{type}/{name}/{path:.+}", basicAuth(http.HandlerFunc(ft.fsHandler())))
	// r.Handle("/fs", http.HandlerFunc(ft.fsHandler()))
	// r.Handle("/fs/{name}", http.HandlerFunc(ft.fsByNameHandler()))

	r.Handle("/upload", basicAuth(http.HandlerFunc(ft.uploadHandler())))

	// Hook the standard endpint
	r.Handle("/dir/{ref}", dirHandler)
	r.Handle("/file/{ref}", fileHandler)

	// r.Handle("/index/{ref}", basicAuth(http.HandlerFunc(ft.indexHandler())))

	// Enable shortcut path from the root
	root.Handle("/d/{ref}", dirHandler)
	root.Handle("/f/{ref}", fileHandler)
}

// Node holds the data about the file node (either file/dir), analog to a Meta
type Node struct {
	Name     string  `json:"name"`
	Type     string  `json:"type"`
	Size     int     `json:"size"`
	Hash     string  `json:"ref"`
	Children []*Node `json:"children,omitempty"`

	// FIXME(ts): rename to Metadata
	Data map[string]interface{} `json:"metadata,omitempty"`
	Info *Info                  `json:"info,omitempty"`

	Meta   *rnode.RawNode `json:"-"`
	parent *Node          `json:"-"`
	fs     *FS            `json:"-"`

	URL string `json:"url",omitempty`
}

// Update the given node with the given meta
func (ft *FileTreeExt) Update(n *Node, m *rnode.RawNode) (*Node, error) {
	newNode, err := metaToNode(m)
	if err != nil {
		return nil, err
	}
	newNode.fs = n.fs
	newNode.parent = n.parent
	// fmt.Printf("\n\n\n###Update: n=%+v\nn.meta=%+v\nn.parent=%+v\nm=%+v\nnewNode=%+v\n\n\n###", n, n.meta, n.parent, m, newNode)
	// if n.parent != nil {
	// 	fmt.Printf("n.parent.meta=%+v\n", n.parent.meta)

	// }
	if n.parent == nil {
		n.fs.Ref = newNode.Hash
		js, err := json.Marshal(n.fs)
		if err != nil {
			return nil, err
		}
		if ft.kvStore.Put(context.TODO(), fmt.Sprintf(FSKeyFmt, n.fs.Name), "", js, -1); err != nil {
			return nil, err
		}
		return newNode, nil
	}

	newRefs := []interface{}{newNode.Hash}
	newChildren := []*Node{newNode}

	for _, c := range newNode.parent.Children {
		if c.Name != n.Name {
			newRefs = append(newRefs, c.Hash)
			newChildren = append(newChildren, c)
		}
	}

	newNode.parent.Meta.Refs = newRefs
	newNode.parent.Children = newChildren

	// parentMeta := n.parent.meta
	// parentMeta.Refs = newRefs
	// n.parent.Children = newChildren

	newRef, data := newNode.parent.Meta.Encode()
	newNode.parent.Hash = newRef
	newNode.parent.Meta.Hash = newRef
	if err := ft.blobStore.Put(context.TODO(), &blob.Blob{Hash: newRef, Data: data}); err != nil {
		return nil, err
	}
	// n.parent.Hash = newRef
	// parentMeta.Hash = newRef
	// Propagate the change to the parents
	if _, err := ft.Update(newNode.parent, newNode.parent.Meta); err != nil {
		return nil, err
	}
	return newNode, nil
}

func (n *Node) Close() error {
	return nil
}

type byName []*Node

func (s byName) Len() int           { return len(s) }
func (s byName) Less(i, j int) bool { return s[i].Name < s[j].Name }
func (s byName) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }

func metaToNode(m *rnode.RawNode) (*Node, error) {
	n := &Node{
		Name: m.Name,
		Type: m.Type,
		Size: m.Size,
		Data: m.Metadata,
		Hash: m.Hash,
		Meta: m,
	}
	return n, nil
}

// fetchDir recursively fetch dir children
func (ft *FileTreeExt) fetchDir(n *Node, depth, maxDepth int) error {
	if depth > maxDepth {
		return nil
	}
	if n.Type == "dir" {
		n.Children = []*Node{}
		for _, ref := range n.Meta.Refs {
			cn, err := ft.nodeByRef(ref.(string))
			if err != nil {
				return err
			}
			n.Children = append(n.Children, cn)
			if err := ft.fetchDir(cn, depth+1, maxDepth); err != nil {
				return err
			}
		}
	}
	// n.meta.Close()
	return nil
}

// FS fetch the FileSystem by name, returns an empty one if not found
func (ft *FileTreeExt) FS(name string, newState bool) (*FS, error) {
	fs := &FS{}
	if !newState {
		kv, err := ft.kvStore.Get(context.TODO(), fmt.Sprintf(FSKeyFmt, name), -1)
		if err != nil && err != vkv.ErrNotFound {
			return nil, err
		}
		switch err {
		case nil:
			if err := json.Unmarshal([]byte(kv.Data), fs); err != nil {
				return nil, err
			}
		case vkv.ErrNotFound:
			// XXX(tsileo): should the `ErrNotFound` be returned here?
		default:
			return nil, err
		}
	}
	fs.Name = name
	fs.ft = ft
	return fs, nil
}

// Root fetch the FS root, and creates a new one if `create` is set to true (but it won't be savec automatically in the BlobStore
func (fs *FS) Root(create bool) (*Node, error) {
	fs.ft.log.Info("Root", "fs", fs)
	node, err := fs.ft.nodeByRef(fs.Ref)
	switch err {
	case blobsfile.ErrBlobNotFound:
		if !create {
			return nil, err
		}
		meta := &rnode.RawNode{
			Type: "dir",
			Name: "_root",
		}
		node, err = metaToNode(meta)
		if err != nil {
{'fraction_non_alphanumeric': 0.16224085720941067, 'fraction_numerical': 0.004600512462147682, 'mean_word_length': 3.4524241638579207, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5279810', 'n_tokens_mistral': 6785, 'n_tokens_neox': 6482, 'n_words': 1570}