<commit_before>// Package fs provides file system handleFuncs that can be used with kite
// library
package fs

import (
	"errors"
	"fmt"
	"log"
	"os"
	"path"
	"sync"

	"github.com/koding/kite"
	"github.com/koding/kite/dnode"
	"gopkg.in/fsnotify.v1"
)

var (
	// watcher variables
	once               sync.Once
	newPaths, oldPaths = make(chan string), make(chan string)

	// Limit of watching folders
	watchCallbacks = make(map[string]func(fsnotify.Event), 100)
	mu             sync.Mutex // protects watchCallbacks
)

func ReadDirectory(r *kite.Request) (interface{}, error) {
	var params struct {
		Path     string
		OnChange dnode.Function
	}

	if r.Args == nil {
		return nil, errors.New("arguments are not passed")
	}

	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		log.Println("params", params)
		return nil, errors.New("{ path: [string], onChange: [function]}")
	}

	response := make(map[string]interface{})

	if params.OnChange.IsValid() {
		onceBody := func() { startWatcher() }
		go once.Do(onceBody)

		var eventType string
		var fileEntry *FileEntry

		changer := func(ev fsnotify.Event) {
			// log.Println("event:", event)
			// if event.Op&fsnotify.Write == fsnotify.Write {
			// 	log.Println("modified file:", event.Name)
			// }

			fmt.Printf("ev %+v\n", ev)

			if ev.Op == fsnotify.Create {
				eventType = "added"
				fileEntry, _ = getInfo(ev.Name)
			} else if ev.Op == fsnotify.Remove {
				eventType = "removed"
				fileEntry = NewFileEntry(path.Base(ev.Name), ev.Name)
			}

			event := map[string]interface{}{
				"event": eventType,
				"file":  fileEntry,
			}

			// send back the result to the client
			params.OnChange.Call(event)
			return
		}

		mu.Lock()
		_, ok := watchCallbacks[params.Path]
		if !ok {
			// notify new paths to the watcher
			newPaths <- params.Path
			watchCallbacks[params.Path] = changer
		}
		mu.Unlock()

		removePath := func() {
			mu.Lock()
			delete(watchCallbacks, params.Path)
			mu.Unlock()

			oldPaths <- params.Path
		}

		// remove the path when the remote client disconnects
		r.Client.OnDisconnect(removePath)

		// this callback is called whenever we receive a 'stopWatching' from the client
		response["stopWatching"] = dnode.Callback(func(r *dnode.Partial) {
			removePath()
		})
	}

	files, err := readDirectory(params.Path)
	if err != nil {
		return nil, err
	}

	response["files"] = files
	return response, nil
}

func startWatcher() {
	var err error
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	go func() {
		for {
			select {
			case p := <-newPaths:
				err := watcher.Add(p)
				if err != nil {
					log.Println("watch path adding", err)
				}
			case p := <-oldPaths:
				err := watcher.Remove(p)
				if err != nil {
					log.Println("watch remove adding", err)
				}
			}
		}
	}()

	for {
		select {
		case event := <-watcher.Events:

			mu.Lock()
			f, ok := watchCallbacks[path.Dir(event.Name)]
			mu.Unlock()

			if !ok {
				continue
			}

			f(event)

		case err := <-watcher.Errors:
			log.Println("watcher error:", err)
		}
	}

}

func Glob(r *kite.Request) (interface{}, error) {
	var params struct {
		Pattern string
	}

	if r.Args.One().Unmarshal(&params) != nil || params.Pattern == "" {
		return nil, errors.New("{ pattern: [string] }")
	}

	return glob(params.Pattern)
}

func ReadFile(r *kite.Request) (interface{}, error) {
	var params struct {
		Path string
	}

	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		return nil, errors.New("{ path: [string] }")
	}

	return readFile(params.Path)
}

type writeFileParams struct {
	Path           string
	Content        []byte
	DoNotOverwrite bool
	Append         bool
}

func WriteFile(r *kite.Request) (interface{}, error) {
	var params writeFileParams
	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		return nil, errors.New("{ path: [string] }")
	}

	return writeFile(params.Path, params.Content, params.DoNotOverwrite, params.Append)
}

func UniquePath(r *kite.Request) (interface{}, error) {
	var params struct {
		Path string
	}

	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		return nil, errors.New("{ path: [string] }")
	}

	return uniquePath(params.Path)
}

func GetInfo(r *kite.Request) (interface{}, error) {
	var params struct {
		Path string
	}

	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		return nil, errors.New("{ path: [string] }")
	}

	return getInfo(params.Path)
}

type setPermissionsParams struct {
	Path      string
	Mode      os.FileMode
	Recursive bool
}

func SetPermissions(r *kite.Request) (interface{}, error) {
	var params setPermissionsParams

	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		return nil, errors.New("{ path: [string], mode: [integer], recursive: [bool] }")
	}

	err := setPermissions(params.Path, params.Mode, params.Recursive)
	if err != nil {
		return nil, err
	}

	return true, nil
}

func Remove(r *kite.Request) (interface{}, error) {
	var params struct {
		Path      string
		Recursive bool
	}

	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		return nil, errors.New("{ path: [string], recursive: [bool] }")
	}

	if err := remove(params.Path, params.Recursive); err != nil {
		return nil, err
	}

	return true, nil
}

func Rename(r *kite.Request) (interface{}, error) {
	var params struct {
		OldPath string
		NewPath string
	}

	if r.Args.One().Unmarshal(&params) != nil || params.OldPath == "" || params.NewPath == "" {
		return nil, errors.New("{ oldPath: [string], newPath: [string] }")
	}

	err := rename(params.OldPath, params.NewPath)
	if err != nil {
		return nil, err
	}

	return true, nil
}

func CreateDirectory(r *kite.Request) (interface{}, error) {
	var params struct {
		Path      string
		Recursive bool
	}

	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		return nil, errors.New("{ path: [string], recursive: [bool] }")
	}

	err := createDirectory(params.Path, params.Recursive)
	if err != nil {
		return nil, err
	}

	return true, nil
}

func Move(r *kite.Request) (interface{}, error) {
	var params struct {
		OldPath string
		NewPath string
	}

	if r.Args.One().Unmarshal(&params) != nil || params.OldPath == "" || params.NewPath == "" {
		return nil, errors.New("{ oldPath: [string], newPath: [string] }")
	}

	err := rename(params.OldPath, params.NewPath)
	if err != nil {
		return nil, err
	}

	return true, nil
}

func Copy(r *kite.Request) (interface{}, error) {
	var params struct {
		SrcPath string
		DstPath string
	}

	if r.Args.One().Unmarshal(&params) != nil || params.SrcPath == "" || params.DstPath == "" {
		return nil, errors.New("{ srcPath: [string], dstPath: [string] }")
	}

	err := cp(params.SrcPath, params.DstPath)
	if err != nil {
		return nil, err
	}

	return true, nil
}
<commit_msg>kite-handler: treat renaming as removed<commit_after>// Package fs provides file system handleFuncs that can be used with kite
// library
package fs

import (
	"errors"
	"log"
	"os"
	"path"
	"sync"

	"github.com/koding/kite"
	"github.com/koding/kite/dnode"
	"gopkg.in/fsnotify.v1"
)

var (
	// watcher variables
	once               sync.Once
	newPaths, oldPaths = make(chan string), make(chan string)

	// Limit of watching folders
	watchCallbacks = make(map[string]func(fsnotify.Event), 100)
	mu             sync.Mutex // protects watchCallbacks
)

func ReadDirectory(r *kite.Request) (interface{}, error) {
	var params struct {
		Path     string
		OnChange dnode.Function
	}

	if r.Args == nil {
		return nil, errors.New("arguments are not passed")
	}

	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		log.Println("params", params)
		return nil, errors.New("{ path: [string], onChange: [function]}")
	}

	response := make(map[string]interface{})

	if params.OnChange.IsValid() {
		onceBody := func() { startWatcher() }
		go once.Do(onceBody)

		var eventType string
		var fileEntry *FileEntry

		changer := func(ev fsnotify.Event) {
			switch ev.Op {
			case fsnotify.Create:
				eventType = "added"
				fileEntry, _ = getInfo(ev.Name)
			case fsnotify.Remove, fsnotify.Rename:
				eventType = "removed"
				fileEntry = NewFileEntry(path.Base(ev.Name), ev.Name)
			}

			event := map[string]interface{}{
				"event": eventType,
				"file":  fileEntry,
			}

			// send back the result to the client
			params.OnChange.Call(event)
			return
		}

		mu.Lock()
		_, ok := watchCallbacks[params.Path]
		if !ok {
			// notify new paths to the watcher
			newPaths <- params.Path
			watchCallbacks[params.Path] = changer
		}
		mu.Unlock()

		removePath := func() {
			mu.Lock()
			delete(watchCallbacks, params.Path)
			mu.Unlock()

			oldPaths <- params.Path
		}

		// remove the path when the remote client disconnects
		r.Client.OnDisconnect(removePath)

		// this callback is called whenever we receive a 'stopWatching' from the client
		response["stopWatching"] = dnode.Callback(func(r *dnode.Partial) {
			removePath()
		})
	}

	files, err := readDirectory(params.Path)
	if err != nil {
		return nil, err
	}

	response["files"] = files
	return response, nil
}

func startWatcher() {
	var err error
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	go func() {
		for {
			select {
			case p := <-newPaths:
				err := watcher.Add(p)
				if err != nil {
					log.Println("watch path adding", err)
				}
			case p := <-oldPaths:
				err := watcher.Remove(p)
				if err != nil {
					log.Println("watch remove adding", err)
				}
			}
		}
	}()

	for {
		select {
		case event := <-watcher.Events:

			mu.Lock()
			f, ok := watchCallbacks[path.Dir(event.Name)]
			mu.Unlock()

			if !ok {
				continue
			}

			f(event)

		case err := <-watcher.Errors:
			log.Println("watcher error:", err)
		}
	}

}

func Glob(r *kite.Request) (interface{}, error) {
	var params struct {
		Pattern string
	}

	if r.Args.One().Unmarshal(&params) != nil || params.Pattern == "" {
		return nil, errors.New("{ pattern: [string] }")
	}

	return glob(params.Pattern)
}

func ReadFile(r *kite.Request) (interface{}, error) {
	var params struct {
		Path string
	}

	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		return nil, errors.New("{ path: [string] }")
	}

	return readFile(params.Path)
}

type writeFileParams struct {
	Path           string
	Content        []byte
	DoNotOverwrite bool
	Append         bool
}

func WriteFile(r *kite.Request) (interface{}, error) {
	var params writeFileParams
	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		return nil, errors.New("{ path: [string] }")
	}

	return writeFile(params.Path, params.Content, params.DoNotOverwrite, params.Append)
}

func UniquePath(r *kite.Request) (interface{}, error) {
	var params struct {
		Path string
	}

	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		return nil, errors.New("{ path: [string] }")
	}

	return uniquePath(params.Path)
}

func GetInfo(r *kite.Request) (interface{}, error) {
	var params struct {
		Path string
	}

	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		return nil, errors.New("{ path: [string] }")
	}

	return getInfo(params.Path)
}

type setPermissionsParams struct {
	Path      string
	Mode      os.FileMode
	Recursive bool
}

func SetPermissions(r *kite.Request) (interface{}, error) {
	var params setPermissionsParams

	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		return nil, errors.New("{ path: [string], mode: [integer], recursive: [bool] }")
	}

	err := setPermissions(params.Path, params.Mode, params.Recursive)
	if err != nil {
		return nil, err
	}

	return true, nil
}

func Remove(r *kite.Request) (interface{}, error) {
	var params struct {
		Path      string
		Recursive bool
	}

	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		return nil, errors.New("{ path: [string], recursive: [bool] }")
	}

	if err := remove(params.Path, params.Recursive); err != nil {
		return nil, err
	}

	return true, nil
}

func Rename(r *kite.Request) (interface{}, error) {
	var params struct {
		OldPath string
		NewPath string
	}

	if r.Args.One().Unmarshal(&params) != nil || params.OldPath == "" || params.NewPath == "" {
		return nil, errors.New("{ oldPath: [string], newPath: [string] }")
	}

	err := rename(params.OldPath, params.NewPath)
	if err != nil {
		return nil, err
	}

	return true, nil
}

func CreateDirectory(r *kite.Request) (interface{}, error) {
	var params struct {
		Path      string
		Recursive bool
	}

	if r.Args.One().Unmarshal(&params) != nil || params.Path == "" {
		return nil, errors.New("{ path: [string], recursive: [bool] }")
	}

	err := createDirectory(params.Path, params.Recursive)
	if err != nil {
		return nil, err
	}

	return true, nil
}

func Move(r *kite.Request) (interface{}, error) {
	var params struct {
		OldPath string
		NewPath string
	}

	if r.Args.One().Unmarshal(&params) != nil || params.OldPath == "" || params.NewPath == "" {
		return nil, errors.New("{ oldPath: [string], newPath: [string] }")
	}

	err := rename(params.OldPath, params.NewPath)
	if err != nil {
		return nil, err
	}

	return true, nil
}

func Copy(r *kite.Request) (interface{}, error) {
	var params struct {
		SrcPath string
		DstPath string
	}

	if r.Args.One().Unmarshal(&params) != nil || params.SrcPath == "" || params.DstPath == "" {
		return nil, errors.New("{ srcPath: [string], dstPath: [string] }")
	}

	err := cp(params.SrcPath, params.DstPath)
	if err != nil {
		return nil, err
	}

	return true, nil
}

{'fraction_non_alphanumeric': 0.15668100252113304, 'fraction_numerical': 0.0005932077710218003, 'mean_word_length': 3.2802284988892416, 'pattern_counts': {'":': 4, '<': 15, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5377381', 'n_tokens_mistral': 5158, 'n_tokens_neox': 4668, 'n_words': 1198}