<commit_before>// syncwatcher.go
package main

import (
	"bufio"
	"bytes"
	"crypto/tls"
	"encoding/json"
	"encoding/xml"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"text/tabwriter"
	"time"

	"github.com/cenkalti/backoff"
	"github.com/zillode/notify"
)

type Configuration struct {
	Version int
	Folders []FolderConfiguration
}

type FolderConfiguration struct {
	ID              string
	Path            string
	ReadOnly        bool
	RescanIntervalS int
}

type Pattern struct {
	match   *regexp.Regexp
	include bool
}

type Event struct {
	ID   int         `json:"id"`
	Time time.Time   `json:"time"`
	Type string      `json:"type"`
	Data interface{} `json:"data"`
}

type STEvent struct {
	Path     string
	Finished bool
}

type STNestedConfig struct {
	Config STConfig `xml:"gui"`
}

type STConfig struct {
	CsrfFile string
	ApiKey   string `xml:"apikey"`
	Target   string `xml:"address"`
	AuthUser string `xml:"user"`
	AuthPass string `xml:"password"`
	TLS      bool   `xml:"tls,attr"`
}

type folderSlice []string

type progressTime struct {
	fsEvent bool
	time    time.Time
}

func (fs *folderSlice) String() string {
	return fmt.Sprint(*fs)
}
func (fs *folderSlice) Set(value string) error {
	for _, f := range strings.Split(value, ",") {
		*fs = append(*fs, f)
	}
	return nil
}

// HTTP Authentication
var (
	target    string
	authUser  string
	authPass  string
	csrfToken string
	csrfFile  string
	apiKey    string
)

// HTTP Timeouts
var (
	requestTimeout = 180 * time.Second
)

// HTTP Debounce
var (
	debounceTimeout   = 500 * time.Millisecond
	configSyncTimeout = 5 * time.Second
	fsEventTimeout    = 5 * time.Second
	dirVsFiles        = 256
	maxFiles          = 5000
)

// Main
var (
	stop         = make(chan int)
	ignorePaths  = []string{".stversions", ".syncthing.", "~syncthing~"}
	Version      = "unknown-dev"
	Discard      = log.New(ioutil.Discard, "", log.Ldate)
	Warning      = Discard // 1
	OK           = Discard
	Trace        = Discard
	Debug        = Discard // 4
	watchFolders folderSlice
	skipFolders  folderSlice
	delayScan    = 3600
)

const (
	usage      = "syncthing-inotify [options]"
	extraUsage = `
The -logflags value is a sum of the following:

   1  Date
   2  Time
   4  Microsecond time
   8  Long filename
  16  Short filename

I.e. to prefix each log line with date and time, set -logflags=3 (1 + 2 from
above). The value 0 is used to disable all of the above. The default is to
show time only (2).`
)

func init() {
	c, _ := getSTConfig(getSTDefaultConfDir())
	if !strings.Contains(c.Target, "://") {
		if c.TLS {
			target = "https://" + c.Target
		} else {
			target = "http://" + c.Target
		}
	}

	var verbosity int
	var logflags int
	var home string
	var apiKeyStdin bool
	var authPassStdin bool
	var showVersion bool
	flag.IntVar(&verbosity, "verbosity", 2, "Logging level [1..4]")
	flag.IntVar(&logflags, "logflags", 2, "Select information in log line prefix")
	flag.StringVar(&home, "home", home, "Specify the home Syncthing dir to sniff configuration settings")
	flag.StringVar(&target, "target", target, "Target url (prepend with https:// for TLS)")
	flag.StringVar(&authUser, "user", c.AuthUser, "Username")
	flag.StringVar(&authPass, "password", "***", "Password")
	flag.StringVar(&csrfFile, "csrf", "", "CSRF token file")
	flag.StringVar(&apiKey, "api", c.ApiKey, "API key")
	flag.BoolVar(&apiKeyStdin, "api-stdin", false, "Provide API key through stdin")
	flag.BoolVar(&authPassStdin, "password-stdin", false, "Provide password through stdin")
	flag.Var(&watchFolders, "folders", "A comma-separated list of folders to watch (all by default)")
	flag.Var(&skipFolders, "skip-folders", "A comma-separated list of folders to skip inotify watching")
	flag.IntVar(&delayScan, "delay-scan", delayScan, "Automatically delay next scan interval (in seconds)")
	flag.BoolVar(&showVersion, "version", false, "Show version")

	flag.Usage = usageFor(flag.CommandLine, usage, fmt.Sprintf(extraUsage))
	flag.Parse()

	if showVersion {
		fmt.Printf("syncthing-inotify %s (%s %s-%s)\n", Version, runtime.Version(), runtime.GOOS, runtime.GOARCH)
		os.Exit(0)
	}

	if verbosity >= 1 {
		Warning = log.New(os.Stdout, "[WARNING] ", logflags)
	}
	if verbosity >= 2 {
		OK = log.New(os.Stdout, "[OK] ", logflags)
	}
	if verbosity >= 3 {
		Trace = log.New(os.Stdout, "[TRACE] ", logflags)
	}
	if verbosity >= 4 {
		Debug = log.New(os.Stdout, "[DEBUG] ", logflags)
	}

	if len(home) > 0 {
		c, err := getSTConfig(home)
		if err != nil {
			log.Fatalln(err)
		}
		if !strings.Contains(c.Target, "://") {
			if c.TLS {
				target = "https://" + c.Target
			} else {
				target = "http://" + c.Target
			}
			apiKey = c.ApiKey
		}
	}
	if !strings.Contains(target, "://") {
		target = "http://" + target
	}
	if len(csrfFile) > 0 {
		fd, err := os.Open(csrfFile)
		if err != nil {
			log.Fatalln(err)
		}
		s := bufio.NewScanner(fd)
		for s.Scan() {
			csrfToken = s.Text()
		}
		fd.Close()
	}
	if apiKeyStdin && authPassStdin {
		log.Fatalln("Either provide an API or password through stdin")
	}
	if apiKeyStdin {
		stdin := bufio.NewReader(os.Stdin)
		apiKey, _ = stdin.ReadString('\n')
	}
	if authPassStdin {
		stdin := bufio.NewReader(os.Stdin)
		authPass, _ = stdin.ReadString('\n')
	}
	if len(watchFolders) != 0 && len(skipFolders) != 0 {
		log.Fatalln("Either provide a list of folders to be watched or to be ignored, not both.")
	}
	if delayScan > 0 && delayScan < 60 {
		log.Fatalln("A delay scan interval shorter than 60 is not supported.")
	}
}

func main() {

	backoff.Retry(testWebGuiPost, backoff.NewExponentialBackOff())

	allFolders := getFolders()
	folders := filterFolders(allFolders)
	if len(folders) == 0 {
		log.Fatalln("No folders to be watched, exiting...")
	}
	stChans := make(map[string]chan STEvent, len(folders))
	for _, folder := range folders {
		Debug.Println("Installing watch for " + folder.ID)
		stChan := make(chan STEvent)
		stChans[folder.ID] = stChan
		go watchFolder(folder, stChan)
	}
	go watchSTEvents(stChans, allFolders) // Note: Lose thread ownership of stChans

	code := <-stop
	OK.Println("Exiting")
	os.Exit(code)

}

func restart() bool {
	pgm, err := exec.LookPath(os.Args[0])
	if err != nil {
		Warning.Println("Cannot restart:", err)
		return false
	}
	env := os.Environ()
	newEnv := make([]string, 0, len(env))
	for _, s := range env {
		newEnv = append(newEnv, s)
	}
	proc, err := os.StartProcess(pgm, os.Args, &os.ProcAttr{
		Env:   newEnv,
		Files: []*os.File{os.Stdin, os.Stdout, os.Stderr},
	})
	if err != nil {
		Warning.Println("Cannot restart:", err)
		return false
	}
	proc.Release()
	stop <- 3
	return true
}

func filterFolders(folders []FolderConfiguration) []FolderConfiguration {
	if len(watchFolders) > 0 {
		var fs []FolderConfiguration
		for _, f := range folders {
			for _, watch := range watchFolders {
				if f.ID == watch {
					fs = append(fs, f)
					break
				}
			}
		}
		return fs
	}
	if len(skipFolders) > 0 {
		var fs []FolderConfiguration
		for _, f := range folders {
			keep := true
			for _, skip := range skipFolders {
				if f.ID == skip {
					keep = false
					break
				}
			}
			if keep {
				fs = append(fs, f)
				break
			}
		}
		return fs
	}
	return folders
}

func getIgnorePatterns(folder string) []Pattern {
	for {
		Trace.Println("Getting ignore patterns for " + folder)
		r, err := http.NewRequest("GET", target+"/rest/db/ignores?folder="+url.QueryEscape(folder), nil)
		res, err := performRequest(r)
		defer func() {
			if res != nil && res.Body != nil {
				res.Body.Close()
			}
		}()
		if err != nil {
			Warning.Println("Failed to perform request /rest/db/ignores?folder="+url.QueryEscape(folder), err)
			time.Sleep(configSyncTimeout)
			continue
		}
		if res.StatusCode == 500 {
			Warning.Println("Syncthing not ready in " + folder + " for /rest/db/ignores")
			time.Sleep(configSyncTimeout)
			continue
		}
		if res.StatusCode != 200 {
			log.Fatalf("Status %d != 200 for GET /rest/db/ignores?folder=%s: %v\n", res.StatusCode, folder, res)
		}
		bs, err := ioutil.ReadAll(res.Body)
		if err != nil {
			log.Fatalln(err)
		}
		var ignores map[string][]string
		err = json.Unmarshal(bs, &ignores)
		if err != nil {
			log.Fatalln(err)
		}
		patterns := make([]Pattern, len(ignores["patterns"]))
		for i, str := range ignores["patterns"] {
			pattern := strings.TrimPrefix(str, "(?exclude)")
			regexp, err := regexp.Compile(pattern)
			if err != nil {
				log.Fatalln(err)
			}
			patterns[i] = Pattern{regexp, str == pattern}
		}
		return patterns
	}
}

func getFolders() []FolderConfiguration {
	Trace.Println("Getting Folders")
	r, err := http.NewRequest("GET", target+"/rest/system/config", nil)
	res, err := performRequest(r)
	defer func() {
		if res != nil && res.Body != nil {
			res.Body.Close()
		}
	}()
	if err != nil {
		log.Fatalln("Failed to perform request /rest/system/config: ", err)
	}
	if res.StatusCode != 200 {
		log.Fatalf("Status %d != 200 for GET /rest/system/config: ", res.StatusCode)
	}
	bs, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Fatalln(err)
	}
	var cfg Configuration
	err = json.Unmarshal(bs, &cfg)
	if err != nil {
		log.Fatalln(err)
	}
	return cfg.Folders
}

func watchFolder(folder FolderConfiguration, stInput chan STEvent) {
	folderPath := expandTilde(folder.Path)
	ignorePatterns := getIgnorePatterns(folder.ID)
	fsInput := make(chan string)
	c := make(chan notify.EventInfo, maxFiles)
	if err := notify.Watch(filepath.Join(folderPath, "..."), c, notify.All); err != nil {
		Warning.Println("Failed to install inotify handlers", err)
		informError("Failed to install inotify handler for " + folder.ID + ": " + err.Error())
		return
	}
	defer notify.Stop(c)
	go accumulateChanges(debounceTimeout, folder.ID, folderPath, dirVsFiles, stInput, fsInput, informChange)
	OK.Println("Watching " + folder.ID + ": " + folderPath)
	if folder.RescanIntervalS < 1800 && delayScan <= 0 {
		OK.Printf("The rescan interval of folder %s can be increased to 3600 (an hour) or even 86400 (a day) as changes should be observed immediately while syncthing-inotify is running.", folder.ID)
	}
	for {
		evPath := waitForEvent(c)
		Debug.Println("Change detected in: " + evPath + " (could still be ignored)")
		ev := relativePath(evPath, folderPath)
		if shouldIgnore(ignorePaths, ignorePatterns, ev) {
			continue
		}
		Trace.Println("Change detected in: " + evPath)
		fsInput <- ev
	}
}

func relativePath(path string, folderPath string) string {
	path = strings.TrimPrefix(path, folderPath)
	if len(path) == 0 {
		return path
	}
	if os.IsPathSeparator(path[0]) {
		path = path[1:len(path)]
	}
	return path
}

func waitForEvent(c chan notify.EventInfo) string {
	select {
	case ev, ok := <-c:
		if !ok {
			Warning.Println("Error: channel closed")
		}
		return ev.Path()
	}
	return ""
}

func shouldIgnore(ignorePaths []string, ignorePatterns []Pattern, path string) bool {
	if len(path) == 0 {
		return false
	}
	for _, ignorePath := range ignorePaths {
		if strings.Contains(path, ignorePath) {
			Debug.Println("Ignoring", path)
			return true
		}
	}
	for _, p1 := range ignorePatterns {
		if p1.include && p1.match.MatchString(path) {
			keep := false
			for _, p2 := range ignorePatterns {
				if !p2.include && p2.match.MatchString(path) {
					Debug.Println("Keeping", path, "because", p2.match.String())
					keep = true
					break
				}
			}
			if !keep {
				Debug.Println("Ignoring", path)
				return true
			}
		}
	}
	return false
}

func performRequest(r *http.Request) (*http.Response, error) {
	if r == nil {
		return nil, errors.New("Invalid HTTP Request object")
	}
	if len(csrfToken) > 0 {
		r.Header.Set("X-CSRF-Token", csrfToken)
	}
	if len(authUser) > 0 {
		r.SetBasicAuth(authUser, authPass)
	}
	if len(apiKey) > 0 {
		r.Header.Set("X-API-Key", apiKey)
	}
	tr := &http.Transport{
		TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},
		ResponseHeaderTimeout: requestTimeout,
		DisableKeepAlives:     true,
	}
	client := &http.Client{
		Transport: tr,
		Timeout:   requestTimeout,
	}
	res, err := client.Do(r)
	return res, err
}

func testWebGuiPost() error {
	Trace.Println("Testing WebGUI")
	r, err := http.NewRequest("GET", target+"/rest/404", nil)
	res, err := performRequest(r)
	defer func() {
		if res != nil && res.Body != nil {
			res.Body.Close()
		}
	}()
	if err != nil {
		Warning.Println("Cannot connect to Syncthing:", err)
		return err
	}
	body, _ := ioutil.ReadAll(res.Body)
	if res.StatusCode != 404 {
		Warning.Printf("Cannot connect to Syncthing, Status %d != 404 for GET\n", res.StatusCode, string(body))
		return errors.New("Invalid HTTP status code")
	}
	return nil
}

func informError(msg string) error {
	Trace.Printf("Informing ST about inotify error: %v", msg)
	r, _ := http.NewRequest("POST", target+"/rest/system/error", strings.NewReader("[Inotify] "+msg))
	r.Header.Set("Content-Type", "plain/text")
	res, err := performRequest(r)
	defer func() {
		if res != nil && res.Body != nil {
			res.Body.Close()
		}
	}()
	if err != nil {
		Warning.Println("Failed to inform Syncthing about", msg, err)
		return err
	}
	if res.StatusCode == 403 {
		Warning.Printf("Error: HTTP POST forbidden. Missing API key?")
		return errors.New("HTTP POST forbidden")
	}
	if res.StatusCode != 200 {
		Warning.Printf("Error: Status %d != 200 for POST.\n%v", res.StatusCode, msg)
		return errors.New("Invalid HTTP status code")
	}
	return err
}

func informChange(folder string, subs []string) error {
	data := url.Values{}
	data.Set("folder", folder)
	for _, sub := range subs {
		data.Add("sub", sub)
	}
	if delayScan > 0 {
		data.Set("next", strconv.Itoa(delayScan))
	}
	Trace.Printf("Informing ST: %v: %v", folder, subs)
	r, _ := http.NewRequest("POST", target+"/rest/db/scan?"+data.Encode(), nil)
	res, err := performRequest(r)
	defer func() {
		if res != nil && res.Body != nil {
			res.Body.Close()
		}
	}()
	if err != nil {
		Warning.Println("Failed to perform request", err)
		return err
	}
	if res.StatusCode == 403 {
		Warning.Printf("Error: HTTP POST forbidden. missing API key?")
		return errors.New("HTTP POST forbidden")
	}
	if res.StatusCode != 200 {
		Warning.Printf("Error: Status %d != 200 for POST.\n%v", res.StatusCode, folder)
		return errors.New("Invalid HTTP status code")
	} else {
		OK.Printf("Syncthing is indexing change in %v: %v", folder, subs)
	}
	// Wait until scan finishes
	_, err = ioutil.ReadAll(res.Body)<commit_msg>Reduce amount of temporary dir and file aggregation<commit_after>// syncwatcher.go
package main

import (
	"bufio"
	"bytes"
	"crypto/tls"
	"encoding/json"
	"encoding/xml"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"text/tabwriter"
	"time"

	"github.com/cenkalti/backoff"
	"github.com/zillode/notify"
)

type Configuration struct {
	Version int
	Folders []FolderConfiguration
}

type FolderConfiguration struct {
	ID              string
	Path            string
	ReadOnly        bool
	RescanIntervalS int
}

type Pattern struct {
	match   *regexp.Regexp
	include bool
}

type Event struct {
	ID   int         `json:"id"`
	Time time.Time   `json:"time"`
	Type string      `json:"type"`
	Data interface{} `json:"data"`
}

type STEvent struct {
	Path     string
	Finished bool
}

type STNestedConfig struct {
	Config STConfig `xml:"gui"`
}

type STConfig struct {
	CsrfFile string
	ApiKey   string `xml:"apikey"`
	Target   string `xml:"address"`
	AuthUser string `xml:"user"`
	AuthPass string `xml:"password"`
	TLS      bool   `xml:"tls,attr"`
}

type folderSlice []string

type progressTime struct {
	fsEvent bool
	time    time.Time
}

func (fs *folderSlice) String() string {
	return fmt.Sprint(*fs)
}
func (fs *folderSlice) Set(value string) error {
	for _, f := range strings.Split(value, ",") {
		*fs = append(*fs, f)
	}
	return nil
}

// HTTP Authentication
var (
	target    string
	authUser  string
	authPass  string
	csrfToken string
	csrfFile  string
	apiKey    string
)

// HTTP Timeouts
var (
	requestTimeout = 180 * time.Second
)

// HTTP Debounce
var (
	debounceTimeout   = 500 * time.Millisecond
	configSyncTimeout = 5 * time.Second
	fsEventTimeout    = 5 * time.Second
	dirVsFiles        = 128
	maxFiles          = 512
)

// Main
var (
	stop         = make(chan int)
	ignorePaths  = []string{".stversions", ".syncthing.", "~syncthing~"}
	Version      = "unknown-dev"
	Discard      = log.New(ioutil.Discard, "", log.Ldate)
	Warning      = Discard // 1
	OK           = Discard
	Trace        = Discard
	Debug        = Discard // 4
	watchFolders folderSlice
	skipFolders  folderSlice
	delayScan    = 3600
)

const (
	usage      = "syncthing-inotify [options]"
	extraUsage = `
The -logflags value is a sum of the following:

   1  Date
   2  Time
   4  Microsecond time
   8  Long filename
  16  Short filename

I.e. to prefix each log line with date and time, set -logflags=3 (1 + 2 from
above). The value 0 is used to disable all of the above. The default is to
show time only (2).`
)

func init() {
	c, _ := getSTConfig(getSTDefaultConfDir())
	if !strings.Contains(c.Target, "://") {
		if c.TLS {
			target = "https://" + c.Target
		} else {
			target = "http://" + c.Target
		}
	}

	var verbosity int
	var logflags int
	var home string
	var apiKeyStdin bool
	var authPassStdin bool
	var showVersion bool
	flag.IntVar(&verbosity, "verbosity", 2, "Logging level [1..4]")
	flag.IntVar(&logflags, "logflags", 2, "Select information in log line prefix")
	flag.StringVar(&home, "home", home, "Specify the home Syncthing dir to sniff configuration settings")
	flag.StringVar(&target, "target", target, "Target url (prepend with https:// for TLS)")
	flag.StringVar(&authUser, "user", c.AuthUser, "Username")
	flag.StringVar(&authPass, "password", "***", "Password")
	flag.StringVar(&csrfFile, "csrf", "", "CSRF token file")
	flag.StringVar(&apiKey, "api", c.ApiKey, "API key")
	flag.BoolVar(&apiKeyStdin, "api-stdin", false, "Provide API key through stdin")
	flag.BoolVar(&authPassStdin, "password-stdin", false, "Provide password through stdin")
	flag.Var(&watchFolders, "folders", "A comma-separated list of folders to watch (all by default)")
	flag.Var(&skipFolders, "skip-folders", "A comma-separated list of folders to skip inotify watching")
	flag.IntVar(&delayScan, "delay-scan", delayScan, "Automatically delay next scan interval (in seconds)")
	flag.BoolVar(&showVersion, "version", false, "Show version")

	flag.Usage = usageFor(flag.CommandLine, usage, fmt.Sprintf(extraUsage))
	flag.Parse()

	if showVersion {
		fmt.Printf("syncthing-inotify %s (%s %s-%s)\n", Version, runtime.Version(), runtime.GOOS, runtime.GOARCH)
		os.Exit(0)
	}

	if verbosity >= 1 {
		Warning = log.New(os.Stdout, "[WARNING] ", logflags)
	}
	if verbosity >= 2 {
		OK = log.New(os.Stdout, "[OK] ", logflags)
	}
	if verbosity >= 3 {
		Trace = log.New(os.Stdout, "[TRACE] ", logflags)
	}
	if verbosity >= 4 {
		Debug = log.New(os.Stdout, "[DEBUG] ", logflags)
	}

	if len(home) > 0 {
		c, err := getSTConfig(home)
		if err != nil {
			log.Fatalln(err)
		}
		if !strings.Contains(c.Target, "://") {
			if c.TLS {
				target = "https://" + c.Target
			} else {
				target = "http://" + c.Target
			}
			apiKey = c.ApiKey
		}
	}
	if !strings.Contains(target, "://") {
		target = "http://" + target
	}
	if len(csrfFile) > 0 {
		fd, err := os.Open(csrfFile)
		if err != nil {
			log.Fatalln(err)
		}
		s := bufio.NewScanner(fd)
		for s.Scan() {
			csrfToken = s.Text()
		}
		fd.Close()
	}
	if apiKeyStdin && authPassStdin {
		log.Fatalln("Either provide an API or password through stdin")
	}
	if apiKeyStdin {
		stdin := bufio.NewReader(os.Stdin)
		apiKey, _ = stdin.ReadString('\n')
	}
	if authPassStdin {
		stdin := bufio.NewReader(os.Stdin)
		authPass, _ = stdin.ReadString('\n')
	}
	if len(watchFolders) != 0 && len(skipFolders) != 0 {
		log.Fatalln("Either provide a list of folders to be watched or to be ignored, not both.")
	}
	if delayScan > 0 && delayScan < 60 {
		log.Fatalln("A delay scan interval shorter than 60 is not supported.")
	}
}

func main() {

	backoff.Retry(testWebGuiPost, backoff.NewExponentialBackOff())

	allFolders := getFolders()
	folders := filterFolders(allFolders)
	if len(folders) == 0 {
		log.Fatalln("No folders to be watched, exiting...")
	}
	stChans := make(map[string]chan STEvent, len(folders))
	for _, folder := range folders {
		Debug.Println("Installing watch for " + folder.ID)
		stChan := make(chan STEvent)
		stChans[folder.ID] = stChan
		go watchFolder(folder, stChan)
	}
	go watchSTEvents(stChans, allFolders) // Note: Lose thread ownership of stChans

	code := <-stop
	OK.Println("Exiting")
	os.Exit(code)

}

func restart() bool {
	pgm, err := exec.LookPath(os.Args[0])
	if err != nil {
		Warning.Println("Cannot restart:", err)
		return false
	}
	env := os.Environ()
	newEnv := make([]string, 0, len(env))
	for _, s := range env {
		newEnv = append(newEnv, s)
	}
	proc, err := os.StartProcess(pgm, os.Args, &os.ProcAttr{
		Env:   newEnv,
		Files: []*os.File{os.Stdin, os.Stdout, os.Stderr},
	})
	if err != nil {
		Warning.Println("Cannot restart:", err)
		return false
	}
	proc.Release()
	stop <- 3
	return true
}

func filterFolders(folders []FolderConfiguration) []FolderConfiguration {
	if len(watchFolders) > 0 {
		var fs []FolderConfiguration
		for _, f := range folders {
			for _, watch := range watchFolders {
				if f.ID == watch {
					fs = append(fs, f)
					break
				}
			}
		}
		return fs
	}
	if len(skipFolders) > 0 {
		var fs []FolderConfiguration
		for _, f := range folders {
			keep := true
			for _, skip := range skipFolders {
				if f.ID == skip {
					keep = false
					break
				}
			}
			if keep {
				fs = append(fs, f)
				break
			}
		}
		return fs
	}
	return folders
}

func getIgnorePatterns(folder string) []Pattern {
	for {
		Trace.Println("Getting ignore patterns for " + folder)
		r, err := http.NewRequest("GET", target+"/rest/db/ignores?folder="+url.QueryEscape(folder), nil)
		res, err := performRequest(r)
		defer func() {
			if res != nil && res.Body != nil {
				res.Body.Close()
			}
		}()
		if err != nil {
			Warning.Println("Failed to perform request /rest/db/ignores?folder="+url.QueryEscape(folder), err)
			time.Sleep(configSyncTimeout)
			continue
		}
		if res.StatusCode == 500 {
			Warning.Println("Syncthing not ready in " + folder + " for /rest/db/ignores")
			time.Sleep(configSyncTimeout)
			continue
		}
		if res.StatusCode != 200 {
			log.Fatalf("Status %d != 200 for GET /rest/db/ignores?folder=%s: %v\n", res.StatusCode, folder, res)
		}
		bs, err := ioutil.ReadAll(res.Body)
		if err != nil {
			log.Fatalln(err)
		}
		var ignores map[string][]string
		err = json.Unmarshal(bs, &ignores)
		if err != nil {
			log.Fatalln(err)
		}
		patterns := make([]Pattern, len(ignores["patterns"]))
		for i, str := range ignores["patterns"] {
			pattern := strings.TrimPrefix(str, "(?exclude)")
			regexp, err := regexp.Compile(pattern)
			if err != nil {
				log.Fatalln(err)
			}
			patterns[i] = Pattern{regexp, str == pattern}
		}
		return patterns
	}
}

func getFolders() []FolderConfiguration {
	Trace.Println("Getting Folders")
	r, err := http.NewRequest("GET", target+"/rest/system/config", nil)
	res, err := performRequest(r)
	defer func() {
		if res != nil && res.Body != nil {
			res.Body.Close()
		}
	}()
	if err != nil {
		log.Fatalln("Failed to perform request /rest/system/config: ", err)
	}
	if res.StatusCode != 200 {
		log.Fatalf("Status %d != 200 for GET /rest/system/config: ", res.StatusCode)
	}
	bs, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Fatalln(err)
	}
	var cfg Configuration
	err = json.Unmarshal(bs, &cfg)
	if err != nil {
		log.Fatalln(err)
	}
	return cfg.Folders
}

func watchFolder(folder FolderConfiguration, stInput chan STEvent) {
	folderPath := expandTilde(folder.Path)
	ignorePatterns := getIgnorePatterns(folder.ID)
	fsInput := make(chan string)
	c := make(chan notify.EventInfo, maxFiles)
	if err := notify.Watch(filepath.Join(folderPath, "..."), c, notify.All); err != nil {
		Warning.Println("Failed to install inotify handlers", err)
		informError("Failed to install inotify handler for " + folder.ID + ": " + err.Error())
		return
	}
	defer notify.Stop(c)
	go accumulateChanges(debounceTimeout, folder.ID, folderPath, dirVsFiles, stInput, fsInput, informChange)
	OK.Println("Watching " + folder.ID + ": " + folderPath)
	if folder.RescanIntervalS < 1800 && delayScan <= 0 {
		OK.Printf("The rescan interval of folder %s can be increased to 3600 (an hour) or even 86400 (a day) as changes should be observed immediately while syncthing-inotify is running.", folder.ID)
	}
	for {
		evPath := waitForEvent(c)
		Debug.Println("Change detected in: " + evPath + " (could still be ignored)")
		ev := relativePath(evPath, folderPath)
		if shouldIgnore(ignorePaths, ignorePatterns, ev) {
			continue
		}
		Trace.Println("Change detected in: " + evPath)
		fsInput <- ev
	}
}

func relativePath(path string, folderPath string) string {
	path = strings.TrimPrefix(path, folderPath)
	if len(path) == 0 {
		return path
	}
	if os.IsPathSeparator(path[0]) {
		path = path[1:len(path)]
	}
	return path
}

func waitForEvent(c chan notify.EventInfo) string {
	select {
	case ev, ok := <-c:
		if !ok {
			Warning.Println("Error: channel closed")
		}
		return ev.Path()
	}
	return ""
}

func shouldIgnore(ignorePaths []string, ignorePatterns []Pattern, path string) bool {
	if len(path) == 0 {
		return false
	}
	for _, ignorePath := range ignorePaths {
		if strings.Contains(path, ignorePath) {
			Debug.Println("Ignoring", path)
			return true
		}
	}
	for _, p1 := range ignorePatterns {
		if p1.include && p1.match.MatchString(path) {
			keep := false
			for _, p2 := range ignorePatterns {
				if !p2.include && p2.match.MatchString(path) {
					Debug.Println("Keeping", path, "because", p2.match.String())
					keep = true
					break
				}
			}
			if !keep {
				Debug.Println("Ignoring", path)
				return true
			}
		}
	}
	return false
}

func performRequest(r *http.Request) (*http.Response, error) {
	if r == nil {
		return nil, errors.New("Invalid HTTP Request object")
	}
	if len(csrfToken) > 0 {
		r.Header.Set("X-CSRF-Token", csrfToken)
	}
	if len(authUser) > 0 {
		r.SetBasicAuth(authUser, authPass)
	}
	if len(apiKey) > 0 {
		r.Header.Set("X-API-Key", apiKey)
	}
	tr := &http.Transport{
		TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},
		ResponseHeaderTimeout: requestTimeout,
		DisableKeepAlives:     true,
	}
	client := &http.Client{
		Transport: tr,
		Timeout:   requestTimeout,
	}
	res, err := client.Do(r)
	return res, err
}

func testWebGuiPost() error {
	Trace.Println("Testing WebGUI")
	r, err := http.NewRequest("GET", target+"/rest/404", nil)
	res, err := performRequest(r)
	defer func() {
		if res != nil && res.Body != nil {
			res.Body.Close()
		}
	}()
	if err != nil {
		Warning.Println("Cannot connect to Syncthing:", err)
		return err
	}
	body, _ := ioutil.ReadAll(res.Body)
	if res.StatusCode != 404 {
		Warning.Printf("Cannot connect to Syncthing, Status %d != 404 for GET\n", res.StatusCode, string(body))
		return errors.New("Invalid HTTP status code")
	}
	return nil
}

func informError(msg string) error {
	Trace.Printf("Informing ST about inotify error: %v", msg)
	r, _ := http.NewRequest("POST", target+"/rest/system/error", strings.NewReader("[Inotify] "+msg))
	r.Header.Set("Content-Type", "plain/text")
	res, err := performRequest(r)
	defer func() {
		if res != nil && res.Body != nil {
			res.Body.Close()
		}
	}()
	if err != nil {
		Warning.Println("Failed to inform Syncthing about", msg, err)
		return err
	}
	if res.StatusCode == 403 {
		Warning.Printf("Error: HTTP POST forbidden. Missing API key?")
		return errors.New("HTTP POST forbidden")
	}
	if res.StatusCode != 200 {
		Warning.Printf("Error: Status %d != 200 for POST: %v\n", res.StatusCode, msg)
		return errors.New("Invalid HTTP status code")
	}
	return err
}

func informChange(folder string, subs []string) error {
	data := url.Values{}
	data.Set("folder", folder)
	for _, sub := range subs {
		data.Add("sub", sub)
	}
	if delayScan > 0 {
		data.Set("next", strconv.Itoa(delayScan))
	}
	Trace.Printf("Informing ST: %v: %v", folder, subs)
	r, _ := http.NewRequest("POST", target+"/rest/db/scan?"+data.Encode(), nil)
	res, err := performRequest(r)
	defer func() {
		if res != nil && res.Body != nil {
			res.Body.Close()
		}
	}()
	if err != nil {
		Warning.Println("Failed to perform request", err)
		return err
	}
	if res.StatusCode == 403 {
		Warning.Printf("Error: HTTP POST forbidden. missing API key?")
		return errors.New("HTTP POST forbidden")
	}
	if res.StatusCode != 200 {
		msg, _ := ioutil.ReadAll(res.Body)
		Warning.Println(target + "/rest/db/scan?" + data.Encode())
		Warning.Printf("Error: Status %d != 200 for POST: %v, %s\n", res.StatusCode, folder, msg)
		return errors.New("Invalid HTTP status code")
	} else {
		OK.Printf("Syncthing is indexing change in %v: %v", folder, subs)
	}
	// Wait until scan finishes
	_, err = ioutil.ReadAll(res.Body)
{'fraction_non_alphanumeric': 0.1472692254780575, 'fraction_numerical': 0.008701334433897373, 'mean_word_length': 3.5206778606965172, 'pattern_counts': {'":': 10, '<': 17, '<?xml version=': 0, '>': 29, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '177363', 'n_tokens_mistral': 11322, 'n_tokens_neox': 10452, 'n_words': 2719}