<commit_before>// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package model

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net"
	"path/filepath"
	"reflect"
	"regexp"
	"runtime"
	"strings"
	stdsync "sync"
	"time"

	"github.com/syncthing/syncthing/lib/config"
	"github.com/syncthing/syncthing/lib/connections"
	"github.com/syncthing/syncthing/lib/db"
	"github.com/syncthing/syncthing/lib/events"
	"github.com/syncthing/syncthing/lib/fs"
	"github.com/syncthing/syncthing/lib/ignore"
	"github.com/syncthing/syncthing/lib/osutil"
	"github.com/syncthing/syncthing/lib/protocol"
	"github.com/syncthing/syncthing/lib/scanner"
	"github.com/syncthing/syncthing/lib/stats"
	"github.com/syncthing/syncthing/lib/sync"
	"github.com/syncthing/syncthing/lib/upgrade"
	"github.com/syncthing/syncthing/lib/util"
	"github.com/syncthing/syncthing/lib/versioner"
	"github.com/thejerf/suture"
)

// How many files to send in each Index/IndexUpdate message.
const (
	maxBatchSizeBytes = 250 * 1024 // Aim for making index messages no larger than 250 KiB (uncompressed)
	maxBatchSizeFiles = 1000       // Either way, don't include more files than this
)

type service interface {
	BringToFront(string)
	Override()
	Revert()
	DelayScan(d time.Duration)
	SchedulePull()                                    // something relevant changed, we should try a pull
	Jobs(page, perpage int) ([]string, []string, int) // In progress, Queued, skipped
	Scan(subs []string) error
	Serve()
	Stop()
	CheckHealth() error
	Errors() []FileError
	WatchError() error
	ForceRescan(file protocol.FileInfo) error
	GetStatistics() stats.FolderStatistics

	getState() (folderState, time.Time, error)
}

type Availability struct {
	ID            protocol.DeviceID `json:"id"`
	FromTemporary bool              `json:"fromTemporary"`
}

type Model interface {
	suture.Service

	connections.Model

	AddFolder(cfg config.FolderConfiguration)
	RestartFolder(from, to config.FolderConfiguration)
	StartFolder(folder string)
	ResetFolder(folder string)
	DelayScan(folder string, next time.Duration)
	ScanFolder(folder string) error
	ScanFolders() map[string]error
	ScanFolderSubdirs(folder string, subs []string) error
	State(folder string) (string, time.Time, error)
	FolderErrors(folder string) ([]FileError, error)
	WatchError(folder string) error
	Override(folder string)
	Revert(folder string)
	BringToFront(folder, file string)
	GetIgnores(folder string) ([]string, []string, error)
	SetIgnores(folder string, content []string) error

	GetFolderVersions(folder string) (map[string][]versioner.FileVersion, error)
	RestoreFolderVersions(folder string, versions map[string]time.Time) (map[string]string, error)

	LocalChangedFiles(folder string, page, perpage int) []db.FileInfoTruncated
	NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, []db.FileInfoTruncated, []db.FileInfoTruncated)
	RemoteNeedFolderFiles(device protocol.DeviceID, folder string, page, perpage int) ([]db.FileInfoTruncated, error)
	CurrentFolderFile(folder string, file string) (protocol.FileInfo, bool)
	CurrentGlobalFile(folder string, file string) (protocol.FileInfo, bool)
	Availability(folder string, file protocol.FileInfo, block protocol.BlockInfo) []Availability

	GlobalSize(folder string) db.Counts
	LocalSize(folder string) db.Counts
	NeedSize(folder string) db.Counts
	ReceiveOnlyChangedSize(folder string) db.Counts

	CurrentSequence(folder string) (int64, bool)
	RemoteSequence(folder string) (int64, bool)

	Completion(device protocol.DeviceID, folder string) FolderCompletion
	ConnectionStats() map[string]interface{}
	DeviceStatistics() map[string]stats.DeviceStatistics
	FolderStatistics() map[string]stats.FolderStatistics
	UsageReportingStats(version int, preview bool) map[string]interface{}

	StartDeadlockDetector(timeout time.Duration)
	GlobalDirectoryTree(folder, prefix string, levels int, dirsonly bool) map[string]interface{}
}

type model struct {
	*suture.Supervisor

	cfg               config.Wrapper
	db                *db.Lowlevel
	finder            *db.BlockFinder
	progressEmitter   *ProgressEmitter
	id                protocol.DeviceID
	shortID           protocol.ShortID
	cacheIgnoredFiles bool
	protectedFiles    []string

	clientName    string
	clientVersion string

	fmut               sync.RWMutex                                           // protects the below
	folderCfgs         map[string]config.FolderConfiguration                  // folder -> cfg
	folderFiles        map[string]*db.FileSet                                 // folder -> files
	deviceStatRefs     map[protocol.DeviceID]*stats.DeviceStatisticsReference // deviceID -> statsRef
	folderIgnores      map[string]*ignore.Matcher                             // folder -> matcher object
	folderRunners      map[string]service                                     // folder -> puller or scanner
	folderRunnerTokens map[string][]suture.ServiceToken                       // folder -> tokens for puller or scanner
	folderRestartMuts  syncMutexMap                                           // folder -> restart mutex

	pmut                sync.RWMutex // protects the below
	conn                map[protocol.DeviceID]connections.Connection
	connRequestLimiters map[protocol.DeviceID]*byteSemaphore
	closed              map[protocol.DeviceID]chan struct{}
	helloMessages       map[protocol.DeviceID]protocol.HelloResult
	deviceDownloads     map[protocol.DeviceID]*deviceDownloadState
	remotePausedFolders map[protocol.DeviceID][]string // deviceID -> folders

	foldersRunning int32 // for testing only
}

type folderFactory func(*model, *db.FileSet, *ignore.Matcher, config.FolderConfiguration, versioner.Versioner, fs.Filesystem) service

var (
	folderFactories = make(map[config.FolderType]folderFactory)
)

var (
	errDeviceUnknown     = errors.New("unknown device")
	errDevicePaused      = errors.New("device is paused")
	errDeviceIgnored     = errors.New("device is ignored")
	ErrFolderPaused      = errors.New("folder is paused")
	errFolderNotRunning  = errors.New("folder is not running")
	errFolderMissing     = errors.New("no such folder")
	errNetworkNotAllowed = errors.New("network not allowed")
	// errors about why a connection is closed
	errIgnoredFolderRemoved = errors.New("folder no longer ignored")
	errReplacingConnection  = errors.New("replacing connection")
	errStopped              = errors.New("Syncthing is being stopped")
)

// NewModel creates and starts a new model. The model starts in read-only mode,
// where it sends index information to connected peers and responds to requests
// for file data without altering the local folder in any way.
func NewModel(cfg config.Wrapper, id protocol.DeviceID, clientName, clientVersion string, ldb *db.Lowlevel, protectedFiles []string) Model {
	m := &model{
		Supervisor: suture.New("model", suture.Spec{
			Log: func(line string) {
				l.Debugln(line)
			},
			PassThroughPanics: true,
		}),
		cfg:                 cfg,
		db:                  ldb,
		finder:              db.NewBlockFinder(ldb),
		progressEmitter:     NewProgressEmitter(cfg),
		id:                  id,
		shortID:             id.Short(),
		cacheIgnoredFiles:   cfg.Options().CacheIgnoredFiles,
		protectedFiles:      protectedFiles,
		clientName:          clientName,
		clientVersion:       clientVersion,
		folderCfgs:          make(map[string]config.FolderConfiguration),
		folderFiles:         make(map[string]*db.FileSet),
		deviceStatRefs:      make(map[protocol.DeviceID]*stats.DeviceStatisticsReference),
		folderIgnores:       make(map[string]*ignore.Matcher),
		folderRunners:       make(map[string]service),
		folderRunnerTokens:  make(map[string][]suture.ServiceToken),
		conn:                make(map[protocol.DeviceID]connections.Connection),
		connRequestLimiters: make(map[protocol.DeviceID]*byteSemaphore),
		closed:              make(map[protocol.DeviceID]chan struct{}),
		helloMessages:       make(map[protocol.DeviceID]protocol.HelloResult),
		deviceDownloads:     make(map[protocol.DeviceID]*deviceDownloadState),
		remotePausedFolders: make(map[protocol.DeviceID][]string),
		fmut:                sync.NewRWMutex(),
		pmut:                sync.NewRWMutex(),
	}
	m.Add(m.progressEmitter)
	scanLimiter.setCapacity(cfg.Options().MaxConcurrentScans)
	cfg.Subscribe(m)

	return m
}

func (m *model) Stop() {
	m.Supervisor.Stop()
	devs := m.cfg.Devices()
	ids := make([]protocol.DeviceID, 0, len(devs))
	for id := range devs {
		ids = append(ids, id)
	}
	w := m.closeConns(ids, errStopped)
	w.Wait()
}

// StartDeadlockDetector starts a deadlock detector on the models locks which
// causes panics in case the locks cannot be acquired in the given timeout
// period.
func (m *model) StartDeadlockDetector(timeout time.Duration) {
	l.Infof("Starting deadlock detector with %v timeout", timeout)
	detector := newDeadlockDetector(timeout)
	detector.Watch("fmut", m.fmut)
	detector.Watch("pmut", m.pmut)
}

// StartFolder constructs the folder service and starts it.
func (m *model) StartFolder(folder string) {
	m.fmut.Lock()
	defer m.fmut.Unlock()
	folderCfg := m.folderCfgs[folder]
	m.startFolderLocked(folderCfg)

	l.Infof("Ready to synchronize %s (%s)", folderCfg.Description(), folderCfg.Type)
}

// Need to hold lock on m.fmut when calling this.
func (m *model) startFolderLocked(cfg config.FolderConfiguration) {
	_, ok := m.folderRunners[cfg.ID]
	if ok {
		l.Warnln("Cannot start already running folder", cfg.Description())
		panic("cannot start already running folder")
	}

	folderFactory, ok := folderFactories[cfg.Type]
	if !ok {
		panic(fmt.Sprintf("unknown folder type 0x%x", cfg.Type))
	}

	folder := cfg.ID

	fset := m.folderFiles[folder]

	// Find any devices for which we hold the index in the db, but the folder
	// is not shared, and drop it.
	expected := mapDevices(cfg.DeviceIDs())
	for _, available := range fset.ListDevices() {
		if _, ok := expected[available]; !ok {
			l.Debugln("dropping", folder, "state for", available)
			fset.Drop(available)
		}
	}

	// Close connections to affected devices
	m.fmut.Unlock()
	m.closeConns(cfg.DeviceIDs(), fmt.Errorf("started folder %v", cfg.Description()))
	m.fmut.Lock()

	v, ok := fset.Sequence(protocol.LocalDeviceID), true
	indexHasFiles := ok && v > 0
	if !indexHasFiles {
		// It's a blank folder, so this may the first time we're looking at
		// it. Attempt to create and tag with our marker as appropriate. We
		// don't really do anything with errors at this point except warn -
		// if these things don't work, we still want to start the folder and
		// it'll show up as errored later.

		if err := cfg.CreateRoot(); err != nil {
			l.Warnln("Failed to create folder root directory", err)
		} else if err = cfg.CreateMarker(); err != nil {
			l.Warnln("Failed to create folder marker:", err)
		}
	}

	ver := cfg.Versioner()
	if service, ok := ver.(suture.Service); ok {
		// The versioner implements the suture.Service interface, so
		// expects to be run in the background in addition to being called
		// when files are going to be archived.
		token := m.Add(service)
		m.folderRunnerTokens[folder] = append(m.folderRunnerTokens[folder], token)
	}

	ffs := fset.MtimeFS()

	// These are our metadata files, and they should always be hidden.
	ffs.Hide(config.DefaultMarkerName)
	ffs.Hide(".stversions")
	ffs.Hide(".stignore")

	p := folderFactory(m, fset, m.folderIgnores[folder], cfg, ver, ffs)

	m.folderRunners[folder] = p

	m.warnAboutOverwritingProtectedFiles(folder)

	token := m.Add(p)
	m.folderRunnerTokens[folder] = append(m.folderRunnerTokens[folder], token)
}

func (m *model) warnAboutOverwritingProtectedFiles(folder string) {
	if m.folderCfgs[folder].Type == config.FolderTypeSendOnly {
		return
	}

	// This is a bit of a hack.
	ffs := m.folderCfgs[folder].Filesystem()
	if ffs.Type() != fs.FilesystemTypeBasic {
		return
	}
	folderLocation := ffs.URI()
	ignores := m.folderIgnores[folder]

	var filesAtRisk []string
	for _, protectedFilePath := range m.protectedFiles {
		// check if file is synced in this folder
		if protectedFilePath != folderLocation && !fs.IsParent(protectedFilePath, folderLocation) {
			continue
		}

		// check if file is ignored
		relPath, _ := filepath.Rel(folderLocation, protectedFilePath)
		if ignores.Match(relPath).IsIgnored() {
			continue
		}

		filesAtRisk = append(filesAtRisk, protectedFilePath)
	}

	if len(filesAtRisk) > 0 {
		l.Warnln("Some protected files may be overwritten and cause issues. See https://docs.syncthing.net/users/config.html#syncing-configuration-files for more information. The at risk files are:", strings.Join(filesAtRisk, ", "))
	}
}

func (m *model) AddFolder(cfg config.FolderConfiguration) {
	if len(cfg.ID) == 0 {
		panic("cannot add empty folder id")
	}

	if len(cfg.Path) == 0 {
		panic("cannot add empty folder path")
	}

	// Creating the fileset can take a long time (metadata calculation) so
	// we do it outside of the lock.
	fset := db.NewFileSet(cfg.ID, cfg.Filesystem(), m.db)

	m.fmut.Lock()
	defer m.fmut.Unlock()
	m.addFolderLocked(cfg, fset)
}

func (m *model) addFolderLocked(cfg config.FolderConfiguration, fset *db.FileSet) {
	m.folderCfgs[cfg.ID] = cfg
	m.folderFiles[cfg.ID] = fset

	ignores := ignore.New(cfg.Filesystem(), ignore.WithCache(m.cacheIgnoredFiles))
	if err := ignores.Load(".stignore"); err != nil && !fs.IsNotExist(err) {
		l.Warnln("Loading ignores:", err)
	}
	m.folderIgnores[cfg.ID] = ignores
}

func (m *model) RemoveFolder(cfg config.FolderConfiguration) {
	m.fmut.Lock()
	defer m.fmut.Unlock()

	// Delete syncthing specific files
	cfg.Filesystem().RemoveAll(config.DefaultMarkerName)

	m.tearDownFolderLocked(cfg, fmt.Errorf("removing folder %v", cfg.Description()))
	// Remove it from the database
	db.DropFolder(m.db, cfg.ID)
}

// Need to hold lock on m.fmut when calling this.
func (m *model) tearDownFolderLocked(cfg config.FolderConfiguration, err error) {
	// Stop the services running for this folder and wait for them to finish
	// stopping to prevent races on restart.
	tokens := m.folderRunnerTokens[cfg.ID]

	m.fmut.Unlock()

	// Close connections to affected devices
	// Must happen before stopping the folder service to abort ongoing
	// transmissions and thus allow timely service termination.
	w := m.closeConns(cfg.DeviceIDs(), err)

	for _, id := range tokens {
		m.RemoveAndWait(id, 0)
	}

	// Wait for connections to stop to ensure that no more calls to methods
	// expecting this folder to exist happen (e.g. .IndexUpdate).
	w.Wait()

	m.fmut.Lock()

	// Clean up our config maps
	delete(m.folderCfgs, cfg.ID)
	delete(m.folderFiles, cfg.ID)
	delete(m.folderIgnores, cfg.ID)
	delete(m.folderRunners, cfg.ID)
	delete(m.folderRunnerTokens, cfg.ID)
}

func (m *model) RestartFolder(from, to config.FolderConfiguration) {
	if len(to.ID) == 0 {
		panic("bug: cannot restart empty folder ID")
	}
	if to.ID != from.ID {
		l.Warnf("bug: folder restart cannot change ID %q -> %q", from.ID, to.ID)
		panic("bug: folder restart cannot change ID")
	}

	// This mutex protects the entirety of the restart operation, preventing
	// there from being more than one folder restart operation in progress
	// at any given time. The usual fmut/pmut stuff doesn't cover this,
	// because those locks are released while we are waiting for the folder
	// to shut down (and must be so because the folder might need them as
	// part of its operations before shutting down).
	restartMut := m.folderRestartMuts.Get(to.ID)
	restartMut.Lock()
	defer restartMut.Unlock()

	var infoMsg string
	var errMsg string
	switch {
	case to.Paused:
		infoMsg = "Paused"
		errMsg = "pausing"
	case from.Paused:
		infoMsg = "Unpaused"
		errMsg = "unpausing"
	default:
		infoMsg = "Restarted"
		errMsg = "restarting"
	}

	m.fmut.Lock()
	defer m.fmut.Unlock()

	m.tearDownFolderLocked(from, fmt.Errorf("%v folder %v", errMsg, to.Description()))
	if !to.Paused {
		// Creating the fileset can take a long time (metadata calculation)
		// so we do it outside of the lock.
		m.fmut.Unlock()
		fset := db.NewFileSet(to.ID, to.Filesystem(), m.db)
		m.fmut.Lock()
		m.addFolderLocked(to, fset)
		m.startFolderLocked(to)
	}
	l.Infof("%v folder %v (%v)", infoMsg, to.Description(), to.Type)
}

func (m *model) UsageReportingStats(version int, preview bool) map[string]interface{} {
	stats := make(map[string]interface{})
	if version >= 3 {
		// Block stats
		blockStatsMut.Lock()
		copyBlockStats := make(map[string]int)
		for k, v := range blockStats {
			copyBlockStats[k] = v
			if !preview {
				blockStats[k] = 0
			}
		}
		blockStatsMut.Unlock()
		stats["blockStats"] = copyBlockStats

		// Transport stats
		m.pmut.RLock()
		transportStats := make(map[string]int)
		for _, conn := range m.conn {
			transportStats[conn.Transport()]++
		}
		m.pmut.RUnlock()
		stats["transportStats"] = transportStats

		// Ignore stats
		ignoreStats := map[string]int{
			"lines":           0,
			"inverts":         0,
			"folded":          0,
			"deletable":       0,
			"rooted":          0,
			"includes":        0,
			"escapedIncludes": 0,
			"doubleStars":     0,
			"stars":           0,
		}
		var seenPrefix [3]bool
		for folder := range m.cfg.Folders() {
			lines, _, err := m.GetIgnores(folder)
			if err != nil {
				continue
			}
			ignoreStats["lines"] += len(lines)

			for _, line := range lines {
				// Allow prefixes to be specified in any order, but only once.
				for {
					if strings.HasPrefix(line, "!") && !seenPrefix[0] {
						seenPrefix[0] = true
						line = line[1:]
						ignoreStats["inverts"] += 1
					} else if strings.HasPrefix(line, "(?i)") && !seenPrefix[1] {
						seenPrefix[1] = true
						line = line[4:]
						ignoreStats["folded"] += 1
					} else if strings.HasPrefix(line, "(?d)") && !seenPrefix[2] {
						seenPrefix[2] = true
						line = line[4:]
						ignoreStats["deletable"] += 1
					} else {
						seenPrefix[0] = false
						seenPrefix[1] = false
						seenPrefix[2] = false
						break
					}
				}

				// Noops, remove
				line = strings.TrimSuffix(line, "**")
				line = strings.TrimPrefix(line, "**/")

				if strings.HasPrefix(line, "/") {
					ignoreStats["rooted"] += 1
				} else if strings.HasPrefix(line, "#include ") {
					ignoreStats["includes"] += 1
					if strings.Contains(line, "..") {
						ignoreStats["escapedIncludes"] += 1
					}
				}

				if strings.Contains(line, "**") {
					ignoreStats["doubleStars"] += 1
					// Remove not to trip up star checks.
					line = strings.Replace(line, "**", "", -1)
				}

				if strings.Contains(line, "*") {
					ignoreStats["stars"] += 1
				}
			}
		}
		stats["ignoreStats"] = ignoreStats
	}
	return stats
}

type ConnectionInfo struct {
	protocol.Statistics
	Connected     bool
	Paused        bool
	Address       string
	ClientVersion string
	Type          string
	Crypto        string
}

func (info ConnectionInfo) MarshalJSON() ([]byte, error) {
	return json.Marshal(map[string]interface{}{
		"at":            info.At,
		"inBytesTotal":  info.InBytesTotal,
		"outBytesTotal": info.OutBytesTotal,
		"connected":     info.Connected,
		"paused":        info.Paused,
		"address":       info.Address,
		"clientVersion": info.ClientVersion,
		"type":          info.Type,
		"crypto":        info.Crypto,
	})
}

// ConnectionStats returns a map with connection statistics for each device.
func (m *model) ConnectionStats() map[string]interface{} {
	m.pmut.RLock()
	defer m.pmut.RUnlock()

	res := make(map[string]interface{})
	devs := m.cfg.Devices()
	conns := make(map[string]ConnectionInfo, len(devs))
	for device, deviceCfg := range devs {
		hello := m.helloMessages[device]
		versionString := hello.ClientVersion
		if hello.ClientName != "syncthing" {
			versionString = hello.ClientName + " " + hello.ClientVersion
		}
		ci := ConnectionInfo{
			ClientVersion: strings.TrimSpace(versionString),
			Paused:        deviceCfg.Paused,
		}
		if conn, ok := m.conn[device]; ok {
			ci.Type = conn.Type()
			ci.Crypto = conn.Crypto()
			ci.Connected = ok
			ci.Statistics = conn.Statistics()
			if addr := conn.RemoteAddr(); addr != nil {
				ci.Address = addr.String()
			}
		}

		conns[device.String()] = ci
	}

	res["connections"] = conns

	in, out := protocol.TotalInOut()
	res["total"] = ConnectionInfo{
		Statistics: protocol.Statistics{
			At:            time.Now(),
			InBytesTotal:  in,
			OutBytesTotal: out,
		},
	}

	return res
}

// DeviceStatistics returns statistics about each device
func (m *model) DeviceStatistics() map[string]stats.DeviceStatistics {
	res := make(map[string]stats.DeviceStatistics)
	for id := range m.cfg.Devices() {
		res[id.String()] = m.deviceStatRef(id).GetStatistics()
	}
	return res
}

// FolderStatistics returns statistics about each folder
func (m *model) FolderStatistics() map[string]stats.FolderStatistics {
	res := make(map[string]stats.FolderStatistics)
	m.fmut.RLock()
	defer m.fmut.RUnlock()
	for id, runner := range m.folderRunners {
		res[id] = runner.GetStatistics()
	}
	return res
}

type FolderCompletion struct {
	CompletionPct float64
	NeedBytes     int64
	NeedItems     int64
	GlobalBytes   int64
	NeedDeletes   int64
}

// Map returns the members as a map, e.g. used in api to serialize as Json.
func (comp FolderCompletion) Map() map[string]interface{} {
	return map[string]interface{}{
		"completion":  comp.CompletionPct,
		"needBytes":   comp.NeedBytes,
		"needItems":   comp.NeedItems,
		"globalBytes": comp.GlobalBytes,
		"needDeletes": comp.NeedDeletes,
	}
}

// Completion returns the completion status, in percent, for the given device
// and folder.
func (m *model) Completion(device protocol.DeviceID, folder string) FolderCompletion {
	m.fmut.RLock()
	rf, ok := m.folderFiles[folder]
	m.fmut.RUnlock()
	if !ok {
		return FolderCompletion{} // Folder doesn't exist, so we hardly have any of it
	}

	tot := rf.GlobalSize().Bytes
	if tot == 0 {
		// Folder is empty, so we have all of it
		return FolderCompletion{
			CompletionPct: 100,
		}
	}

	m.pmut.RLock()
	counts := m.deviceDownloads[device].GetBlockCounts(folder)
	m.pmut.RUnlock()

	var need, items, fileNeed, downloaded, deletes int64
	rf.WithNeedTruncated(device, func(f db.FileIntf) bool {
		ft := f.(db.FileInfoTruncated)

		// If the file is deleted, we account it only in the deleted column.
		if ft.Deleted {
			deletes++
			return true
		}

		// This might might be more than it really is, because some blocks can be of a smaller size.
		downloaded = int64(counts[ft.Name]) * int64(ft.BlockSize())

		fileNeed = ft.FileSize() - downloaded
		if fileNeed < 0 {
			fileNeed = 0
		}

		need += fileNeed
		items++

		return true
	})

	needRatio := float64(need) / float64(tot)
	completionPct := 100 * (1 - needRatio)

	// If the completion is 100% but there are deletes we need to handle,
	// drop it down a notch. Hack for consumers that look only at the
	// percentage (our own GUI does the same calculation as here on its own
	// and needs the same fixup).
	if need == 0 && deletes > 0 {
		completionPct = 95 // chosen by fair dice roll
	}

	l.Debugf("%v Completion(%s, %q): %f (%d / %d = %f)", m, device, folder, completionPct, need, tot, needRatio)

	return FolderCompletion{
		CompletionPct: completionPct,
		NeedBytes:     need,
		NeedItems:     items,
		GlobalBytes:   tot,
		NeedDeletes:   deletes,
	}
}

func addSizeOfFile(s *db.Counts, f db.FileIntf) {
	switch {
	case f.IsDeleted():
		s.Deleted++
	case f.IsDirectory():
		s.Directories++
	case f.IsSymlink():
		s.Symlinks++
	default:
		s.Files++
	}
	s.Bytes += f.FileSize()
}

// GlobalSize returns the number of files, deleted files and total bytes for all
// files in the global model.
func (m *model) GlobalSize(folder string) db.Counts {
	m.fmut.RLock()
	defer m.fmut.RUnlock()
	if rf, ok := m.folderFiles[folder]; ok {
		return rf.GlobalSize()
	}
	return db.Counts{}
}

// LocalSize returns the number of files, deleted files and total bytes for all
// files in the local folder.
func (m *model) LocalSize(folder string) db.Counts {
	m.fmut.RLock()
	defer m.fmut.RUnlock()
	if rf, ok := m.folderFiles[folder]; ok {
		return rf.LocalSize()
	}
	return db.Counts{}
}

// ReceiveOnlyChangedSize returns the number of files, deleted files and
// total bytes for all files that have changed locally in a receieve only
// folder.
func (m *model) ReceiveOnlyChangedSize(folder string) db.Counts {
	m.fmut.RLock()
	defer m.fmut.RUnlock()
	if rf, ok := m.folderFiles[folder]; ok {
		return rf.ReceiveOnlyChangedSize()
	}
	return db.Counts{}
}

// NeedSize returns the number of currently needed files and their total size
// minus the amount that has already been downloaded.
func (m *model) NeedSize(folder string) db.Counts {
	m.fmut.RLock()
	rf, ok := m.folderFiles[folder]
	cfg := m.folderCfgs[folder]
	m.fmut.RUnlock()

	var result db.Counts
	if ok {
		rf.WithNeedTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool {
			if cfg.IgnoreDelete && f.IsDeleted() {
				return true
			}

			addSizeOfFile(&result, f)
			return true
		})
	}
	result.Bytes -= m.progressEmitter.BytesCompleted(folder)
	l.Debugf("%v NeedSize(%q): %v", m, folder, result)
	return result
}

// NeedFolderFiles returns paginated list of currently needed files in
// progress, queued, and to be queued on next puller iteration.
func (m *model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, []db.FileInfoTruncated, []db.FileInfoTruncated) {
	m.fmut.RLock()
	rf, rfOk := m.folderFiles[folder]
	runner, runnerOk := m.folderRunners[folder]
	cfg := m.folderCfgs[folder]
	m.fmut.RUnlock()

	if !rfOk {
		return nil, nil, nil
	}

	var progress, queued, rest []db.FileInfoTruncated
	var seen map[string]struct{}

	skip := (page - 1) * perpage
	get := perpage

	if runnerOk {
		progressNames, queuedNames, skipped := runner.Jobs(page, perpage)

		progress = make([]db.FileInfoTruncated, len(progressNames))
		queued = make([]db.FileInfoTruncated, len(queuedNames))
		seen = make(map[string]struct{}, len(progressNames)+len(queuedNames))

		for i, name := range progressNames {
			if f, ok := rf.GetGlobalTruncated(name); ok {
				progress[i] = f
				seen[name] = struct{}{}
			}
		}

		for i, name := range queuedNames {
			if f, ok := rf.GetGlobalTruncated(name); ok {
				queued[i] = f
				seen[name] = struct{}{}
			}
		}

		get -= len(seen)
		if get == 0 {
			return progress, queued, nil
		}
		skip -= skipped
	}

	rest = make([]db.FileInfoTruncated, 0, perpage)
	rf.WithNeedTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool {
		if cfg.IgnoreDelete && f.IsDeleted() {
			return true
		}

		if skip > 0 {
			skip--
			return true
		}
		ft := f.(db.FileInfoTruncated)
		if _, ok := seen[ft.Name]; !ok {
			rest = append(rest, ft)
			get--
		}
		return get > 0
	})

	return progress, queued, rest
}

// LocalChangedFiles returns a paginated list of currently needed files in
// progress, queued, and to be queued on next puller iteration, as well as the
// total number of files currently needed.
func (m *model) LocalChangedFiles(folder string, page, perpage int) []db.FileInfoTruncated {
	m.fmut.RLock()
	rf, ok := m.folderFiles[folder]
	fcfg := m.folderCfgs[folder]
	m.fmut.RUnlock()

	if !ok {
		return nil
	}
	if fcfg.Type != config.FolderTypeReceiveOnly {
		return nil
	}
	if rf.ReceiveOnlyChangedSize().TotalItems() == 0 {
		return nil
	}

	files := make([]db.FileInfoTruncated, 0, perpage)

	skip := (page - 1) * perpage
	get := perpage

	rf.WithHaveTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool {
		if !f.IsReceiveOnlyChanged() {
			return true
		}
		if skip > 0 {
			skip--
			return true
		}
		ft := f.(db.FileInfoTruncated)
		files = append(files, ft)
		get--
		return get > 0
	})

	return files
}

// RemoteNeedFolderFiles returns paginated list of currently needed files in
// progress, queued, and to be queued on next puller iteration, as well as the
// total number of files currently needed.
func (m *model) RemoteNeedFolderFiles(device protocol.DeviceID, folder string, page, perpage int) ([]db.FileInfoTruncated, error) {
	m.fmut.RLock()
	m.pmut.RLock()
	err := m.checkDeviceFolderConnectedLocked(device, folder)
	rf := m.folderFiles[folder]
	m.pmut.RUnlock()
	m.fmut.RUnlock()
	if err != nil {
		return nil, err
	}

	files := make([]db.FileInfoTruncated, 0, perpage)
	skip := (page - 1) * perpage
	get := perpage
	rf.WithNeedTruncated(device, func(f db.FileIntf) bool {
		if skip > 0 {
			skip--
			return true
		}
		files = append(files, f.(db.FileInfoTruncated))
		get--
		return get > 0
	})

	return files, nil
}

// Index is called when a new device is connected and we receive their full index.
// Implements the protocol.Model interface.
func (m *model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo) {
	m.handleIndex(deviceID, folder, fs, false)
}

// IndexUpdate is called for incremental updates to connected devices' indexes.
// Implements the protocol.Model interface.
func (m *model) IndexUpdate(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo) {
	m.handleIndex(deviceID, folder, fs, true)
}

func (m *model) handleIndex(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo, update bool) {
	op := "Index"
	if update {
		op += " update"
	}

	l.Debugf("%v (in): %s / %q: %d files", op, deviceID, folder, len(fs))

	if cfg, ok := m.cfg.Folder(folder); !ok || !cfg.SharedWith(deviceID) {
		l.Infof("%v for unexpected folder ID %q sent from device %q; ensure that the folder exists and that this device is selected under \"Share With\" in the folder configuration.", op, folder, deviceID)
		return
	} else if cfg.Paused {
		l.Debugf("%v for paused folder (ID %q) sent from device %q.", op, folder, deviceID)
		return
	}

	m.fmut.RLock()
	files, existing := m.folderFiles[folder]
	runner, running := m.folderRunners[folder]
	m.fmut.RUnlock()

	if !existing {
		l.Warnf("%v for nonexistent folder %q", op, folder)
		panic("handling index for nonexistent folder")
	}

	if running {
		defer runner.SchedulePull()
	} else if update {
		// Runner may legitimately not be set if this is the "cleanup" Index
		// message at startup.
		l.Warnf("%v for not running folder %q", op, folder)
		panic("handling index for not running folder")
	}

	m.pmut.RLock()
	m.deviceDownloads[deviceID].Update(folder, makeForgetUpdate(fs))
	m.pmut.RUnlock()

	if !update {
		files.Drop(deviceID)
	}
	for i := range fs {
		// The local flags should never be transmitted over the wire. Make
		// sure they look like they weren't.
		fs[i].LocalFlags = 0
	}
	files.Update(deviceID, fs)

	events.Default.Log(events.RemoteIndexUpdated, map[string]interface{}{
		"device":  deviceID.String(),
		"folder":  folder,
		"items":   len(fs),
		"version": files.Sequence(deviceID),
	})
}

func (m *model) ClusterConfig(deviceID protocol.DeviceID, cm protocol.ClusterConfig) {
	// Check the peer device's announced folders against our own. Emits events
	// for folders that we don't expect (unknown or not shared).
	// Also, collect a list of folders we do share, and if he's interested in
	// temporary indexes, subscribe the connection.

	tempIndexFolders := make([]string, 0, len(cm.Folders))

	m.pmut.RLock()
	conn, ok := m.conn[deviceID]
	closed := m.closed[deviceID]
	hello := m.helloMessages[deviceID]
	m.pmut.RUnlock()
	if !ok {
		panic("bug: ClusterConfig called on closed or nonexistent connection")
	}

	changed := false
	deviceCfg := m.cfg.Devices()[deviceID]

	// See issue #3802 - in short, we can't send modern symlink entries to older
	// clients.
	dropSymlinks := false
	if hello.ClientName == m.clientName && upgrade.CompareVersions(hello.ClientVersion, "v0.14.14") < 0 {
		l.Warnln("Not sending symlinks to old client", deviceID, "- please upgrade to v0.14.14 or newer")
		dropSymlinks = true
	}

	// Needs to happen outside of the fmut, as can cause CommitConfiguration
	if deviceCfg.AutoAcceptFolders {
		for _, folder := range cm.Folders {
			changed = m.handleAutoAccepts(deviceCfg, folder) || changed
		}
	}

	m.fmut.Lock()
	defer m.fmut.Unlock()
	var paused []string
	for _, folder := range cm.Folders {
		cfg, ok := m.cfg.Folder(folder.ID)
		if !ok || !cfg.SharedWith(deviceID) {
			if deviceCfg.IgnoredFolder(folder.ID) {
				l.Infof("Ignoring folder %s from device %s since we are configured to", folder.Description(), deviceID)
				continue
			}
			m.cfg.AddOrUpdatePendingFolder(folder.ID, folder.Label, deviceID)
			changed = true
			events.Default.Log(events.FolderRejected, map[string]string{
				"folder":      folder.ID,
				"folderLabel": folder.Label,
				"device":      deviceID.String(),
			})
			l.Infof("Unexpected folder %s sent from device %q; ensure that the folder exists and that this device is selected under \"Share With\" in the folder configuration.", folder.Description(), deviceID)
			continue
		}
		if folder.Paused {
			paused = append(paused, folder.ID)
			continue
		}
		if cfg.Paused {
			continue
		}
		fs, ok := m.folderFiles[folder.ID]
		if !ok {
			// Shouldn't happen because !cfg.Paused, but might happen
			// if the folder is about to be unpaused, but not yet.
			continue
		}

		if !folder.DisableTempIndexes {
			tempIndexFolders = append(tempIndexFolders, folder.ID)
		}

		myIndexID := fs.IndexID(protocol.LocalDeviceID)
		mySequence := fs.Sequence(protocol.LocalDeviceID)
		var startSequence int64

		for _, dev := range folder.Devices {
			if dev.ID == m.id {
				// This is the other side's description of what it knows
				// about us. Lets check to see if we can start sending index
				// updates directly or need to send the index from start...

				if dev.IndexID == myIndexID {
					// They say they've seen our index ID before, so we can
					// send a delta update only.

					if dev.MaxSequence > mySequence {
						// Safety check. They claim to have more or newer
						// index data than we have - either we have lost
						// index data, or reset the index without resetting
						// the IndexID, or something else weird has
						// happened. We send a full index to reset the
						// situation.
						l.Infof("Device %v folder %s is delta index compatible, but seems out of sync with reality", deviceID, folder.Description())
						startSequence = 0
						continue
					}

					l.Debugf("Device %v folder %s is delta index compatible (mlv=%d)", deviceID, folder.Description(), dev.MaxSequence)
					startSequence = dev.MaxSequence
				} else if dev.IndexID != 0 {
					// They say they've seen an index ID from us, but it's
					// not the right one. Either they are confused or we
					// must have reset our database since last talking to
					// them. We'll start with a full index transfer.
					l.Infof("Device %v folder %s has mismatching index ID for us (%v != %v)", deviceID, folder.Description(), dev.IndexID, myIndexID)
					startSequence = 0
				}
			} else if dev.ID == deviceID {
				// This is the other side's description of themselves. We
				// check to see that it matches the IndexID we have on file,
				// otherwise we drop our old index data and expect to get a
				// completely new set.

				theirIndexID := fs.IndexID(deviceID)
				if dev.IndexID == 0 {
					// They're not announcing an index ID. This means they
					// do not support delta indexes and we should clear any
					// information we have from them before accepting their
					// index, which will presumably be a full index.
					fs.Drop(deviceID)
				} else if dev.IndexID != theirIndexID {
					// The index ID we have on file is not what they're
					// announcing. They must have reset their database and
					// will probably send us a full index. We drop any
					// information we have and remember this new index ID
					// instead.
					l.Infof("Device %v folder %s has a new index ID (%v)", deviceID, folder.Description(), dev.IndexID)
					fs.Drop(deviceID)
					fs.SetIndexID(deviceID, dev.IndexID)
				} else {
					// They're sending a recognized index ID and will most
					// likely use delta indexes. We might already have files
					// that we need to pull so let the folder runner know
					// that it should recheck the index data.
					if runner := m.folderRunners[folder.ID]; runner != nil {
						defer runner.SchedulePull()
					}
				}
			}
		}

		is := &indexSender{
			conn:         conn,
			connClosed:   closed,
			folder:       folder.ID,
			fset:         fs,
			prevSequence: startSequence,
			dropSymlinks: dropSymlinks,
		}
		is.Service = util.AsService(is.serve)
		// The token isn't tracked as the service stops when the connection
		// terminates and is automatically removed from supervisor (by
		// implementing suture.IsCompletable).
		m.Add(is)
	}

	m.pmut.Lock()
	m.remotePausedFolders[deviceID] = paused
	m.pmut.Unlock()

	// This breaks if we send multiple CM messages during the same connection.
	if len(tempIndexFolders) > 0 {
		m.pmut.RLock()
		conn, ok := m.conn[deviceID]
		m.pmut.RUnlock()
		// In case we've got ClusterConfig, and the connection disappeared<commit_msg>lib/model: Less locking in ClusterConfig (#5943)<commit_after>// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package model

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net"
	"path/filepath"
	"reflect"
	"regexp"
	"runtime"
	"strings"
	stdsync "sync"
	"time"

	"github.com/syncthing/syncthing/lib/config"
	"github.com/syncthing/syncthing/lib/connections"
	"github.com/syncthing/syncthing/lib/db"
	"github.com/syncthing/syncthing/lib/events"
	"github.com/syncthing/syncthing/lib/fs"
	"github.com/syncthing/syncthing/lib/ignore"
	"github.com/syncthing/syncthing/lib/osutil"
	"github.com/syncthing/syncthing/lib/protocol"
	"github.com/syncthing/syncthing/lib/scanner"
	"github.com/syncthing/syncthing/lib/stats"
	"github.com/syncthing/syncthing/lib/sync"
	"github.com/syncthing/syncthing/lib/upgrade"
	"github.com/syncthing/syncthing/lib/util"
	"github.com/syncthing/syncthing/lib/versioner"
	"github.com/thejerf/suture"
)

// How many files to send in each Index/IndexUpdate message.
const (
	maxBatchSizeBytes = 250 * 1024 // Aim for making index messages no larger than 250 KiB (uncompressed)
	maxBatchSizeFiles = 1000       // Either way, don't include more files than this
)

type service interface {
	BringToFront(string)
	Override()
	Revert()
	DelayScan(d time.Duration)
	SchedulePull()                                    // something relevant changed, we should try a pull
	Jobs(page, perpage int) ([]string, []string, int) // In progress, Queued, skipped
	Scan(subs []string) error
	Serve()
	Stop()
	CheckHealth() error
	Errors() []FileError
	WatchError() error
	ForceRescan(file protocol.FileInfo) error
	GetStatistics() stats.FolderStatistics

	getState() (folderState, time.Time, error)
}

type Availability struct {
	ID            protocol.DeviceID `json:"id"`
	FromTemporary bool              `json:"fromTemporary"`
}

type Model interface {
	suture.Service

	connections.Model

	AddFolder(cfg config.FolderConfiguration)
	RestartFolder(from, to config.FolderConfiguration)
	StartFolder(folder string)
	ResetFolder(folder string)
	DelayScan(folder string, next time.Duration)
	ScanFolder(folder string) error
	ScanFolders() map[string]error
	ScanFolderSubdirs(folder string, subs []string) error
	State(folder string) (string, time.Time, error)
	FolderErrors(folder string) ([]FileError, error)
	WatchError(folder string) error
	Override(folder string)
	Revert(folder string)
	BringToFront(folder, file string)
	GetIgnores(folder string) ([]string, []string, error)
	SetIgnores(folder string, content []string) error

	GetFolderVersions(folder string) (map[string][]versioner.FileVersion, error)
	RestoreFolderVersions(folder string, versions map[string]time.Time) (map[string]string, error)

	LocalChangedFiles(folder string, page, perpage int) []db.FileInfoTruncated
	NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, []db.FileInfoTruncated, []db.FileInfoTruncated)
	RemoteNeedFolderFiles(device protocol.DeviceID, folder string, page, perpage int) ([]db.FileInfoTruncated, error)
	CurrentFolderFile(folder string, file string) (protocol.FileInfo, bool)
	CurrentGlobalFile(folder string, file string) (protocol.FileInfo, bool)
	Availability(folder string, file protocol.FileInfo, block protocol.BlockInfo) []Availability

	GlobalSize(folder string) db.Counts
	LocalSize(folder string) db.Counts
	NeedSize(folder string) db.Counts
	ReceiveOnlyChangedSize(folder string) db.Counts

	CurrentSequence(folder string) (int64, bool)
	RemoteSequence(folder string) (int64, bool)

	Completion(device protocol.DeviceID, folder string) FolderCompletion
	ConnectionStats() map[string]interface{}
	DeviceStatistics() map[string]stats.DeviceStatistics
	FolderStatistics() map[string]stats.FolderStatistics
	UsageReportingStats(version int, preview bool) map[string]interface{}

	StartDeadlockDetector(timeout time.Duration)
	GlobalDirectoryTree(folder, prefix string, levels int, dirsonly bool) map[string]interface{}
}

type model struct {
	*suture.Supervisor

	cfg               config.Wrapper
	db                *db.Lowlevel
	finder            *db.BlockFinder
	progressEmitter   *ProgressEmitter
	id                protocol.DeviceID
	shortID           protocol.ShortID
	cacheIgnoredFiles bool
	protectedFiles    []string

	clientName    string
	clientVersion string

	fmut               sync.RWMutex                                           // protects the below
	folderCfgs         map[string]config.FolderConfiguration                  // folder -> cfg
	folderFiles        map[string]*db.FileSet                                 // folder -> files
	deviceStatRefs     map[protocol.DeviceID]*stats.DeviceStatisticsReference // deviceID -> statsRef
	folderIgnores      map[string]*ignore.Matcher                             // folder -> matcher object
	folderRunners      map[string]service                                     // folder -> puller or scanner
	folderRunnerTokens map[string][]suture.ServiceToken                       // folder -> tokens for puller or scanner
	folderRestartMuts  syncMutexMap                                           // folder -> restart mutex

	pmut                sync.RWMutex // protects the below
	conn                map[protocol.DeviceID]connections.Connection
	connRequestLimiters map[protocol.DeviceID]*byteSemaphore
	closed              map[protocol.DeviceID]chan struct{}
	helloMessages       map[protocol.DeviceID]protocol.HelloResult
	deviceDownloads     map[protocol.DeviceID]*deviceDownloadState
	remotePausedFolders map[protocol.DeviceID][]string // deviceID -> folders

	foldersRunning int32 // for testing only
}

type folderFactory func(*model, *db.FileSet, *ignore.Matcher, config.FolderConfiguration, versioner.Versioner, fs.Filesystem) service

var (
	folderFactories = make(map[config.FolderType]folderFactory)
)

var (
	errDeviceUnknown     = errors.New("unknown device")
	errDevicePaused      = errors.New("device is paused")
	errDeviceIgnored     = errors.New("device is ignored")
	ErrFolderPaused      = errors.New("folder is paused")
	errFolderNotRunning  = errors.New("folder is not running")
	errFolderMissing     = errors.New("no such folder")
	errNetworkNotAllowed = errors.New("network not allowed")
	// errors about why a connection is closed
	errIgnoredFolderRemoved = errors.New("folder no longer ignored")
	errReplacingConnection  = errors.New("replacing connection")
	errStopped              = errors.New("Syncthing is being stopped")
)

// NewModel creates and starts a new model. The model starts in read-only mode,
// where it sends index information to connected peers and responds to requests
// for file data without altering the local folder in any way.
func NewModel(cfg config.Wrapper, id protocol.DeviceID, clientName, clientVersion string, ldb *db.Lowlevel, protectedFiles []string) Model {
	m := &model{
		Supervisor: suture.New("model", suture.Spec{
			Log: func(line string) {
				l.Debugln(line)
			},
			PassThroughPanics: true,
		}),
		cfg:                 cfg,
		db:                  ldb,
		finder:              db.NewBlockFinder(ldb),
		progressEmitter:     NewProgressEmitter(cfg),
		id:                  id,
		shortID:             id.Short(),
		cacheIgnoredFiles:   cfg.Options().CacheIgnoredFiles,
		protectedFiles:      protectedFiles,
		clientName:          clientName,
		clientVersion:       clientVersion,
		folderCfgs:          make(map[string]config.FolderConfiguration),
		folderFiles:         make(map[string]*db.FileSet),
		deviceStatRefs:      make(map[protocol.DeviceID]*stats.DeviceStatisticsReference),
		folderIgnores:       make(map[string]*ignore.Matcher),
		folderRunners:       make(map[string]service),
		folderRunnerTokens:  make(map[string][]suture.ServiceToken),
		conn:                make(map[protocol.DeviceID]connections.Connection),
		connRequestLimiters: make(map[protocol.DeviceID]*byteSemaphore),
		closed:              make(map[protocol.DeviceID]chan struct{}),
		helloMessages:       make(map[protocol.DeviceID]protocol.HelloResult),
		deviceDownloads:     make(map[protocol.DeviceID]*deviceDownloadState),
		remotePausedFolders: make(map[protocol.DeviceID][]string),
		fmut:                sync.NewRWMutex(),
		pmut:                sync.NewRWMutex(),
	}
	m.Add(m.progressEmitter)
	scanLimiter.setCapacity(cfg.Options().MaxConcurrentScans)
	cfg.Subscribe(m)

	return m
}

func (m *model) Stop() {
	m.Supervisor.Stop()
	devs := m.cfg.Devices()
	ids := make([]protocol.DeviceID, 0, len(devs))
	for id := range devs {
		ids = append(ids, id)
	}
	w := m.closeConns(ids, errStopped)
	w.Wait()
}

// StartDeadlockDetector starts a deadlock detector on the models locks which
// causes panics in case the locks cannot be acquired in the given timeout
// period.
func (m *model) StartDeadlockDetector(timeout time.Duration) {
	l.Infof("Starting deadlock detector with %v timeout", timeout)
	detector := newDeadlockDetector(timeout)
	detector.Watch("fmut", m.fmut)
	detector.Watch("pmut", m.pmut)
}

// StartFolder constructs the folder service and starts it.
func (m *model) StartFolder(folder string) {
	m.fmut.Lock()
	defer m.fmut.Unlock()
	folderCfg := m.folderCfgs[folder]
	m.startFolderLocked(folderCfg)

	l.Infof("Ready to synchronize %s (%s)", folderCfg.Description(), folderCfg.Type)
}

// Need to hold lock on m.fmut when calling this.
func (m *model) startFolderLocked(cfg config.FolderConfiguration) {
	_, ok := m.folderRunners[cfg.ID]
	if ok {
		l.Warnln("Cannot start already running folder", cfg.Description())
		panic("cannot start already running folder")
	}

	folderFactory, ok := folderFactories[cfg.Type]
	if !ok {
		panic(fmt.Sprintf("unknown folder type 0x%x", cfg.Type))
	}

	folder := cfg.ID

	fset := m.folderFiles[folder]

	// Find any devices for which we hold the index in the db, but the folder
	// is not shared, and drop it.
	expected := mapDevices(cfg.DeviceIDs())
	for _, available := range fset.ListDevices() {
		if _, ok := expected[available]; !ok {
			l.Debugln("dropping", folder, "state for", available)
			fset.Drop(available)
		}
	}

	// Close connections to affected devices
	m.fmut.Unlock()
	m.closeConns(cfg.DeviceIDs(), fmt.Errorf("started folder %v", cfg.Description()))
	m.fmut.Lock()

	v, ok := fset.Sequence(protocol.LocalDeviceID), true
	indexHasFiles := ok && v > 0
	if !indexHasFiles {
		// It's a blank folder, so this may the first time we're looking at
		// it. Attempt to create and tag with our marker as appropriate. We
		// don't really do anything with errors at this point except warn -
		// if these things don't work, we still want to start the folder and
		// it'll show up as errored later.

		if err := cfg.CreateRoot(); err != nil {
			l.Warnln("Failed to create folder root directory", err)
		} else if err = cfg.CreateMarker(); err != nil {
			l.Warnln("Failed to create folder marker:", err)
		}
	}

	ver := cfg.Versioner()
	if service, ok := ver.(suture.Service); ok {
		// The versioner implements the suture.Service interface, so
		// expects to be run in the background in addition to being called
		// when files are going to be archived.
		token := m.Add(service)
		m.folderRunnerTokens[folder] = append(m.folderRunnerTokens[folder], token)
	}

	ffs := fset.MtimeFS()

	// These are our metadata files, and they should always be hidden.
	ffs.Hide(config.DefaultMarkerName)
	ffs.Hide(".stversions")
	ffs.Hide(".stignore")

	p := folderFactory(m, fset, m.folderIgnores[folder], cfg, ver, ffs)

	m.folderRunners[folder] = p

	m.warnAboutOverwritingProtectedFiles(folder)

	token := m.Add(p)
	m.folderRunnerTokens[folder] = append(m.folderRunnerTokens[folder], token)
}

func (m *model) warnAboutOverwritingProtectedFiles(folder string) {
	if m.folderCfgs[folder].Type == config.FolderTypeSendOnly {
		return
	}

	// This is a bit of a hack.
	ffs := m.folderCfgs[folder].Filesystem()
	if ffs.Type() != fs.FilesystemTypeBasic {
		return
	}
	folderLocation := ffs.URI()
	ignores := m.folderIgnores[folder]

	var filesAtRisk []string
	for _, protectedFilePath := range m.protectedFiles {
		// check if file is synced in this folder
		if protectedFilePath != folderLocation && !fs.IsParent(protectedFilePath, folderLocation) {
			continue
		}

		// check if file is ignored
		relPath, _ := filepath.Rel(folderLocation, protectedFilePath)
		if ignores.Match(relPath).IsIgnored() {
			continue
		}

		filesAtRisk = append(filesAtRisk, protectedFilePath)
	}

	if len(filesAtRisk) > 0 {
		l.Warnln("Some protected files may be overwritten and cause issues. See https://docs.syncthing.net/users/config.html#syncing-configuration-files for more information. The at risk files are:", strings.Join(filesAtRisk, ", "))
	}
}

func (m *model) AddFolder(cfg config.FolderConfiguration) {
	if len(cfg.ID) == 0 {
		panic("cannot add empty folder id")
	}

	if len(cfg.Path) == 0 {
		panic("cannot add empty folder path")
	}

	// Creating the fileset can take a long time (metadata calculation) so
	// we do it outside of the lock.
	fset := db.NewFileSet(cfg.ID, cfg.Filesystem(), m.db)

	m.fmut.Lock()
	defer m.fmut.Unlock()
	m.addFolderLocked(cfg, fset)
}

func (m *model) addFolderLocked(cfg config.FolderConfiguration, fset *db.FileSet) {
	m.folderCfgs[cfg.ID] = cfg
	m.folderFiles[cfg.ID] = fset

	ignores := ignore.New(cfg.Filesystem(), ignore.WithCache(m.cacheIgnoredFiles))
	if err := ignores.Load(".stignore"); err != nil && !fs.IsNotExist(err) {
		l.Warnln("Loading ignores:", err)
	}
	m.folderIgnores[cfg.ID] = ignores
}

func (m *model) RemoveFolder(cfg config.FolderConfiguration) {
	m.fmut.Lock()
	defer m.fmut.Unlock()

	// Delete syncthing specific files
	cfg.Filesystem().RemoveAll(config.DefaultMarkerName)

	m.tearDownFolderLocked(cfg, fmt.Errorf("removing folder %v", cfg.Description()))
	// Remove it from the database
	db.DropFolder(m.db, cfg.ID)
}

// Need to hold lock on m.fmut when calling this.
func (m *model) tearDownFolderLocked(cfg config.FolderConfiguration, err error) {
	// Stop the services running for this folder and wait for them to finish
	// stopping to prevent races on restart.
	tokens := m.folderRunnerTokens[cfg.ID]

	m.fmut.Unlock()

	// Close connections to affected devices
	// Must happen before stopping the folder service to abort ongoing
	// transmissions and thus allow timely service termination.
	w := m.closeConns(cfg.DeviceIDs(), err)

	for _, id := range tokens {
		m.RemoveAndWait(id, 0)
	}

	// Wait for connections to stop to ensure that no more calls to methods
	// expecting this folder to exist happen (e.g. .IndexUpdate).
	w.Wait()

	m.fmut.Lock()

	// Clean up our config maps
	delete(m.folderCfgs, cfg.ID)
	delete(m.folderFiles, cfg.ID)
	delete(m.folderIgnores, cfg.ID)
	delete(m.folderRunners, cfg.ID)
	delete(m.folderRunnerTokens, cfg.ID)
}

func (m *model) RestartFolder(from, to config.FolderConfiguration) {
	if len(to.ID) == 0 {
		panic("bug: cannot restart empty folder ID")
	}
	if to.ID != from.ID {
		l.Warnf("bug: folder restart cannot change ID %q -> %q", from.ID, to.ID)
		panic("bug: folder restart cannot change ID")
	}

	// This mutex protects the entirety of the restart operation, preventing
	// there from being more than one folder restart operation in progress
	// at any given time. The usual fmut/pmut stuff doesn't cover this,
	// because those locks are released while we are waiting for the folder
	// to shut down (and must be so because the folder might need them as
	// part of its operations before shutting down).
	restartMut := m.folderRestartMuts.Get(to.ID)
	restartMut.Lock()
	defer restartMut.Unlock()

	var infoMsg string
	var errMsg string
	switch {
	case to.Paused:
		infoMsg = "Paused"
		errMsg = "pausing"
	case from.Paused:
		infoMsg = "Unpaused"
		errMsg = "unpausing"
	default:
		infoMsg = "Restarted"
		errMsg = "restarting"
	}

	m.fmut.Lock()
	defer m.fmut.Unlock()

	m.tearDownFolderLocked(from, fmt.Errorf("%v folder %v", errMsg, to.Description()))
	if !to.Paused {
		// Creating the fileset can take a long time (metadata calculation)
		// so we do it outside of the lock.
		m.fmut.Unlock()
		fset := db.NewFileSet(to.ID, to.Filesystem(), m.db)
		m.fmut.Lock()
		m.addFolderLocked(to, fset)
		m.startFolderLocked(to)
	}
	l.Infof("%v folder %v (%v)", infoMsg, to.Description(), to.Type)
}

func (m *model) UsageReportingStats(version int, preview bool) map[string]interface{} {
	stats := make(map[string]interface{})
	if version >= 3 {
		// Block stats
		blockStatsMut.Lock()
		copyBlockStats := make(map[string]int)
		for k, v := range blockStats {
			copyBlockStats[k] = v
			if !preview {
				blockStats[k] = 0
			}
		}
		blockStatsMut.Unlock()
		stats["blockStats"] = copyBlockStats

		// Transport stats
		m.pmut.RLock()
		transportStats := make(map[string]int)
		for _, conn := range m.conn {
			transportStats[conn.Transport()]++
		}
		m.pmut.RUnlock()
		stats["transportStats"] = transportStats

		// Ignore stats
		ignoreStats := map[string]int{
			"lines":           0,
			"inverts":         0,
			"folded":          0,
			"deletable":       0,
			"rooted":          0,
			"includes":        0,
			"escapedIncludes": 0,
			"doubleStars":     0,
			"stars":           0,
		}
		var seenPrefix [3]bool
		for folder := range m.cfg.Folders() {
			lines, _, err := m.GetIgnores(folder)
			if err != nil {
				continue
			}
			ignoreStats["lines"] += len(lines)

			for _, line := range lines {
				// Allow prefixes to be specified in any order, but only once.
				for {
					if strings.HasPrefix(line, "!") && !seenPrefix[0] {
						seenPrefix[0] = true
						line = line[1:]
						ignoreStats["inverts"] += 1
					} else if strings.HasPrefix(line, "(?i)") && !seenPrefix[1] {
						seenPrefix[1] = true
						line = line[4:]
						ignoreStats["folded"] += 1
					} else if strings.HasPrefix(line, "(?d)") && !seenPrefix[2] {
						seenPrefix[2] = true
						line = line[4:]
						ignoreStats["deletable"] += 1
					} else {
						seenPrefix[0] = false
						seenPrefix[1] = false
						seenPrefix[2] = false
						break
					}
				}

				// Noops, remove
				line = strings.TrimSuffix(line, "**")
				line = strings.TrimPrefix(line, "**/")

				if strings.HasPrefix(line, "/") {
					ignoreStats["rooted"] += 1
				} else if strings.HasPrefix(line, "#include ") {
					ignoreStats["includes"] += 1
					if strings.Contains(line, "..") {
						ignoreStats["escapedIncludes"] += 1
					}
				}

				if strings.Contains(line, "**") {
					ignoreStats["doubleStars"] += 1
					// Remove not to trip up star checks.
					line = strings.Replace(line, "**", "", -1)
				}

				if strings.Contains(line, "*") {
					ignoreStats["stars"] += 1
				}
			}
		}
		stats["ignoreStats"] = ignoreStats
	}
	return stats
}

type ConnectionInfo struct {
	protocol.Statistics
	Connected     bool
	Paused        bool
	Address       string
	ClientVersion string
	Type          string
	Crypto        string
}

func (info ConnectionInfo) MarshalJSON() ([]byte, error) {
	return json.Marshal(map[string]interface{}{
		"at":            info.At,
		"inBytesTotal":  info.InBytesTotal,
		"outBytesTotal": info.OutBytesTotal,
		"connected":     info.Connected,
		"paused":        info.Paused,
		"address":       info.Address,
		"clientVersion": info.ClientVersion,
		"type":          info.Type,
		"crypto":        info.Crypto,
	})
}

// ConnectionStats returns a map with connection statistics for each device.
func (m *model) ConnectionStats() map[string]interface{} {
	m.pmut.RLock()
	defer m.pmut.RUnlock()

	res := make(map[string]interface{})
	devs := m.cfg.Devices()
	conns := make(map[string]ConnectionInfo, len(devs))
	for device, deviceCfg := range devs {
		hello := m.helloMessages[device]
		versionString := hello.ClientVersion
		if hello.ClientName != "syncthing" {
			versionString = hello.ClientName + " " + hello.ClientVersion
		}
		ci := ConnectionInfo{
			ClientVersion: strings.TrimSpace(versionString),
			Paused:        deviceCfg.Paused,
		}
		if conn, ok := m.conn[device]; ok {
			ci.Type = conn.Type()
			ci.Crypto = conn.Crypto()
			ci.Connected = ok
			ci.Statistics = conn.Statistics()
			if addr := conn.RemoteAddr(); addr != nil {
				ci.Address = addr.String()
			}
		}

		conns[device.String()] = ci
	}

	res["connections"] = conns

	in, out := protocol.TotalInOut()
	res["total"] = ConnectionInfo{
		Statistics: protocol.Statistics{
			At:            time.Now(),
			InBytesTotal:  in,
			OutBytesTotal: out,
		},
	}

	return res
}

// DeviceStatistics returns statistics about each device
func (m *model) DeviceStatistics() map[string]stats.DeviceStatistics {
	res := make(map[string]stats.DeviceStatistics)
	for id := range m.cfg.Devices() {
		res[id.String()] = m.deviceStatRef(id).GetStatistics()
	}
	return res
}

// FolderStatistics returns statistics about each folder
func (m *model) FolderStatistics() map[string]stats.FolderStatistics {
	res := make(map[string]stats.FolderStatistics)
	m.fmut.RLock()
	defer m.fmut.RUnlock()
	for id, runner := range m.folderRunners {
		res[id] = runner.GetStatistics()
	}
	return res
}

type FolderCompletion struct {
	CompletionPct float64
	NeedBytes     int64
	NeedItems     int64
	GlobalBytes   int64
	NeedDeletes   int64
}

// Map returns the members as a map, e.g. used in api to serialize as Json.
func (comp FolderCompletion) Map() map[string]interface{} {
	return map[string]interface{}{
		"completion":  comp.CompletionPct,
		"needBytes":   comp.NeedBytes,
		"needItems":   comp.NeedItems,
		"globalBytes": comp.GlobalBytes,
		"needDeletes": comp.NeedDeletes,
	}
}

// Completion returns the completion status, in percent, for the given device
// and folder.
func (m *model) Completion(device protocol.DeviceID, folder string) FolderCompletion {
	m.fmut.RLock()
	rf, ok := m.folderFiles[folder]
	m.fmut.RUnlock()
	if !ok {
		return FolderCompletion{} // Folder doesn't exist, so we hardly have any of it
	}

	tot := rf.GlobalSize().Bytes
	if tot == 0 {
		// Folder is empty, so we have all of it
		return FolderCompletion{
			CompletionPct: 100,
		}
	}

	m.pmut.RLock()
	counts := m.deviceDownloads[device].GetBlockCounts(folder)
	m.pmut.RUnlock()

	var need, items, fileNeed, downloaded, deletes int64
	rf.WithNeedTruncated(device, func(f db.FileIntf) bool {
		ft := f.(db.FileInfoTruncated)

		// If the file is deleted, we account it only in the deleted column.
		if ft.Deleted {
			deletes++
			return true
		}

		// This might might be more than it really is, because some blocks can be of a smaller size.
		downloaded = int64(counts[ft.Name]) * int64(ft.BlockSize())

		fileNeed = ft.FileSize() - downloaded
		if fileNeed < 0 {
			fileNeed = 0
		}

		need += fileNeed
		items++

		return true
	})

	needRatio := float64(need) / float64(tot)
	completionPct := 100 * (1 - needRatio)

	// If the completion is 100% but there are deletes we need to handle,
	// drop it down a notch. Hack for consumers that look only at the
	// percentage (our own GUI does the same calculation as here on its own
	// and needs the same fixup).
	if need == 0 && deletes > 0 {
		completionPct = 95 // chosen by fair dice roll
	}

	l.Debugf("%v Completion(%s, %q): %f (%d / %d = %f)", m, device, folder, completionPct, need, tot, needRatio)

	return FolderCompletion{
		CompletionPct: completionPct,
		NeedBytes:     need,
		NeedItems:     items,
		GlobalBytes:   tot,
		NeedDeletes:   deletes,
	}
}

func addSizeOfFile(s *db.Counts, f db.FileIntf) {
	switch {
	case f.IsDeleted():
		s.Deleted++
	case f.IsDirectory():
		s.Directories++
	case f.IsSymlink():
		s.Symlinks++
	default:
		s.Files++
	}
	s.Bytes += f.FileSize()
}

// GlobalSize returns the number of files, deleted files and total bytes for all
// files in the global model.
func (m *model) GlobalSize(folder string) db.Counts {
	m.fmut.RLock()
	defer m.fmut.RUnlock()
	if rf, ok := m.folderFiles[folder]; ok {
		return rf.GlobalSize()
	}
	return db.Counts{}
}

// LocalSize returns the number of files, deleted files and total bytes for all
// files in the local folder.
func (m *model) LocalSize(folder string) db.Counts {
	m.fmut.RLock()
	defer m.fmut.RUnlock()
	if rf, ok := m.folderFiles[folder]; ok {
		return rf.LocalSize()
	}
	return db.Counts{}
}

// ReceiveOnlyChangedSize returns the number of files, deleted files and
// total bytes for all files that have changed locally in a receieve only
// folder.
func (m *model) ReceiveOnlyChangedSize(folder string) db.Counts {
	m.fmut.RLock()
	defer m.fmut.RUnlock()
	if rf, ok := m.folderFiles[folder]; ok {
		return rf.ReceiveOnlyChangedSize()
	}
	return db.Counts{}
}

// NeedSize returns the number of currently needed files and their total size
// minus the amount that has already been downloaded.
func (m *model) NeedSize(folder string) db.Counts {
	m.fmut.RLock()
	rf, ok := m.folderFiles[folder]
	cfg := m.folderCfgs[folder]
	m.fmut.RUnlock()

	var result db.Counts
	if ok {
		rf.WithNeedTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool {
			if cfg.IgnoreDelete && f.IsDeleted() {
				return true
			}

			addSizeOfFile(&result, f)
			return true
		})
	}
	result.Bytes -= m.progressEmitter.BytesCompleted(folder)
	l.Debugf("%v NeedSize(%q): %v", m, folder, result)
	return result
}

// NeedFolderFiles returns paginated list of currently needed files in
// progress, queued, and to be queued on next puller iteration.
func (m *model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, []db.FileInfoTruncated, []db.FileInfoTruncated) {
	m.fmut.RLock()
	rf, rfOk := m.folderFiles[folder]
	runner, runnerOk := m.folderRunners[folder]
	cfg := m.folderCfgs[folder]
	m.fmut.RUnlock()

	if !rfOk {
		return nil, nil, nil
	}

	var progress, queued, rest []db.FileInfoTruncated
	var seen map[string]struct{}

	skip := (page - 1) * perpage
	get := perpage

	if runnerOk {
		progressNames, queuedNames, skipped := runner.Jobs(page, perpage)

		progress = make([]db.FileInfoTruncated, len(progressNames))
		queued = make([]db.FileInfoTruncated, len(queuedNames))
		seen = make(map[string]struct{}, len(progressNames)+len(queuedNames))

		for i, name := range progressNames {
			if f, ok := rf.GetGlobalTruncated(name); ok {
				progress[i] = f
				seen[name] = struct{}{}
			}
		}

		for i, name := range queuedNames {
			if f, ok := rf.GetGlobalTruncated(name); ok {
				queued[i] = f
				seen[name] = struct{}{}
			}
		}

		get -= len(seen)
		if get == 0 {
			return progress, queued, nil
		}
		skip -= skipped
	}

	rest = make([]db.FileInfoTruncated, 0, perpage)
	rf.WithNeedTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool {
		if cfg.IgnoreDelete && f.IsDeleted() {
			return true
		}

		if skip > 0 {
			skip--
			return true
		}
		ft := f.(db.FileInfoTruncated)
		if _, ok := seen[ft.Name]; !ok {
			rest = append(rest, ft)
			get--
		}
		return get > 0
	})

	return progress, queued, rest
}

// LocalChangedFiles returns a paginated list of currently needed files in
// progress, queued, and to be queued on next puller iteration, as well as the
// total number of files currently needed.
func (m *model) LocalChangedFiles(folder string, page, perpage int) []db.FileInfoTruncated {
	m.fmut.RLock()
	rf, ok := m.folderFiles[folder]
	fcfg := m.folderCfgs[folder]
	m.fmut.RUnlock()

	if !ok {
		return nil
	}
	if fcfg.Type != config.FolderTypeReceiveOnly {
		return nil
	}
	if rf.ReceiveOnlyChangedSize().TotalItems() == 0 {
		return nil
	}

	files := make([]db.FileInfoTruncated, 0, perpage)

	skip := (page - 1) * perpage
	get := perpage

	rf.WithHaveTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool {
		if !f.IsReceiveOnlyChanged() {
			return true
		}
		if skip > 0 {
			skip--
			return true
		}
		ft := f.(db.FileInfoTruncated)
		files = append(files, ft)
		get--
		return get > 0
	})

	return files
}

// RemoteNeedFolderFiles returns paginated list of currently needed files in
// progress, queued, and to be queued on next puller iteration, as well as the
// total number of files currently needed.
func (m *model) RemoteNeedFolderFiles(device protocol.DeviceID, folder string, page, perpage int) ([]db.FileInfoTruncated, error) {
	m.fmut.RLock()
	m.pmut.RLock()
	err := m.checkDeviceFolderConnectedLocked(device, folder)
	rf := m.folderFiles[folder]
	m.pmut.RUnlock()
	m.fmut.RUnlock()
	if err != nil {
		return nil, err
	}

	files := make([]db.FileInfoTruncated, 0, perpage)
	skip := (page - 1) * perpage
	get := perpage
	rf.WithNeedTruncated(device, func(f db.FileIntf) bool {
		if skip > 0 {
			skip--
			return true
		}
		files = append(files, f.(db.FileInfoTruncated))
		get--
		return get > 0
	})

	return files, nil
}

// Index is called when a new device is connected and we receive their full index.
// Implements the protocol.Model interface.
func (m *model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo) {
	m.handleIndex(deviceID, folder, fs, false)
}

// IndexUpdate is called for incremental updates to connected devices' indexes.
// Implements the protocol.Model interface.
func (m *model) IndexUpdate(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo) {
	m.handleIndex(deviceID, folder, fs, true)
}

func (m *model) handleIndex(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo, update bool) {
	op := "Index"
	if update {
		op += " update"
	}

	l.Debugf("%v (in): %s / %q: %d files", op, deviceID, folder, len(fs))

	if cfg, ok := m.cfg.Folder(folder); !ok || !cfg.SharedWith(deviceID) {
		l.Infof("%v for unexpected folder ID %q sent from device %q; ensure that the folder exists and that this device is selected under \"Share With\" in the folder configuration.", op, folder, deviceID)
		return
	} else if cfg.Paused {
		l.Debugf("%v for paused folder (ID %q) sent from device %q.", op, folder, deviceID)
		return
	}

	m.fmut.RLock()
	files, existing := m.folderFiles[folder]
	runner, running := m.folderRunners[folder]
	m.fmut.RUnlock()

	if !existing {
		l.Warnf("%v for nonexistent folder %q", op, folder)
		panic("handling index for nonexistent folder")
	}

	if running {
		defer runner.SchedulePull()
	} else if update {
		// Runner may legitimately not be set if this is the "cleanup" Index
		// message at startup.
		l.Warnf("%v for not running folder %q", op, folder)
		panic("handling index for not running folder")
	}

	m.pmut.RLock()
	m.deviceDownloads[deviceID].Update(folder, makeForgetUpdate(fs))
	m.pmut.RUnlock()

	if !update {
		files.Drop(deviceID)
	}
	for i := range fs {
		// The local flags should never be transmitted over the wire. Make
		// sure they look like they weren't.
		fs[i].LocalFlags = 0
	}
	files.Update(deviceID, fs)

	events.Default.Log(events.RemoteIndexUpdated, map[string]interface{}{
		"device":  deviceID.String(),
		"folder":  folder,
		"items":   len(fs),
		"version": files.Sequence(deviceID),
	})
}

func (m *model) ClusterConfig(deviceID protocol.DeviceID, cm protocol.ClusterConfig) {
	// Check the peer device's announced folders against our own. Emits events
	// for folders that we don't expect (unknown or not shared).
	// Also, collect a list of folders we do share, and if he's interested in
	// temporary indexes, subscribe the connection.

	tempIndexFolders := make([]string, 0, len(cm.Folders))

	m.pmut.RLock()
	conn, ok := m.conn[deviceID]
	closed := m.closed[deviceID]
	hello := m.helloMessages[deviceID]
	m.pmut.RUnlock()
	if !ok {
		panic("bug: ClusterConfig called on closed or nonexistent connection")
	}

	changed := false
	deviceCfg := m.cfg.Devices()[deviceID]

	// See issue #3802 - in short, we can't send modern symlink entries to older
	// clients.
	dropSymlinks := false
	if hello.ClientName == m.clientName && upgrade.CompareVersions(hello.ClientVersion, "v0.14.14") < 0 {
		l.Warnln("Not sending symlinks to old client", deviceID, "- please upgrade to v0.14.14 or newer")
		dropSymlinks = true
	}

	// Needs to happen outside of the fmut, as can cause CommitConfiguration
	if deviceCfg.AutoAcceptFolders {
		for _, folder := range cm.Folders {
			changed = m.handleAutoAccepts(deviceCfg, folder) || changed
		}
	}

	m.fmut.RLock()
	var paused []string
	for _, folder := range cm.Folders {
		cfg, ok := m.cfg.Folder(folder.ID)
		if !ok || !cfg.SharedWith(deviceID) {
			if deviceCfg.IgnoredFolder(folder.ID) {
				l.Infof("Ignoring folder %s from device %s since we are configured to", folder.Description(), deviceID)
				continue
			}
			m.cfg.AddOrUpdatePendingFolder(folder.ID, folder.Label, deviceID)
			changed = true
			events.Default.Log(events.FolderRejected, map[string]string{
				"folder":      folder.ID,
				"folderLabel": folder.Label,
				"device":      deviceID.String(),
			})
			l.Infof("Unexpected folder %s sent from device %q; ensure that the folder exists and that this device is selected under \"Share With\" in the folder configuration.", folder.Description(), deviceID)
			continue
		}
		if folder.Paused {
			paused = append(paused, folder.ID)
			continue
		}
		if cfg.Paused {
			continue
		}
		fs, ok := m.folderFiles[folder.ID]
		if !ok {
			// Shouldn't happen because !cfg.Paused, but might happen
			// if the folder is about to be unpaused, but not yet.
			continue
		}

		if !folder.DisableTempIndexes {
			tempIndexFolders = append(tempIndexFolders, folder.ID)
		}

		myIndexID := fs.IndexID(protocol.LocalDeviceID)
		mySequence := fs.Sequence(protocol.LocalDeviceID)
		var startSequence int64

		for _, dev := range folder.Devices {
			if dev.ID == m.id {
				// This is the other side's description of what it knows
				// about us. Lets check to see if we can start sending index
				// updates directly or need to send the index from start...

				if dev.IndexID == myIndexID {
					// They say they've seen our index ID before, so we can
					// send a delta update only.

					if dev.MaxSequence > mySequence {
						// Safety check. They claim to have more or newer
						// index data than we have - either we have lost
						// index data, or reset the index without resetting
						// the IndexID, or something else weird has
						// happened. We send a full index to reset the
						// situation.
						l.Infof("Device %v folder %s is delta index compatible, but seems out of sync with reality", deviceID, folder.Description())
						startSequence = 0
						continue
					}

					l.Debugf("Device %v folder %s is delta index compatible (mlv=%d)", deviceID, folder.Description(), dev.MaxSequence)
					startSequence = dev.MaxSequence
				} else if dev.IndexID != 0 {
					// They say they've seen an index ID from us, but it's
					// not the right one. Either they are confused or we
					// must have reset our database since last talking to
					// them. We'll start with a full index transfer.
					l.Infof("Device %v folder %s has mismatching index ID for us (%v != %v)", deviceID, folder.Description(), dev.IndexID, myIndexID)
					startSequence = 0
				}
			} else if dev.ID == deviceID {
				// This is the other side's description of themselves. We
				// check to see that it matches the IndexID we have on file,
				// otherwise we drop our old index data and expect to get a
				// completely new set.

				theirIndexID := fs.IndexID(deviceID)
				if dev.IndexID == 0 {
					// They're not announcing an index ID. This means they
					// do not support delta indexes and we should clear any
					// information we have from them before accepting their
					// index, which will presumably be a full index.
					fs.Drop(deviceID)
				} else if dev.IndexID != theirIndexID {
					// The index ID we have on file is not what they're
					// announcing. They must have reset their database and
					// will probably send us a full index. We drop any
					// information we have and remember this new index ID
					// instead.
					l.Infof("Device %v folder %s has a new index ID (%v)", deviceID, folder.Description(), dev.IndexID)
					fs.Drop(deviceID)
					fs.SetIndexID(deviceID, dev.IndexID)
				} else {
					// They're sending a recognized index ID and will most
					// likely use delta indexes. We might already have files
					// that we need to pull so let the folder runner know
					// that it should recheck the index data.
					if runner := m.folderRunners[folder.ID]; runner != nil {
						defer runner.SchedulePull()
					}
				}
			}
		}

		is := &indexSender{
			conn:         conn,
			connClosed:   closed,
			folder:       folder.ID,
			fset:         fs,
			prevSequence: startSequence,
			dropSymlinks: dropSymlinks,
		}
		is.Service = util.AsService(is.serve)
		// The token isn't tracked as the service stops when the connection
		// terminates and is automatically removed from supervisor (by
		// implementing suture.IsCompletable).
		m.Add(is)
	}
	m.fmut.RUnlock()

	m.pmut.Lock()
	m.remotePausedFolders[deviceID] = paused
	m.pmut.Unlock()

	// This breaks if we send multiple CM messages during the same connection.
	if len(tempIndexFolders) > 0 {
		m.pmut.RLock()
		conn, ok := m.conn[deviceID]
		m.pmut.RUnlock()
		// In case we've got ClusterConfig, and the connection disappeared
{'fraction_non_alphanumeric': 0.11867060840454109, 'fraction_numerical': 0.00394173787443135, 'mean_word_length': 3.466417460508863, 'pattern_counts': {'":': 60, '<': 7, '<?xml version=': 0, '>': 45, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 34, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4848710', 'n_tokens_mistral': 26075, 'n_tokens_neox': 23648, 'n_words': 6945}