<commit_before>// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package model

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/d4l3k/messagediff"
	"github.com/syncthing/syncthing/lib/config"
	"github.com/syncthing/syncthing/lib/db"
	"github.com/syncthing/syncthing/lib/fs"
	"github.com/syncthing/syncthing/lib/ignore"
	"github.com/syncthing/syncthing/lib/osutil"
	"github.com/syncthing/syncthing/lib/protocol"
	srand "github.com/syncthing/syncthing/lib/rand"
	"github.com/syncthing/syncthing/lib/scanner"
	"github.com/syncthing/syncthing/lib/versioner"
)

var device1, device2 protocol.DeviceID
var defaultCfgWrapper *config.Wrapper
var defaultFolderConfig config.FolderConfiguration
var defaultFs fs.Filesystem
var defaultCfg config.Configuration
var defaultAutoAcceptCfg config.Configuration
var tmpLocation string

func init() {
	device1, _ = protocol.DeviceIDFromString("AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR")
	device2, _ = protocol.DeviceIDFromString("GYRZZQB-IRNPV4Z-T7TC52W-EQYJ3TT-FDQW6MW-DFLMU42-SSSU6EM-FBK2VAY")
	defaultFs = fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata")

	defaultFolderConfig = config.NewFolderConfiguration(protocol.LocalDeviceID, "default", "default", fs.FilesystemTypeBasic, "testdata")
	defaultFolderConfig.Devices = []config.FolderDeviceConfiguration{{DeviceID: device1}}
	defaultCfg = config.Configuration{
		Version: config.CurrentVersion,
		Folders: []config.FolderConfiguration{defaultFolderConfig},
		Devices: []config.DeviceConfiguration{config.NewDeviceConfiguration(device1, "device1")},
		Options: config.OptionsConfiguration{
			// Don't remove temporaries directly on startup
			KeepTemporariesH: 1,
		},
	}
	defaultAutoAcceptCfg = config.Configuration{
		Devices: []config.DeviceConfiguration{
			{
				DeviceID: protocol.LocalDeviceID, // self
			},
			{
				DeviceID:          device1,
				AutoAcceptFolders: true,
			},
		},
		Options: config.OptionsConfiguration{
			DefaultFolderPath: "testdata",
		},
	}
}

var testDataExpected = map[string]protocol.FileInfo{
	"foo": {
		Name:      "foo",
		Type:      protocol.FileInfoTypeFile,
		ModifiedS: 0,
		Blocks:    []protocol.BlockInfo{{Offset: 0x0, Size: 0x7, Hash: []uint8{0xae, 0xc0, 0x70, 0x64, 0x5f, 0xe5, 0x3e, 0xe3, 0xb3, 0x76, 0x30, 0x59, 0x37, 0x61, 0x34, 0xf0, 0x58, 0xcc, 0x33, 0x72, 0x47, 0xc9, 0x78, 0xad, 0xd1, 0x78, 0xb6, 0xcc, 0xdf, 0xb0, 0x1, 0x9f}}},
	},
	"empty": {
		Name:      "empty",
		Type:      protocol.FileInfoTypeFile,
		ModifiedS: 0,
		Blocks:    []protocol.BlockInfo{{Offset: 0x0, Size: 0x0, Hash: []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}}},
	},
	"bar": {
		Name:      "bar",
		Type:      protocol.FileInfoTypeFile,
		ModifiedS: 0,
		Blocks:    []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}},
	},
}

func init() {
	// Fix expected test data to match reality
	for n, f := range testDataExpected {
		fi, _ := os.Stat("testdata/" + n)
		f.Permissions = uint32(fi.Mode())
		f.ModifiedS = fi.ModTime().Unix()
		f.Size = fi.Size()
		testDataExpected[n] = f
	}
}

func TestMain(m *testing.M) {
	tmpLocation = "/tmp"
	if runtime.GOOS == "windows" {
		tmpLocation = filepath.Join("testdata", "tmp")
		if err := os.MkdirAll(tmpLocation, 0777); err != nil {
			panic(err)
		}
	}

	tmpName := fs.TempName("file")
	if err := osutil.Copy(defaultFs, "tmpfile", tmpName); err != nil {
		panic(err)
	}
	future := time.Now().Add(time.Hour)
	if err := os.Chtimes(filepath.Join("testdata", tmpName), future, future); err != nil {
		panic(err)
	}

	var wrapperPath string
	defaultCfgWrapper, wrapperPath = createTmpWrapper(defaultCfg)

	exitCode := m.Run()

	os.Remove(wrapperPath)
	defaultFs.Remove(tmpName)
	defaultFs.RemoveAll(config.DefaultMarkerName)
	defaultFs.RemoveAll(tmpLocation)

	os.Exit(exitCode)
}

func createTmpWrapper(cfg config.Configuration) (*config.Wrapper, string) {
	tmpFile, err := ioutil.TempFile(tmpLocation, "syncthing-testConfig-")
	if err != nil {
		panic(err)
	}
	wrapper := config.Wrap(tmpFile.Name(), cfg)
	tmpFile.Close()
	return wrapper, tmpFile.Name()
}

func newState(cfg config.Configuration) (*config.Wrapper, *Model) {
	db := db.OpenMemory()

	wcfg, path := createTmpWrapper(cfg)
	defer os.Remove(path)

	m := NewModel(wcfg, protocol.LocalDeviceID, "syncthing", "dev", db, nil)
	for _, folder := range cfg.Folders {
		if !folder.Paused {
			m.AddFolder(folder)
		}
	}
	m.ServeBackground()
	m.AddConnection(&fakeConnection{id: device1}, protocol.HelloResult{})
	return wcfg, m
}

func TestRequest(t *testing.T) {
	db := db.OpenMemory()

	m := NewModel(defaultCfgWrapper, protocol.LocalDeviceID, "syncthing", "dev", db, nil)

	// device1 shares default, but device2 doesn't
	m.AddFolder(defaultFolderConfig)
	m.StartFolder("default")
	m.ServeBackground()
	defer m.Stop()
	m.ScanFolder("default")

	bs := make([]byte, protocol.MinBlockSize)

	// Existing, shared file
	bs = bs[:6]
	err := m.Request(device1, "default", "foo", 0, nil, false, bs)
	if err != nil {
		t.Error(err)
	}
	if !bytes.Equal(bs, []byte("foobar")) {
		t.Errorf("Incorrect data from request: %q", string(bs))
	}

	// Existing, nonshared file
	err = m.Request(device2, "default", "foo", 0, nil, false, bs)
	if err == nil {
		t.Error("Unexpected nil error on insecure file read")
	}

	// Nonexistent file
	err = m.Request(device1, "default", "nonexistent", 0, nil, false, bs)
	if err == nil {
		t.Error("Unexpected nil error on insecure file read")
	}

	// Shared folder, but disallowed file name
	err = m.Request(device1, "default", "../walk.go", 0, nil, false, bs)
	if err == nil {
		t.Error("Unexpected nil error on insecure file read")
	}

	// Negative offset
	err = m.Request(device1, "default", "foo", -4, nil, false, bs[:0])
	if err == nil {
		t.Error("Unexpected nil error on insecure file read")
	}

	// Larger block than available
	bs = bs[:42]
	err = m.Request(device1, "default", "foo", 0, nil, false, bs)
	if err == nil {
		t.Error("Unexpected nil error on insecure file read")
	}
}

func genFiles(n int) []protocol.FileInfo {
	files := make([]protocol.FileInfo, n)
	t := time.Now().Unix()
	for i := 0; i < n; i++ {
		files[i] = protocol.FileInfo{
			Name:      fmt.Sprintf("file%d", i),
			ModifiedS: t,
			Sequence:  int64(i + 1),
			Blocks:    []protocol.BlockInfo{{Offset: 0, Size: 100, Hash: []byte("some hash bytes")}},
			Version:   protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1}}},
		}
	}

	return files
}

func BenchmarkIndex_10000(b *testing.B) {
	benchmarkIndex(b, 10000)
}

func BenchmarkIndex_100(b *testing.B) {
	benchmarkIndex(b, 100)
}

func benchmarkIndex(b *testing.B, nfiles int) {
	db := db.OpenMemory()
	m := NewModel(defaultCfgWrapper, protocol.LocalDeviceID, "syncthing", "dev", db, nil)
	m.AddFolder(defaultFolderConfig)
	m.StartFolder("default")
	m.ServeBackground()
	defer m.Stop()

	files := genFiles(nfiles)
	m.Index(device1, "default", files)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		m.Index(device1, "default", files)
	}
	b.ReportAllocs()
}

func BenchmarkIndexUpdate_10000_10000(b *testing.B) {
	benchmarkIndexUpdate(b, 10000, 10000)
}

func BenchmarkIndexUpdate_10000_100(b *testing.B) {
	benchmarkIndexUpdate(b, 10000, 100)
}

func BenchmarkIndexUpdate_10000_1(b *testing.B) {
	benchmarkIndexUpdate(b, 10000, 1)
}

func benchmarkIndexUpdate(b *testing.B, nfiles, nufiles int) {
	db := db.OpenMemory()
	m := NewModel(defaultCfgWrapper, protocol.LocalDeviceID, "syncthing", "dev", db, nil)
	m.AddFolder(defaultFolderConfig)
	m.StartFolder("default")
	m.ServeBackground()
	defer m.Stop()

	files := genFiles(nfiles)
	ufiles := genFiles(nufiles)

	m.Index(device1, "default", files)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		m.IndexUpdate(device1, "default", ufiles)
	}
	b.ReportAllocs()
}

type downloadProgressMessage struct {
	folder  string
	updates []protocol.FileDownloadProgressUpdate
}

type fakeConnection struct {
	id                       protocol.DeviceID
	downloadProgressMessages []downloadProgressMessage
	closed                   bool
	files                    []protocol.FileInfo
	fileData                 map[string][]byte
	folder                   string
	model                    *Model
	indexFn                  func(string, []protocol.FileInfo)
	requestFn                func(folder, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error)
	mut                      sync.Mutex
}

func (f *fakeConnection) Close() error {
	f.mut.Lock()
	defer f.mut.Unlock()
	f.closed = true
	return nil
}

func (f *fakeConnection) Start() {
}

func (f *fakeConnection) ID() protocol.DeviceID {
	return f.id
}

func (f *fakeConnection) Name() string {
	return ""
}

func (f *fakeConnection) String() string {
	return ""
}

func (f *fakeConnection) Option(string) string {
	return ""
}

func (f *fakeConnection) Index(folder string, fs []protocol.FileInfo) error {
	f.mut.Lock()
	defer f.mut.Unlock()
	if f.indexFn != nil {
		f.indexFn(folder, fs)
	}
	return nil
}

func (f *fakeConnection) IndexUpdate(folder string, fs []protocol.FileInfo) error {
	f.mut.Lock()
	defer f.mut.Unlock()
	if f.indexFn != nil {
		f.indexFn(folder, fs)
	}
	return nil
}

func (f *fakeConnection) Request(folder, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) {
	f.mut.Lock()
	defer f.mut.Unlock()
	if f.requestFn != nil {
		return f.requestFn(folder, name, offset, size, hash, fromTemporary)
	}
	return f.fileData[name], nil
}

func (f *fakeConnection) ClusterConfig(protocol.ClusterConfig) {}

func (f *fakeConnection) Ping() bool {
	f.mut.Lock()
	defer f.mut.Unlock()
	return f.closed
}

func (f *fakeConnection) Closed() bool {
	f.mut.Lock()
	defer f.mut.Unlock()
	return f.closed
}

func (f *fakeConnection) Statistics() protocol.Statistics {
	return protocol.Statistics{}
}

func (f *fakeConnection) RemoteAddr() net.Addr {
	return &fakeAddr{}
}

func (f *fakeConnection) Type() string {
	return "fake"
}
func (f *fakeConnection) Transport() string {
	return "fake"
}
func (f *fakeConnection) Priority() int {
	return 9000
}

func (f *fakeConnection) DownloadProgress(folder string, updates []protocol.FileDownloadProgressUpdate) {
	f.downloadProgressMessages = append(f.downloadProgressMessages, downloadProgressMessage{
		folder:  folder,
		updates: updates,
	})
}

func (f *fakeConnection) addFile(name string, flags uint32, ftype protocol.FileInfoType, data []byte) {
	f.mut.Lock()
	defer f.mut.Unlock()

	blockSize := protocol.BlockSize(int64(len(data)))
	blocks, _ := scanner.Blocks(context.TODO(), bytes.NewReader(data), blockSize, int64(len(data)), nil, true)
	var version protocol.Vector
	version = version.Update(f.id.Short())

	if ftype == protocol.FileInfoTypeFile || ftype == protocol.FileInfoTypeDirectory {
		f.files = append(f.files, protocol.FileInfo{
			Name:         name,
			Type:         ftype,
			Size:         int64(len(data)),
			ModifiedS:    time.Now().Unix(),
			Permissions:  flags,
			Version:      version,
			Sequence:     time.Now().UnixNano(),
			RawBlockSize: int32(blockSize),
			Blocks:       blocks,
		})
	} else {
		// Symlink
		f.files = append(f.files, protocol.FileInfo{
			Name:          name,
			Type:          ftype,
			Version:       version,
			Sequence:      time.Now().UnixNano(),
			SymlinkTarget: string(data),
		})
	}

	if f.fileData == nil {
		f.fileData = make(map[string][]byte)
	}
	f.fileData[name] = data
}

func (f *fakeConnection) deleteFile(name string) {
	f.mut.Lock()
	defer f.mut.Unlock()

	for i, fi := range f.files {
		if fi.Name == name {
			fi.Deleted = true
			fi.ModifiedS = time.Now().Unix()
			fi.Version = fi.Version.Update(f.id.Short())
			fi.Sequence = time.Now().UnixNano()
			fi.Blocks = nil

			f.files = append(append(f.files[:i], f.files[i+1:]...), fi)
			return
		}
	}
}

func (f *fakeConnection) sendIndexUpdate() {
	f.model.IndexUpdate(f.id, f.folder, f.files)
}

func BenchmarkRequestOut(b *testing.B) {
	db := db.OpenMemory()
	m := NewModel(defaultCfgWrapper, protocol.LocalDeviceID, "syncthing", "dev", db, nil)
	m.AddFolder(defaultFolderConfig)
	m.ServeBackground()
	defer m.Stop()
	m.ScanFolder("default")

	const n = 1000
	files := genFiles(n)

	fc := &fakeConnection{id: device1}
	for _, f := range files {
		fc.addFile(f.Name, 0644, protocol.FileInfoTypeFile, []byte("some data to return"))
	}
	m.AddConnection(fc, protocol.HelloResult{})
	m.Index(device1, "default", files)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		data, err := m.requestGlobal(device1, "default", files[i%n].Name, 0, 32, nil, false)
		if err != nil {
			b.Error(err)
		}
		if data == nil {
			b.Error("nil data")
		}
	}
}

func BenchmarkRequestInSingleFile(b *testing.B) {
	db := db.OpenMemory()
	m := NewModel(defaultCfgWrapper, protocol.LocalDeviceID, "syncthing", "dev", db, nil)
	m.AddFolder(defaultFolderConfig)
	m.ServeBackground()
	defer m.Stop()
	m.ScanFolder("default")

	buf := make([]byte, 128<<10)
	rand.Read(buf)
	os.RemoveAll("testdata/request")
	defer os.RemoveAll("testdata/request")
	os.MkdirAll("testdata/request/for/a/file/in/a/couple/of/dirs", 0755)
	ioutil.WriteFile("testdata/request/for/a/file/in/a/couple/of/dirs/128k", buf, 0644)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		if err := m.Request(device1, "default", "request/for/a/file/in/a/couple/of/dirs/128k", 0, nil, false, buf); err != nil {
			b.Error(err)
		}
	}

	b.SetBytes(128 << 10)
}

func TestDeviceRename(t *testing.T) {
	hello := protocol.HelloResult{
		ClientName:    "syncthing",
		ClientVersion: "v0.9.4",
	}
	defer os.Remove("testdata/tmpconfig.xml")

	rawCfg := config.New(device1)
	rawCfg.Devices = []config.DeviceConfiguration{
		{
			DeviceID: device1,
		},
	}
	cfg := config.Wrap("testdata/tmpconfig.xml", rawCfg)

	db := db.OpenMemory()
	m := NewModel(cfg, protocol.LocalDeviceID, "syncthing", "dev", db, nil)

	if cfg.Devices()[device1].Name != "" {
		t.Errorf("Device already has a name")
	}

	conn := &fakeConnection{id: device1}

	m.AddConnection(conn, hello)

	m.ServeBackground()
	defer m.Stop()

	if cfg.Devices()[device1].Name != "" {
		t.Errorf("Device already has a name")
	}

	m.Closed(conn, protocol.ErrTimeout)
	hello.DeviceName = "tester"
	m.AddConnection(conn, hello)

	if cfg.Devices()[device1].Name != "tester" {
		t.Errorf("Device did not get a name")
	}

	m.Closed(conn, protocol.ErrTimeout)
	hello.DeviceName = "tester2"
	m.AddConnection(conn, hello)

	if cfg.Devices()[device1].Name != "tester" {
		t.Errorf("Device name got overwritten")
	}

	cfgw, err := config.Load("testdata/tmpconfig.xml", protocol.LocalDeviceID)
	if err != nil {
		t.Error(err)
		return
	}
	if cfgw.Devices()[device1].Name != "tester" {
		t.Errorf("Device name not saved in config")
	}

	m.Closed(conn, protocol.ErrTimeout)

	opts := cfg.Options()
	opts.OverwriteRemoteDevNames = true
	cfg.SetOptions(opts)

	hello.DeviceName = "tester2"
	m.AddConnection(conn, hello)

	if cfg.Devices()[device1].Name != "tester2" {
		t.Errorf("Device name not overwritten")
	}
}

func TestClusterConfig(t *testing.T) {
	cfg := config.New(device1)
	cfg.Devices = []config.DeviceConfiguration{
		{
			DeviceID:   device1,
			Introducer: true,
		},
		{
			DeviceID: device2,
		},
	}
	cfg.Folders = []config.FolderConfiguration{
		{
			ID:   "folder1",
			Path: "testdata",
			Devices: []config.FolderDeviceConfiguration{
				{DeviceID: device1},
				{DeviceID: device2},
			},
		},
		{
			ID:   "folder2",
			Path: "testdata",
			Devices: []config.FolderDeviceConfiguration{
				{DeviceID: device1},
				{DeviceID: device2},
			},
		},
	}

	db := db.OpenMemory()

	wrapper, path := createTmpWrapper(cfg)
	defer os.Remove(path)
	m := NewModel(wrapper, protocol.LocalDeviceID, "syncthing", "dev", db, nil)
	m.AddFolder(cfg.Folders[0])
	m.AddFolder(cfg.Folders[1])
	m.ServeBackground()
	defer m.Stop()

	cm := m.generateClusterConfig(device2)

	if l := len(cm.Folders); l != 2 {
		t.Fatalf("Incorrect number of folders %d != 2", l)
	}

	r := cm.Folders[0]
	if r.ID != "folder1" {
		t.Errorf("Incorrect folder %q != folder1", r.ID)
	}
	if l := len(r.Devices); l != 2 {
		t.Errorf("Incorrect number of devices %d != 2", l)
	}
	if id := r.Devices[0].ID; id != device1 {
		t.Errorf("Incorrect device ID %s != %s", id, device1)
	}
	if !r.Devices[0].Introducer {
		t.Error("Device1 should be flagged as Introducer")
	}
	if id := r.Devices[1].ID; id != device2 {
		t.Errorf("Incorrect device ID %s != %s", id, device2)
	}
	if r.Devices[1].Introducer {
		t.Error("Device2 should not be flagged as Introducer")
	}

	r = cm.Folders[1]
	if r.ID != "folder2" {
		t.Errorf("Incorrect folder %q != folder2", r.ID)
	}
	if l := len(r.Devices); l != 2 {
		t.Errorf("Incorrect number of devices %d != 2", l)
	}
	if id := r.Devices[0].ID; id != device1 {
		t.Errorf("Incorrect device ID %s != %s", id, device1)
	}
	if !r.Devices[0].Introducer {
		t.Error("Device1 should be flagged as Introducer")
	}
	if id := r.Devices[1].ID; id != device2 {
		t.Errorf("Incorrect device ID %s != %s", id, device2)
	}
	if r.Devices[1].Introducer {
		t.Error("Device2 should not be flagged as Introducer")
	}
}

func TestIntroducer(t *testing.T) {
	var introducedByAnyone protocol.DeviceID

	// LocalDeviceID is a magic value meaning don't check introducer
	contains := func(cfg config.FolderConfiguration, id, introducedBy protocol.DeviceID) bool {
		for _, dev := range cfg.Devices {
			if dev.DeviceID.Equals(id) {
				if introducedBy.Equals(introducedByAnyone) {
					return true
				}
				return introducedBy.Equals(introducedBy)
			}
		}
		return false
	}

	wcfg, m := newState(config.Configuration{
		Devices: []config.DeviceConfiguration{
			{
				DeviceID:   device1,
				Introducer: true,
			},
		},<commit_msg>lib/model: Fix test function for introducer (#4898)<commit_after>// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package model

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/d4l3k/messagediff"
	"github.com/syncthing/syncthing/lib/config"
	"github.com/syncthing/syncthing/lib/db"
	"github.com/syncthing/syncthing/lib/fs"
	"github.com/syncthing/syncthing/lib/ignore"
	"github.com/syncthing/syncthing/lib/osutil"
	"github.com/syncthing/syncthing/lib/protocol"
	srand "github.com/syncthing/syncthing/lib/rand"
	"github.com/syncthing/syncthing/lib/scanner"
	"github.com/syncthing/syncthing/lib/versioner"
)

var device1, device2 protocol.DeviceID
var defaultCfgWrapper *config.Wrapper
var defaultFolderConfig config.FolderConfiguration
var defaultFs fs.Filesystem
var defaultCfg config.Configuration
var defaultAutoAcceptCfg config.Configuration
var tmpLocation string

func init() {
	device1, _ = protocol.DeviceIDFromString("AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR")
	device2, _ = protocol.DeviceIDFromString("GYRZZQB-IRNPV4Z-T7TC52W-EQYJ3TT-FDQW6MW-DFLMU42-SSSU6EM-FBK2VAY")
	defaultFs = fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata")

	defaultFolderConfig = config.NewFolderConfiguration(protocol.LocalDeviceID, "default", "default", fs.FilesystemTypeBasic, "testdata")
	defaultFolderConfig.Devices = []config.FolderDeviceConfiguration{{DeviceID: device1}}
	defaultCfg = config.Configuration{
		Version: config.CurrentVersion,
		Folders: []config.FolderConfiguration{defaultFolderConfig},
		Devices: []config.DeviceConfiguration{config.NewDeviceConfiguration(device1, "device1")},
		Options: config.OptionsConfiguration{
			// Don't remove temporaries directly on startup
			KeepTemporariesH: 1,
		},
	}
	defaultAutoAcceptCfg = config.Configuration{
		Devices: []config.DeviceConfiguration{
			{
				DeviceID: protocol.LocalDeviceID, // self
			},
			{
				DeviceID:          device1,
				AutoAcceptFolders: true,
			},
		},
		Options: config.OptionsConfiguration{
			DefaultFolderPath: "testdata",
		},
	}
}

var testDataExpected = map[string]protocol.FileInfo{
	"foo": {
		Name:      "foo",
		Type:      protocol.FileInfoTypeFile,
		ModifiedS: 0,
		Blocks:    []protocol.BlockInfo{{Offset: 0x0, Size: 0x7, Hash: []uint8{0xae, 0xc0, 0x70, 0x64, 0x5f, 0xe5, 0x3e, 0xe3, 0xb3, 0x76, 0x30, 0x59, 0x37, 0x61, 0x34, 0xf0, 0x58, 0xcc, 0x33, 0x72, 0x47, 0xc9, 0x78, 0xad, 0xd1, 0x78, 0xb6, 0xcc, 0xdf, 0xb0, 0x1, 0x9f}}},
	},
	"empty": {
		Name:      "empty",
		Type:      protocol.FileInfoTypeFile,
		ModifiedS: 0,
		Blocks:    []protocol.BlockInfo{{Offset: 0x0, Size: 0x0, Hash: []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}}},
	},
	"bar": {
		Name:      "bar",
		Type:      protocol.FileInfoTypeFile,
		ModifiedS: 0,
		Blocks:    []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}},
	},
}

func init() {
	// Fix expected test data to match reality
	for n, f := range testDataExpected {
		fi, _ := os.Stat("testdata/" + n)
		f.Permissions = uint32(fi.Mode())
		f.ModifiedS = fi.ModTime().Unix()
		f.Size = fi.Size()
		testDataExpected[n] = f
	}
}

func TestMain(m *testing.M) {
	tmpLocation = "/tmp"
	if runtime.GOOS == "windows" {
		tmpLocation = filepath.Join("testdata", "tmp")
		if err := os.MkdirAll(tmpLocation, 0777); err != nil {
			panic(err)
		}
	}

	tmpName := fs.TempName("file")
	if err := osutil.Copy(defaultFs, "tmpfile", tmpName); err != nil {
		panic(err)
	}
	future := time.Now().Add(time.Hour)
	if err := os.Chtimes(filepath.Join("testdata", tmpName), future, future); err != nil {
		panic(err)
	}

	var wrapperPath string
	defaultCfgWrapper, wrapperPath = createTmpWrapper(defaultCfg)

	exitCode := m.Run()

	os.Remove(wrapperPath)
	defaultFs.Remove(tmpName)
	defaultFs.RemoveAll(config.DefaultMarkerName)
	defaultFs.RemoveAll(tmpLocation)

	os.Exit(exitCode)
}

func createTmpWrapper(cfg config.Configuration) (*config.Wrapper, string) {
	tmpFile, err := ioutil.TempFile(tmpLocation, "syncthing-testConfig-")
	if err != nil {
		panic(err)
	}
	wrapper := config.Wrap(tmpFile.Name(), cfg)
	tmpFile.Close()
	return wrapper, tmpFile.Name()
}

func newState(cfg config.Configuration) (*config.Wrapper, *Model) {
	db := db.OpenMemory()

	wcfg, path := createTmpWrapper(cfg)
	defer os.Remove(path)

	m := NewModel(wcfg, protocol.LocalDeviceID, "syncthing", "dev", db, nil)
	for _, folder := range cfg.Folders {
		if !folder.Paused {
			m.AddFolder(folder)
		}
	}
	m.ServeBackground()
	m.AddConnection(&fakeConnection{id: device1}, protocol.HelloResult{})
	return wcfg, m
}

func TestRequest(t *testing.T) {
	db := db.OpenMemory()

	m := NewModel(defaultCfgWrapper, protocol.LocalDeviceID, "syncthing", "dev", db, nil)

	// device1 shares default, but device2 doesn't
	m.AddFolder(defaultFolderConfig)
	m.StartFolder("default")
	m.ServeBackground()
	defer m.Stop()
	m.ScanFolder("default")

	bs := make([]byte, protocol.MinBlockSize)

	// Existing, shared file
	bs = bs[:6]
	err := m.Request(device1, "default", "foo", 0, nil, false, bs)
	if err != nil {
		t.Error(err)
	}
	if !bytes.Equal(bs, []byte("foobar")) {
		t.Errorf("Incorrect data from request: %q", string(bs))
	}

	// Existing, nonshared file
	err = m.Request(device2, "default", "foo", 0, nil, false, bs)
	if err == nil {
		t.Error("Unexpected nil error on insecure file read")
	}

	// Nonexistent file
	err = m.Request(device1, "default", "nonexistent", 0, nil, false, bs)
	if err == nil {
		t.Error("Unexpected nil error on insecure file read")
	}

	// Shared folder, but disallowed file name
	err = m.Request(device1, "default", "../walk.go", 0, nil, false, bs)
	if err == nil {
		t.Error("Unexpected nil error on insecure file read")
	}

	// Negative offset
	err = m.Request(device1, "default", "foo", -4, nil, false, bs[:0])
	if err == nil {
		t.Error("Unexpected nil error on insecure file read")
	}

	// Larger block than available
	bs = bs[:42]
	err = m.Request(device1, "default", "foo", 0, nil, false, bs)
	if err == nil {
		t.Error("Unexpected nil error on insecure file read")
	}
}

func genFiles(n int) []protocol.FileInfo {
	files := make([]protocol.FileInfo, n)
	t := time.Now().Unix()
	for i := 0; i < n; i++ {
		files[i] = protocol.FileInfo{
			Name:      fmt.Sprintf("file%d", i),
			ModifiedS: t,
			Sequence:  int64(i + 1),
			Blocks:    []protocol.BlockInfo{{Offset: 0, Size: 100, Hash: []byte("some hash bytes")}},
			Version:   protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1}}},
		}
	}

	return files
}

func BenchmarkIndex_10000(b *testing.B) {
	benchmarkIndex(b, 10000)
}

func BenchmarkIndex_100(b *testing.B) {
	benchmarkIndex(b, 100)
}

func benchmarkIndex(b *testing.B, nfiles int) {
	db := db.OpenMemory()
	m := NewModel(defaultCfgWrapper, protocol.LocalDeviceID, "syncthing", "dev", db, nil)
	m.AddFolder(defaultFolderConfig)
	m.StartFolder("default")
	m.ServeBackground()
	defer m.Stop()

	files := genFiles(nfiles)
	m.Index(device1, "default", files)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		m.Index(device1, "default", files)
	}
	b.ReportAllocs()
}

func BenchmarkIndexUpdate_10000_10000(b *testing.B) {
	benchmarkIndexUpdate(b, 10000, 10000)
}

func BenchmarkIndexUpdate_10000_100(b *testing.B) {
	benchmarkIndexUpdate(b, 10000, 100)
}

func BenchmarkIndexUpdate_10000_1(b *testing.B) {
	benchmarkIndexUpdate(b, 10000, 1)
}

func benchmarkIndexUpdate(b *testing.B, nfiles, nufiles int) {
	db := db.OpenMemory()
	m := NewModel(defaultCfgWrapper, protocol.LocalDeviceID, "syncthing", "dev", db, nil)
	m.AddFolder(defaultFolderConfig)
	m.StartFolder("default")
	m.ServeBackground()
	defer m.Stop()

	files := genFiles(nfiles)
	ufiles := genFiles(nufiles)

	m.Index(device1, "default", files)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		m.IndexUpdate(device1, "default", ufiles)
	}
	b.ReportAllocs()
}

type downloadProgressMessage struct {
	folder  string
	updates []protocol.FileDownloadProgressUpdate
}

type fakeConnection struct {
	id                       protocol.DeviceID
	downloadProgressMessages []downloadProgressMessage
	closed                   bool
	files                    []protocol.FileInfo
	fileData                 map[string][]byte
	folder                   string
	model                    *Model
	indexFn                  func(string, []protocol.FileInfo)
	requestFn                func(folder, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error)
	mut                      sync.Mutex
}

func (f *fakeConnection) Close() error {
	f.mut.Lock()
	defer f.mut.Unlock()
	f.closed = true
	return nil
}

func (f *fakeConnection) Start() {
}

func (f *fakeConnection) ID() protocol.DeviceID {
	return f.id
}

func (f *fakeConnection) Name() string {
	return ""
}

func (f *fakeConnection) String() string {
	return ""
}

func (f *fakeConnection) Option(string) string {
	return ""
}

func (f *fakeConnection) Index(folder string, fs []protocol.FileInfo) error {
	f.mut.Lock()
	defer f.mut.Unlock()
	if f.indexFn != nil {
		f.indexFn(folder, fs)
	}
	return nil
}

func (f *fakeConnection) IndexUpdate(folder string, fs []protocol.FileInfo) error {
	f.mut.Lock()
	defer f.mut.Unlock()
	if f.indexFn != nil {
		f.indexFn(folder, fs)
	}
	return nil
}

func (f *fakeConnection) Request(folder, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) {
	f.mut.Lock()
	defer f.mut.Unlock()
	if f.requestFn != nil {
		return f.requestFn(folder, name, offset, size, hash, fromTemporary)
	}
	return f.fileData[name], nil
}

func (f *fakeConnection) ClusterConfig(protocol.ClusterConfig) {}

func (f *fakeConnection) Ping() bool {
	f.mut.Lock()
	defer f.mut.Unlock()
	return f.closed
}

func (f *fakeConnection) Closed() bool {
	f.mut.Lock()
	defer f.mut.Unlock()
	return f.closed
}

func (f *fakeConnection) Statistics() protocol.Statistics {
	return protocol.Statistics{}
}

func (f *fakeConnection) RemoteAddr() net.Addr {
	return &fakeAddr{}
}

func (f *fakeConnection) Type() string {
	return "fake"
}
func (f *fakeConnection) Transport() string {
	return "fake"
}
func (f *fakeConnection) Priority() int {
	return 9000
}

func (f *fakeConnection) DownloadProgress(folder string, updates []protocol.FileDownloadProgressUpdate) {
	f.downloadProgressMessages = append(f.downloadProgressMessages, downloadProgressMessage{
		folder:  folder,
		updates: updates,
	})
}

func (f *fakeConnection) addFile(name string, flags uint32, ftype protocol.FileInfoType, data []byte) {
	f.mut.Lock()
	defer f.mut.Unlock()

	blockSize := protocol.BlockSize(int64(len(data)))
	blocks, _ := scanner.Blocks(context.TODO(), bytes.NewReader(data), blockSize, int64(len(data)), nil, true)
	var version protocol.Vector
	version = version.Update(f.id.Short())

	if ftype == protocol.FileInfoTypeFile || ftype == protocol.FileInfoTypeDirectory {
		f.files = append(f.files, protocol.FileInfo{
			Name:         name,
			Type:         ftype,
			Size:         int64(len(data)),
			ModifiedS:    time.Now().Unix(),
			Permissions:  flags,
			Version:      version,
			Sequence:     time.Now().UnixNano(),
			RawBlockSize: int32(blockSize),
			Blocks:       blocks,
		})
	} else {
		// Symlink
		f.files = append(f.files, protocol.FileInfo{
			Name:          name,
			Type:          ftype,
			Version:       version,
			Sequence:      time.Now().UnixNano(),
			SymlinkTarget: string(data),
		})
	}

	if f.fileData == nil {
		f.fileData = make(map[string][]byte)
	}
	f.fileData[name] = data
}

func (f *fakeConnection) deleteFile(name string) {
	f.mut.Lock()
	defer f.mut.Unlock()

	for i, fi := range f.files {
		if fi.Name == name {
			fi.Deleted = true
			fi.ModifiedS = time.Now().Unix()
			fi.Version = fi.Version.Update(f.id.Short())
			fi.Sequence = time.Now().UnixNano()
			fi.Blocks = nil

			f.files = append(append(f.files[:i], f.files[i+1:]...), fi)
			return
		}
	}
}

func (f *fakeConnection) sendIndexUpdate() {
	f.model.IndexUpdate(f.id, f.folder, f.files)
}

func BenchmarkRequestOut(b *testing.B) {
	db := db.OpenMemory()
	m := NewModel(defaultCfgWrapper, protocol.LocalDeviceID, "syncthing", "dev", db, nil)
	m.AddFolder(defaultFolderConfig)
	m.ServeBackground()
	defer m.Stop()
	m.ScanFolder("default")

	const n = 1000
	files := genFiles(n)

	fc := &fakeConnection{id: device1}
	for _, f := range files {
		fc.addFile(f.Name, 0644, protocol.FileInfoTypeFile, []byte("some data to return"))
	}
	m.AddConnection(fc, protocol.HelloResult{})
	m.Index(device1, "default", files)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		data, err := m.requestGlobal(device1, "default", files[i%n].Name, 0, 32, nil, false)
		if err != nil {
			b.Error(err)
		}
		if data == nil {
			b.Error("nil data")
		}
	}
}

func BenchmarkRequestInSingleFile(b *testing.B) {
	db := db.OpenMemory()
	m := NewModel(defaultCfgWrapper, protocol.LocalDeviceID, "syncthing", "dev", db, nil)
	m.AddFolder(defaultFolderConfig)
	m.ServeBackground()
	defer m.Stop()
	m.ScanFolder("default")

	buf := make([]byte, 128<<10)
	rand.Read(buf)
	os.RemoveAll("testdata/request")
	defer os.RemoveAll("testdata/request")
	os.MkdirAll("testdata/request/for/a/file/in/a/couple/of/dirs", 0755)
	ioutil.WriteFile("testdata/request/for/a/file/in/a/couple/of/dirs/128k", buf, 0644)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		if err := m.Request(device1, "default", "request/for/a/file/in/a/couple/of/dirs/128k", 0, nil, false, buf); err != nil {
			b.Error(err)
		}
	}

	b.SetBytes(128 << 10)
}

func TestDeviceRename(t *testing.T) {
	hello := protocol.HelloResult{
		ClientName:    "syncthing",
		ClientVersion: "v0.9.4",
	}
	defer os.Remove("testdata/tmpconfig.xml")

	rawCfg := config.New(device1)
	rawCfg.Devices = []config.DeviceConfiguration{
		{
			DeviceID: device1,
		},
	}
	cfg := config.Wrap("testdata/tmpconfig.xml", rawCfg)

	db := db.OpenMemory()
	m := NewModel(cfg, protocol.LocalDeviceID, "syncthing", "dev", db, nil)

	if cfg.Devices()[device1].Name != "" {
		t.Errorf("Device already has a name")
	}

	conn := &fakeConnection{id: device1}

	m.AddConnection(conn, hello)

	m.ServeBackground()
	defer m.Stop()

	if cfg.Devices()[device1].Name != "" {
		t.Errorf("Device already has a name")
	}

	m.Closed(conn, protocol.ErrTimeout)
	hello.DeviceName = "tester"
	m.AddConnection(conn, hello)

	if cfg.Devices()[device1].Name != "tester" {
		t.Errorf("Device did not get a name")
	}

	m.Closed(conn, protocol.ErrTimeout)
	hello.DeviceName = "tester2"
	m.AddConnection(conn, hello)

	if cfg.Devices()[device1].Name != "tester" {
		t.Errorf("Device name got overwritten")
	}

	cfgw, err := config.Load("testdata/tmpconfig.xml", protocol.LocalDeviceID)
	if err != nil {
		t.Error(err)
		return
	}
	if cfgw.Devices()[device1].Name != "tester" {
		t.Errorf("Device name not saved in config")
	}

	m.Closed(conn, protocol.ErrTimeout)

	opts := cfg.Options()
	opts.OverwriteRemoteDevNames = true
	cfg.SetOptions(opts)

	hello.DeviceName = "tester2"
	m.AddConnection(conn, hello)

	if cfg.Devices()[device1].Name != "tester2" {
		t.Errorf("Device name not overwritten")
	}
}

func TestClusterConfig(t *testing.T) {
	cfg := config.New(device1)
	cfg.Devices = []config.DeviceConfiguration{
		{
			DeviceID:   device1,
			Introducer: true,
		},
		{
			DeviceID: device2,
		},
	}
	cfg.Folders = []config.FolderConfiguration{
		{
			ID:   "folder1",
			Path: "testdata",
			Devices: []config.FolderDeviceConfiguration{
				{DeviceID: device1},
				{DeviceID: device2},
			},
		},
		{
			ID:   "folder2",
			Path: "testdata",
			Devices: []config.FolderDeviceConfiguration{
				{DeviceID: device1},
				{DeviceID: device2},
			},
		},
	}

	db := db.OpenMemory()

	wrapper, path := createTmpWrapper(cfg)
	defer os.Remove(path)
	m := NewModel(wrapper, protocol.LocalDeviceID, "syncthing", "dev", db, nil)
	m.AddFolder(cfg.Folders[0])
	m.AddFolder(cfg.Folders[1])
	m.ServeBackground()
	defer m.Stop()

	cm := m.generateClusterConfig(device2)

	if l := len(cm.Folders); l != 2 {
		t.Fatalf("Incorrect number of folders %d != 2", l)
	}

	r := cm.Folders[0]
	if r.ID != "folder1" {
		t.Errorf("Incorrect folder %q != folder1", r.ID)
	}
	if l := len(r.Devices); l != 2 {
		t.Errorf("Incorrect number of devices %d != 2", l)
	}
	if id := r.Devices[0].ID; id != device1 {
		t.Errorf("Incorrect device ID %s != %s", id, device1)
	}
	if !r.Devices[0].Introducer {
		t.Error("Device1 should be flagged as Introducer")
	}
	if id := r.Devices[1].ID; id != device2 {
		t.Errorf("Incorrect device ID %s != %s", id, device2)
	}
	if r.Devices[1].Introducer {
		t.Error("Device2 should not be flagged as Introducer")
	}

	r = cm.Folders[1]
	if r.ID != "folder2" {
		t.Errorf("Incorrect folder %q != folder2", r.ID)
	}
	if l := len(r.Devices); l != 2 {
		t.Errorf("Incorrect number of devices %d != 2", l)
	}
	if id := r.Devices[0].ID; id != device1 {
		t.Errorf("Incorrect device ID %s != %s", id, device1)
	}
	if !r.Devices[0].Introducer {
		t.Error("Device1 should be flagged as Introducer")
	}
	if id := r.Devices[1].ID; id != device2 {
		t.Errorf("Incorrect device ID %s != %s", id, device2)
	}
	if r.Devices[1].Introducer {
		t.Error("Device2 should not be flagged as Introducer")
	}
}

func TestIntroducer(t *testing.T) {
	var introducedByAnyone protocol.DeviceID

	// LocalDeviceID is a magic value meaning don't check introducer
	contains := func(cfg config.FolderConfiguration, id, introducedBy protocol.DeviceID) bool {
		for _, dev := range cfg.Devices {
			if dev.DeviceID.Equals(id) {
				if introducedBy.Equals(introducedByAnyone) {
					return true
				}
				return dev.IntroducedBy.Equals(introducedBy)
			}
		}
		return false
	}

	wcfg, m := newState(config.Configuration{
		Devices: []config.DeviceConfiguration{
			{
				DeviceID:   device1,
				Introducer: true,
			},
		},
{'fraction_non_alphanumeric': 0.15389234632601811, 'fraction_numerical': 0.027404738508570805, 'mean_word_length': 3.9704246370913037, 'pattern_counts': {'":': 6, '<': 21, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6238666', 'n_tokens_mistral': 14967, 'n_tokens_neox': 13642, 'n_words': 3058}