<commit_before>package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"runtime/debug"
	"strings"
	"time"

	"github.com/calmh/ini"
	"github.com/calmh/syncthing/discover"
	"github.com/calmh/syncthing/protocol"
)

const BlockSize = 128 * 1024

var cfg Configuration
var Version = "unknown-dev"

var (
	myID string
)

var (
	showVersion bool
	confDir     string
	verbose     bool
)

const (
	usage      = "syncthing [options]"
	extraUsage = `The following enviroment variables are interpreted by syncthing:

 STNORESTART  Do not attempt to restart when requested to, instead just exit.
              Set this variable when running under a service manager such as
              runit, launchd, etc.

 STPROFILER   Set to a listen address such as "127.0.0.1:9090" to start the
              profiler with HTTP access.

 STTRACE      A comma separated string of facilities to trace. The valid
              facility strings:
              - "discover" (the node discovery package)
              - "files"    (file set store)
              - "idx"      (index sending and receiving)
              - "mc"       (multicast beacon)
              - "need"     (file need calculations)
              - "net"      (connecting and disconnecting, network messages)
              - "pull"     (file pull activity)
              - "scanner"  (the file change scanner)
              `
)

func main() {
	flag.StringVar(&confDir, "home", getDefaultConfDir(), "Set configuration directory")
	flag.BoolVar(&showVersion, "version", false, "Show version")
	flag.BoolVar(&verbose, "v", false, "Be more verbose")
	flag.Usage = usageFor(flag.CommandLine, usage, extraUsage)
	flag.Parse()

	if len(os.Getenv("STRESTART")) > 0 {
		// Give the parent process time to exit and release sockets etc.
		time.Sleep(1 * time.Second)
	}

	if showVersion {
		fmt.Println(Version)
		os.Exit(0)
	}

	if len(os.Getenv("GOGC")) == 0 {
		debug.SetGCPercent(25)
	}

	if len(os.Getenv("GOMAXPROCS")) == 0 {
		runtime.GOMAXPROCS(runtime.NumCPU())
	}

	confDir = expandTilde(confDir)

	// Ensure that our home directory exists and that we have a certificate and key.

	ensureDir(confDir, 0700)
	cert, err := loadCert(confDir)
	if err != nil {
		newCertificate(confDir)
		cert, err = loadCert(confDir)
		fatalErr(err)
	}

	myID = string(certID(cert.Certificate[0]))
	log.SetPrefix("[" + myID[0:5] + "] ")
	logger.SetPrefix("[" + myID[0:5] + "] ")

	infoln("Version", Version)
	infoln("My ID:", myID)

	// Prepare to be able to save configuration

	cfgFile := filepath.Join(confDir, "config.xml")
	go saveConfigLoop(cfgFile)

	// Load the configuration file, if it exists.
	// If it does not, create a template.

	cf, err := os.Open(cfgFile)
	if err == nil {
		// Read config.xml
		cfg, err = readConfigXML(cf)
		if err != nil {
			fatalln(err)
		}
		cf.Close()
	} else {
		// No config.xml, let's try the old syncthing.ini
		iniFile := filepath.Join(confDir, "syncthing.ini")
		cf, err := os.Open(iniFile)
		if err == nil {
			infoln("Migrating syncthing.ini to config.xml")
			iniCfg := ini.Parse(cf)
			cf.Close()
			Rename(iniFile, filepath.Join(confDir, "migrated_syncthing.ini"))

			cfg, _ = readConfigXML(nil)
			cfg.Repositories = []RepositoryConfiguration{
				{Directory: iniCfg.Get("repository", "dir")},
			}
			readConfigINI(iniCfg.OptionMap("settings"), &cfg.Options)
			for name, addrs := range iniCfg.OptionMap("nodes") {
				n := NodeConfiguration{
					NodeID:    name,
					Addresses: strings.Fields(addrs),
				}
				cfg.Repositories[0].Nodes = append(cfg.Repositories[0].Nodes, n)
			}

			saveConfig()
		}
	}

	if len(cfg.Repositories) == 0 {
		infoln("No config file; starting with empty defaults")

		cfg, err = readConfigXML(nil)
		cfg.Repositories = []RepositoryConfiguration{
			{
				ID:        "default",
				Directory: filepath.Join(getHomeDir(), "Sync"),
				Nodes: []NodeConfiguration{
					{NodeID: myID, Addresses: []string{"dynamic"}},
				},
			},
		}

		saveConfig()
		infof("Edit %s to taste or use the GUI\n", cfgFile)
	}

	if profiler := os.Getenv("STPROFILER"); len(profiler) > 0 {
		go func() {
			dlog.Println("Starting profiler on", profiler)
			err := http.ListenAndServe(profiler, nil)
			if err != nil {
				dlog.Fatal(err)
			}
		}()
	}

	// The TLS configuration is used for both the listening socket and outgoing
	// connections.

	tlsCfg := &tls.Config{
		Certificates:           []tls.Certificate{cert},
		NextProtos:             []string{"bep/1.0"},
		ServerName:             myID,
		ClientAuth:             tls.RequestClientCert,
		SessionTicketsDisabled: true,
		InsecureSkipVerify:     true,
		MinVersion:             tls.VersionTLS12,
	}

	m := NewModel(cfg.Options.MaxChangeKbps * 1000)
	if cfg.Options.MaxSendKbps > 0 {
		m.LimitRate(cfg.Options.MaxSendKbps)
	}

	for i := range cfg.Repositories {
		cfg.Repositories[i].Nodes = cleanNodeList(cfg.Repositories[i].Nodes, myID)
		dir := expandTilde(cfg.Repositories[i].Directory)
		ensureDir(dir, -1)
		m.AddRepo(cfg.Repositories[i].ID, dir, cfg.Repositories[i].Nodes)
	}

	// GUI
	if cfg.Options.GUIEnabled && cfg.Options.GUIAddress != "" {
		addr, err := net.ResolveTCPAddr("tcp", cfg.Options.GUIAddress)
		if err != nil {
			warnf("Cannot start GUI on %q: %v", cfg.Options.GUIAddress, err)
		} else {
			var hostOpen, hostShow string
			switch {
			case addr.IP == nil:
				hostOpen = "localhost"
				hostShow = "0.0.0.0"
			case addr.IP.IsUnspecified():
				hostOpen = "localhost"
				hostShow = addr.IP.String()
			default:
				hostOpen = addr.IP.String()
				hostShow = hostOpen
			}

			infof("Starting web GUI on http://%s:%d/", hostShow, addr.Port)
			startGUI(cfg.Options.GUIAddress, m)
			if cfg.Options.StartBrowser && len(os.Getenv("STRESTART")) == 0 {
				openURL(fmt.Sprintf("http://%s:%d", hostOpen, addr.Port))
			}
		}
	}

	// Walk the repository and update the local model before establishing any
	// connections to other nodes.

	if verbose {
		infoln("Populating repository index")
	}
	m.LoadIndexes(confDir)
	m.ScanRepos()
	m.SaveIndexes(confDir)

	connOpts := map[string]string{
		"clientId":      "syncthing",
		"clientVersion": Version,
		"clusterHash":   clusterHash(cfg.Repositories[0].Nodes),
	}

	// Routine to connect out to configured nodes
	if verbose {
		infoln("Attempting to connect to other nodes")
	}
	disc := discovery()
	go listenConnect(myID, disc, m, tlsCfg, connOpts)

	// Routine to pull blocks from other nodes to synchronize the local
	// repository. Does not run when we are in read only (publish only) mode.
	if cfg.Options.ReadOnly {
		if verbose {
			okln("Ready to synchronize (read only; no external updates accepted)")
		}
		m.StartRO()
	} else {
		if verbose {
			okln("Ready to synchronize (read-write)")
		}
		m.StartRW(cfg.Options.ParallelRequests)
	}

	select {}
}

func restart() {
	infoln("Restarting")
	if os.Getenv("SMF_FMRI") != "" || os.Getenv("STNORESTART") != "" {
		// Solaris SMF
		infoln("Service manager detected; exit instead of restart")
		os.Exit(0)
	}

	env := os.Environ()
	if len(os.Getenv("STRESTART")) == 0 {
		env = append(env, "STRESTART=1")
	}
	pgm, err := exec.LookPath(os.Args[0])
	if err != nil {
		warnln(err)
		return
	}
	proc, err := os.StartProcess(pgm, os.Args, &os.ProcAttr{
		Env:   env,
		Files: []*os.File{os.Stdin, os.Stdout, os.Stderr},
	})
	if err != nil {
		fatalln(err)
	}
	proc.Release()
	os.Exit(0)
}

var saveConfigCh = make(chan struct{})

func saveConfigLoop(cfgFile string) {
	for _ = range saveConfigCh {
		fd, err := os.Create(cfgFile + ".tmp")
		if err != nil {
			warnln(err)
			continue
		}

		err = writeConfigXML(fd, cfg)
		if err != nil {
			warnln(err)
			fd.Close()
			continue
		}

		err = fd.Close()
		if err != nil {
			warnln(err)
			continue
		}

		err = Rename(cfgFile+".tmp", cfgFile)
		if err != nil {
			warnln(err)
		}
	}
}

func saveConfig() {
	saveConfigCh <- struct{}{}
}

func listenConnect(myID string, disc *discover.Discoverer, m *Model, tlsCfg *tls.Config, connOpts map[string]string) {
	var conns = make(chan *tls.Conn)

	// Listen
	for _, addr := range cfg.Options.ListenAddress {
		addr := addr
		go func() {
			if debugNet {
				dlog.Println("listening on", addr)
			}
			l, err := tls.Listen("tcp", addr, tlsCfg)
			fatalErr(err)

			for {
				conn, err := l.Accept()
				if err != nil {
					warnln(err)
					continue
				}

				if debugNet {
					dlog.Println("connect from", conn.RemoteAddr())
				}

				tc := conn.(*tls.Conn)
				err = tc.Handshake()
				if err != nil {
					warnln(err)
					tc.Close()
					continue
				}

				conns <- tc
			}
		}()
	}

	// Connect
	go func() {
		for {
		nextNode:
			for _, nodeCfg := range cfg.Repositories[0].Nodes {
				if nodeCfg.NodeID == myID {
					continue
				}
				if m.ConnectedTo(nodeCfg.NodeID) {
					continue
				}
				for _, addr := range nodeCfg.Addresses {
					if addr == "dynamic" {
						if disc != nil {
							t := disc.Lookup(nodeCfg.NodeID)
							if len(t) == 0 {
								continue
							}
							addr = t[0] //XXX: Handle all of them
						}
					}

					if debugNet {
						dlog.Println("dial", nodeCfg.NodeID, addr)
					}
					conn, err := tls.Dial("tcp", addr, tlsCfg)
					if err != nil {
						if debugNet {
							dlog.Println(err)
						}
						continue
					}

					conns <- conn
					continue nextNode
				}
			}

			time.Sleep(time.Duration(cfg.Options.ReconnectIntervalS) * time.Second)
		}
	}()

next:
	for conn := range conns {
		remoteID := certID(conn.ConnectionState().PeerCertificates[0].Raw)

		if remoteID == myID {
			warnf("Connected to myself (%s) - should not happen", remoteID)
			conn.Close()
			continue
		}

		if m.ConnectedTo(remoteID) {
			warnf("Connected to already connected node (%s)", remoteID)
			conn.Close()
			continue
		}

		for _, nodeCfg := range cfg.Repositories[0].Nodes {
			if nodeCfg.NodeID == remoteID {
				protoConn := protocol.NewConnection(remoteID, conn, conn, m, connOpts)
				m.AddConnection(conn, protoConn)
				continue next
			}
		}
		conn.Close()
	}
}

func discovery() *discover.Discoverer {
	if !cfg.Options.LocalAnnEnabled {
		return nil
	}

	infoln("Sending local discovery announcements")

	if !cfg.Options.GlobalAnnEnabled {
		cfg.Options.GlobalAnnServer = ""
	} else if verbose {
		infoln("Sending external discovery announcements")
	}

	disc, err := discover.NewDiscoverer(myID, cfg.Options.ListenAddress, cfg.Options.GlobalAnnServer)<commit_msg>Remove pointless -v option<commit_after>package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"runtime/debug"
	"strings"
	"time"

	"github.com/calmh/ini"
	"github.com/calmh/syncthing/discover"
	"github.com/calmh/syncthing/protocol"
)

const BlockSize = 128 * 1024

var cfg Configuration
var Version = "unknown-dev"

var (
	myID string
)

var (
	showVersion bool
	confDir     string
)

const (
	usage      = "syncthing [options]"
	extraUsage = `The following enviroment variables are interpreted by syncthing:

 STNORESTART  Do not attempt to restart when requested to, instead just exit.
              Set this variable when running under a service manager such as
              runit, launchd, etc.

 STPROFILER   Set to a listen address such as "127.0.0.1:9090" to start the
              profiler with HTTP access.

 STTRACE      A comma separated string of facilities to trace. The valid
              facility strings:
              - "discover" (the node discovery package)
              - "files"    (file set store)
              - "idx"      (index sending and receiving)
              - "mc"       (multicast beacon)
              - "need"     (file need calculations)
              - "net"      (connecting and disconnecting, network messages)
              - "pull"     (file pull activity)
              - "scanner"  (the file change scanner)
              `
)

func main() {
	flag.StringVar(&confDir, "home", getDefaultConfDir(), "Set configuration directory")
	flag.BoolVar(&showVersion, "version", false, "Show version")
	flag.Usage = usageFor(flag.CommandLine, usage, extraUsage)
	flag.Parse()

	if len(os.Getenv("STRESTART")) > 0 {
		// Give the parent process time to exit and release sockets etc.
		time.Sleep(1 * time.Second)
	}

	if showVersion {
		fmt.Println(Version)
		os.Exit(0)
	}

	if len(os.Getenv("GOGC")) == 0 {
		debug.SetGCPercent(25)
	}

	if len(os.Getenv("GOMAXPROCS")) == 0 {
		runtime.GOMAXPROCS(runtime.NumCPU())
	}

	confDir = expandTilde(confDir)

	// Ensure that our home directory exists and that we have a certificate and key.

	ensureDir(confDir, 0700)
	cert, err := loadCert(confDir)
	if err != nil {
		newCertificate(confDir)
		cert, err = loadCert(confDir)
		fatalErr(err)
	}

	myID = string(certID(cert.Certificate[0]))
	log.SetPrefix("[" + myID[0:5] + "] ")
	logger.SetPrefix("[" + myID[0:5] + "] ")

	infoln("Version", Version)
	infoln("My ID:", myID)

	// Prepare to be able to save configuration

	cfgFile := filepath.Join(confDir, "config.xml")
	go saveConfigLoop(cfgFile)

	// Load the configuration file, if it exists.
	// If it does not, create a template.

	cf, err := os.Open(cfgFile)
	if err == nil {
		// Read config.xml
		cfg, err = readConfigXML(cf)
		if err != nil {
			fatalln(err)
		}
		cf.Close()
	} else {
		// No config.xml, let's try the old syncthing.ini
		iniFile := filepath.Join(confDir, "syncthing.ini")
		cf, err := os.Open(iniFile)
		if err == nil {
			infoln("Migrating syncthing.ini to config.xml")
			iniCfg := ini.Parse(cf)
			cf.Close()
			Rename(iniFile, filepath.Join(confDir, "migrated_syncthing.ini"))

			cfg, _ = readConfigXML(nil)
			cfg.Repositories = []RepositoryConfiguration{
				{Directory: iniCfg.Get("repository", "dir")},
			}
			readConfigINI(iniCfg.OptionMap("settings"), &cfg.Options)
			for name, addrs := range iniCfg.OptionMap("nodes") {
				n := NodeConfiguration{
					NodeID:    name,
					Addresses: strings.Fields(addrs),
				}
				cfg.Repositories[0].Nodes = append(cfg.Repositories[0].Nodes, n)
			}

			saveConfig()
		}
	}

	if len(cfg.Repositories) == 0 {
		infoln("No config file; starting with empty defaults")

		cfg, err = readConfigXML(nil)
		cfg.Repositories = []RepositoryConfiguration{
			{
				ID:        "default",
				Directory: filepath.Join(getHomeDir(), "Sync"),
				Nodes: []NodeConfiguration{
					{NodeID: myID, Addresses: []string{"dynamic"}},
				},
			},
		}

		saveConfig()
		infof("Edit %s to taste or use the GUI\n", cfgFile)
	}

	if profiler := os.Getenv("STPROFILER"); len(profiler) > 0 {
		go func() {
			dlog.Println("Starting profiler on", profiler)
			err := http.ListenAndServe(profiler, nil)
			if err != nil {
				dlog.Fatal(err)
			}
		}()
	}

	// The TLS configuration is used for both the listening socket and outgoing
	// connections.

	tlsCfg := &tls.Config{
		Certificates:           []tls.Certificate{cert},
		NextProtos:             []string{"bep/1.0"},
		ServerName:             myID,
		ClientAuth:             tls.RequestClientCert,
		SessionTicketsDisabled: true,
		InsecureSkipVerify:     true,
		MinVersion:             tls.VersionTLS12,
	}

	m := NewModel(cfg.Options.MaxChangeKbps * 1000)
	if cfg.Options.MaxSendKbps > 0 {
		m.LimitRate(cfg.Options.MaxSendKbps)
	}

	for i := range cfg.Repositories {
		cfg.Repositories[i].Nodes = cleanNodeList(cfg.Repositories[i].Nodes, myID)
		dir := expandTilde(cfg.Repositories[i].Directory)
		ensureDir(dir, -1)
		m.AddRepo(cfg.Repositories[i].ID, dir, cfg.Repositories[i].Nodes)
	}

	// GUI
	if cfg.Options.GUIEnabled && cfg.Options.GUIAddress != "" {
		addr, err := net.ResolveTCPAddr("tcp", cfg.Options.GUIAddress)
		if err != nil {
			warnf("Cannot start GUI on %q: %v", cfg.Options.GUIAddress, err)
		} else {
			var hostOpen, hostShow string
			switch {
			case addr.IP == nil:
				hostOpen = "localhost"
				hostShow = "0.0.0.0"
			case addr.IP.IsUnspecified():
				hostOpen = "localhost"
				hostShow = addr.IP.String()
			default:
				hostOpen = addr.IP.String()
				hostShow = hostOpen
			}

			infof("Starting web GUI on http://%s:%d/", hostShow, addr.Port)
			startGUI(cfg.Options.GUIAddress, m)
			if cfg.Options.StartBrowser && len(os.Getenv("STRESTART")) == 0 {
				openURL(fmt.Sprintf("http://%s:%d", hostOpen, addr.Port))
			}
		}
	}

	// Walk the repository and update the local model before establishing any
	// connections to other nodes.

	infoln("Populating repository index")
	m.LoadIndexes(confDir)
	m.ScanRepos()
	m.SaveIndexes(confDir)

	connOpts := map[string]string{
		"clientId":      "syncthing",
		"clientVersion": Version,
		"clusterHash":   clusterHash(cfg.Repositories[0].Nodes),
	}

	// Routine to connect out to configured nodes
	disc := discovery()
	go listenConnect(myID, disc, m, tlsCfg, connOpts)

	// Routine to pull blocks from other nodes to synchronize the local
	// repository. Does not run when we are in read only (publish only) mode.
	if cfg.Options.ReadOnly {
		okln("Ready to synchronize (read only; no external updates accepted)")
		m.StartRO()
	} else {
		okln("Ready to synchronize (read-write)")
		m.StartRW(cfg.Options.ParallelRequests)
	}

	select {}
}

func restart() {
	infoln("Restarting")
	if os.Getenv("SMF_FMRI") != "" || os.Getenv("STNORESTART") != "" {
		// Solaris SMF
		infoln("Service manager detected; exit instead of restart")
		os.Exit(0)
	}

	env := os.Environ()
	if len(os.Getenv("STRESTART")) == 0 {
		env = append(env, "STRESTART=1")
	}
	pgm, err := exec.LookPath(os.Args[0])
	if err != nil {
		warnln(err)
		return
	}
	proc, err := os.StartProcess(pgm, os.Args, &os.ProcAttr{
		Env:   env,
		Files: []*os.File{os.Stdin, os.Stdout, os.Stderr},
	})
	if err != nil {
		fatalln(err)
	}
	proc.Release()
	os.Exit(0)
}

var saveConfigCh = make(chan struct{})

func saveConfigLoop(cfgFile string) {
	for _ = range saveConfigCh {
		fd, err := os.Create(cfgFile + ".tmp")
		if err != nil {
			warnln(err)
			continue
		}

		err = writeConfigXML(fd, cfg)
		if err != nil {
			warnln(err)
			fd.Close()
			continue
		}

		err = fd.Close()
		if err != nil {
			warnln(err)
			continue
		}

		err = Rename(cfgFile+".tmp", cfgFile)
		if err != nil {
			warnln(err)
		}
	}
}

func saveConfig() {
	saveConfigCh <- struct{}{}
}

func listenConnect(myID string, disc *discover.Discoverer, m *Model, tlsCfg *tls.Config, connOpts map[string]string) {
	var conns = make(chan *tls.Conn)

	// Listen
	for _, addr := range cfg.Options.ListenAddress {
		addr := addr
		go func() {
			if debugNet {
				dlog.Println("listening on", addr)
			}
			l, err := tls.Listen("tcp", addr, tlsCfg)
			fatalErr(err)

			for {
				conn, err := l.Accept()
				if err != nil {
					warnln(err)
					continue
				}

				if debugNet {
					dlog.Println("connect from", conn.RemoteAddr())
				}

				tc := conn.(*tls.Conn)
				err = tc.Handshake()
				if err != nil {
					warnln(err)
					tc.Close()
					continue
				}

				conns <- tc
			}
		}()
	}

	// Connect
	go func() {
		for {
		nextNode:
			for _, nodeCfg := range cfg.Repositories[0].Nodes {
				if nodeCfg.NodeID == myID {
					continue
				}
				if m.ConnectedTo(nodeCfg.NodeID) {
					continue
				}
				for _, addr := range nodeCfg.Addresses {
					if addr == "dynamic" {
						if disc != nil {
							t := disc.Lookup(nodeCfg.NodeID)
							if len(t) == 0 {
								continue
							}
							addr = t[0] //XXX: Handle all of them
						}
					}

					if debugNet {
						dlog.Println("dial", nodeCfg.NodeID, addr)
					}
					conn, err := tls.Dial("tcp", addr, tlsCfg)
					if err != nil {
						if debugNet {
							dlog.Println(err)
						}
						continue
					}

					conns <- conn
					continue nextNode
				}
			}

			time.Sleep(time.Duration(cfg.Options.ReconnectIntervalS) * time.Second)
		}
	}()

next:
	for conn := range conns {
		remoteID := certID(conn.ConnectionState().PeerCertificates[0].Raw)

		if remoteID == myID {
			warnf("Connected to myself (%s) - should not happen", remoteID)
			conn.Close()
			continue
		}

		if m.ConnectedTo(remoteID) {
			warnf("Connected to already connected node (%s)", remoteID)
			conn.Close()
			continue
		}

		for _, nodeCfg := range cfg.Repositories[0].Nodes {
			if nodeCfg.NodeID == remoteID {
				protoConn := protocol.NewConnection(remoteID, conn, conn, m, connOpts)
				m.AddConnection(conn, protoConn)
				continue next
			}
		}
		conn.Close()
	}
}

func discovery() *discover.Discoverer {
	if !cfg.Options.LocalAnnEnabled {
		return nil
	}

	infoln("Sending local discovery announcements")

	if !cfg.Options.GlobalAnnEnabled {
		cfg.Options.GlobalAnnServer = ""
	} else {
		infoln("Sending external discovery announcements")
	}

	disc, err := discover.NewDiscoverer(myID, cfg.Options.ListenAddress, cfg.Options.GlobalAnnServer)
{'fraction_non_alphanumeric': 0.13755848164761492, 'fraction_numerical': 0.0060772681232817245, 'mean_word_length': 2.9291263975743793, 'pattern_counts': {'":': 6, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4514150', 'n_tokens_mistral': 8383, 'n_tokens_neox': 7240, 'n_words': 1750}