<commit_before>package main

import (
	"crypto/sha1"
	"crypto/tls"
	"fmt"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/calmh/ini"
	"github.com/calmh/syncthing/discover"
	flags "github.com/calmh/syncthing/github.com/jessevdk/go-flags"
	"github.com/calmh/syncthing/protocol"
)

type Options struct {
	ConfDir    string           `short:"c" long:"cfg" description:"Configuration directory" default:"~/.syncthing" value-name:"DIR"`
	Listen     string           `short:"l" long:"listen" description:"Listen address" default:":22000" value-name:"ADDR"`
	ReadOnly   bool             `short:"r" long:"ro" description:"Repository is read only"`
	Delete     bool             `short:"d" long:"delete" description:"Delete files deleted from cluster"`
	NoSymlinks bool             `long:"no-symlinks" description:"Don't follow first level symlinks in the repo"`
	Discovery  DiscoveryOptions `group:"Discovery Options"`
	Advanced   AdvancedOptions  `group:"Advanced Options"`
	Debug      DebugOptions     `group:"Debugging Options"`
}

type DebugOptions struct {
	LogSource bool   `long:"log-source"`
	TraceFile bool   `long:"trace-file"`
	TraceNet  bool   `long:"trace-net"`
	TraceIdx  bool   `long:"trace-idx"`
	Profiler  string `long:"profiler" value-name:"ADDR"`
}

type DiscoveryOptions struct {
	ExternalServer      string `long:"ext-server" description:"External discovery server" value-name:"NAME" default:"syncthing.nym.se"`
	ExternalPort        int    `short:"e" long:"ext-port" description:"External listen port" value-name:"PORT" default:"22000"`
	NoExternalDiscovery bool   `short:"n" long:"no-ext-announce" description:"Do not announce presence externally"`
	NoLocalDiscovery    bool   `short:"N" long:"no-local-announce" description:"Do not announce presence locally"`
}

type AdvancedOptions struct {
	RequestsInFlight int           `long:"reqs-in-flight" description:"Parallell in flight requests per file" default:"8" value-name:"REQS"`
	FilesInFlight    int           `long:"files-in-flight" description:"Parallell in flight file pulls" default:"4" value-name:"FILES"`
	ScanInterval     time.Duration `long:"scan-intv" description:"Repository scan interval" default:"60s" value-name:"INTV"`
	ConnInterval     time.Duration `long:"conn-intv" description:"Node reconnect interval" default:"60s" value-name:"INTV"`
}

var opts Options
var Version string = "unknown-dev"

const (
	confDirName  = ".syncthing"
	confFileName = "syncthing.ini"
)

var (
	config    ini.Config
	nodeAddrs = make(map[string][]string)
)
<commit_msg>Change default queue parameters to optimize better for small files<commit_after>package main

import (
	"crypto/sha1"
	"crypto/tls"
	"fmt"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/calmh/ini"
	"github.com/calmh/syncthing/discover"
	flags "github.com/calmh/syncthing/github.com/jessevdk/go-flags"
	"github.com/calmh/syncthing/protocol"
)

type Options struct {
	ConfDir    string           `short:"c" long:"cfg" description:"Configuration directory" default:"~/.syncthing" value-name:"DIR"`
	Listen     string           `short:"l" long:"listen" description:"Listen address" default:":22000" value-name:"ADDR"`
	ReadOnly   bool             `short:"r" long:"ro" description:"Repository is read only"`
	Delete     bool             `short:"d" long:"delete" description:"Delete files deleted from cluster"`
	NoSymlinks bool             `long:"no-symlinks" description:"Don't follow first level symlinks in the repo"`
	Discovery  DiscoveryOptions `group:"Discovery Options"`
	Advanced   AdvancedOptions  `group:"Advanced Options"`
	Debug      DebugOptions     `group:"Debugging Options"`
}

type DebugOptions struct {
	LogSource bool   `long:"log-source"`
	TraceFile bool   `long:"trace-file"`
	TraceNet  bool   `long:"trace-net"`
	TraceIdx  bool   `long:"trace-idx"`
	Profiler  string `long:"profiler" value-name:"ADDR"`
}

type DiscoveryOptions struct {
	ExternalServer      string `long:"ext-server" description:"External discovery server" value-name:"NAME" default:"syncthing.nym.se"`
	ExternalPort        int    `short:"e" long:"ext-port" description:"External listen port" value-name:"PORT" default:"22000"`
	NoExternalDiscovery bool   `short:"n" long:"no-ext-announce" description:"Do not announce presence externally"`
	NoLocalDiscovery    bool   `short:"N" long:"no-local-announce" description:"Do not announce presence locally"`
}

type AdvancedOptions struct {
	RequestsInFlight int           `long:"reqs-in-flight" description:"Parallell in flight requests per file" default:"4" value-name:"REQS"`
	FilesInFlight    int           `long:"files-in-flight" description:"Parallell in flight file pulls" default:"8" value-name:"FILES"`
	ScanInterval     time.Duration `long:"scan-intv" description:"Repository scan interval" default:"60s" value-name:"INTV"`
	ConnInterval     time.Duration `long:"conn-intv" description:"Node reconnect interval" default:"60s" value-name:"INTV"`
}

var opts Options
var Version string = "unknown-dev"

const (
	confDirName  = ".syncthing"
	confFileName = "syncthing.ini"
)

var (
	config    ini.Config
	nodeAddrs = make(map[string][]string)
)

{'fraction_non_alphanumeric': 0.13172506222477504, 'fraction_numerical': 0.006509668772735976, 'mean_word_length': 4.475890985324948, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '427328', 'n_tokens_mistral': 1665, 'n_tokens_neox': 1631, 'n_words': 395}