<commit_before>package hostdb

// scan.go contians the functions which periodically scan the list of all hosts
// to see which hosts are online or offline, and to get any updates to the
// settings of the hosts.

import (
	"crypto/rand"
	"math/big"
	"net"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

const (
	DefaultScanSleep = 2*time.Hour + 18*time.Minute
	MaxScanSleep     = 6 * time.Hour
	MinScanSleep     = 1 * time.Hour

	MaxActiveHosts              = 500
	InactiveHostCheckupQuantity = 250

	maxSettingsLen = 1024

	hostRequestTimeout = 15 * time.Second

	// scanningThreads is the number of threads that will be probing hosts for
	// their settings and checking for reliability.
	scanningThreads = 4
)

var (
	MaxReliability     = types.NewCurrency64(50) // Given the scanning defaults, about 1 week of survival.
	DefaultReliability = types.NewCurrency64(20) // Given the scanning defaults, about 3 days of survival.
	UnreachablePenalty = types.NewCurrency64(1)
)

// decrementReliability reduces the reliability of a node, moving it out of the
// set of active hosts or deleting it entirely if necessary.
func (hdb *HostDB) decrementReliability(addr modules.NetAddress, penalty types.Currency) {
	// Look up the entry and decrement the reliability.
	entry, exists := hdb.allHosts[addr]
	if !exists {
		return
	}
	entry.reliability = entry.reliability.Sub(penalty)

	// If the entry is in the active database, remove it from the active
	// database.
	node, exists := hdb.activeHosts[addr]
	if exists {
		delete(hdb.activeHosts, entry.IPAddress)
		node.removeNode()
		hdb.notifySubscribers()
	}

	// If the reliability has fallen to 0, remove the host from the
	// database entirely.
	if entry.reliability.IsZero() {
		delete(hdb.allHosts, addr)
	}
}

// threadedProbeHost tries to fetch the settings of a host. If successful, the
// host is put in the set of active hosts. If unsuccessful, the host id deleted
// from the set of active hosts.
func (hdb *HostDB) threadedProbeHosts() {
	for hostEntry := range hdb.scanPool {
		// Request settings from the queue'd host entry.
		var settings modules.HostSettings
		err := func() error {
			conn, err := net.DialTimeout("tcp", string(hostEntry.IPAddress), hostRequestTimeout)
			if err != nil {
				return err
			}
			defer conn.Close()
			err = encoding.WriteObject(conn, [8]byte{'S', 'e', 't', 't', 'i', 'n', 'g', 's'})
			if err != nil {
				return err
			}
			return encoding.ReadObject(conn, &settings, maxSettingsLen)
		}()

		// Now that network communication is done, lock the hostdb to modify the
		// host entry.
		id := hdb.mu.Lock()
		defer hdb.mu.Unlock(id)
		if err != nil {
			hdb.decrementReliability(hostEntry.IPAddress, UnreachablePenalty)
			return
		}

		// Update the host settings, reliability, and weight. The old IPAddress
		// must be preserved.
		settings.IPAddress = hostEntry.HostSettings.IPAddress
		hostEntry.HostSettings = settings
		hostEntry.reliability = MaxReliability
		hostEntry.weight = hdb.hostWeight(*hostEntry)

		// If the host is not already in the database and 'MaxActiveHosts' has not
		// been reached, add the host to the database.
		_, exists1 := hdb.activeHosts[hostEntry.IPAddress]
		_, exists2 := hdb.allHosts[hostEntry.IPAddress]
		if !exists1 && exists2 && len(hdb.activeHosts) < MaxActiveHosts {
			hdb.insertNode(hostEntry)
			hdb.notifySubscribers()
		}
	}
}
<commit_msg>fix deadlocks<commit_after>package hostdb

// scan.go contians the functions which periodically scan the list of all hosts
// to see which hosts are online or offline, and to get any updates to the
// settings of the hosts.

import (
	"crypto/rand"
	"math/big"
	"net"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

const (
	DefaultScanSleep = 2*time.Hour + 18*time.Minute
	MaxScanSleep     = 6 * time.Hour
	MinScanSleep     = 1 * time.Hour

	MaxActiveHosts              = 500
	InactiveHostCheckupQuantity = 250

	maxSettingsLen = 1024

	hostRequestTimeout = 15 * time.Second

	// scanningThreads is the number of threads that will be probing hosts for
	// their settings and checking for reliability.
	scanningThreads = 4
)

var (
	MaxReliability     = types.NewCurrency64(50) // Given the scanning defaults, about 1 week of survival.
	DefaultReliability = types.NewCurrency64(20) // Given the scanning defaults, about 3 days of survival.
	UnreachablePenalty = types.NewCurrency64(1)
)

// decrementReliability reduces the reliability of a node, moving it out of the
// set of active hosts or deleting it entirely if necessary.
func (hdb *HostDB) decrementReliability(addr modules.NetAddress, penalty types.Currency) {
	// Look up the entry and decrement the reliability.
	entry, exists := hdb.allHosts[addr]
	if !exists {
		return
	}
	entry.reliability = entry.reliability.Sub(penalty)

	// If the entry is in the active database, remove it from the active
	// database.
	node, exists := hdb.activeHosts[addr]
	if exists {
		delete(hdb.activeHosts, entry.IPAddress)
		node.removeNode()
		hdb.notifySubscribers()
	}

	// If the reliability has fallen to 0, remove the host from the
	// database entirely.
	if entry.reliability.IsZero() {
		delete(hdb.allHosts, addr)
	}
}

// threadedProbeHost tries to fetch the settings of a host. If successful, the
// host is put in the set of active hosts. If unsuccessful, the host id deleted
// from the set of active hosts.
func (hdb *HostDB) threadedProbeHosts() {
	for hostEntry := range hdb.scanPool {
		// Request settings from the queue'd host entry.
		var settings modules.HostSettings
		err := func() error {
			conn, err := net.DialTimeout("tcp", string(hostEntry.IPAddress), hostRequestTimeout)
			if err != nil {
				return err
			}
			defer conn.Close()
			err = encoding.WriteObject(conn, [8]byte{'S', 'e', 't', 't', 'i', 'n', 'g', 's'})
			if err != nil {
				return err
			}
			return encoding.ReadObject(conn, &settings, maxSettingsLen)
		}()

		// Now that network communication is done, lock the hostdb to modify the
		// host entry.
		id := hdb.mu.Lock()
		{
			if err != nil {
				hdb.decrementReliability(hostEntry.IPAddress, UnreachablePenalty)
				hdb.mu.Unlock(id)
				continue
			}

			// Update the host settings, reliability, and weight. The old IPAddress
			// must be preserved.
			settings.IPAddress = hostEntry.HostSettings.IPAddress
			hostEntry.HostSettings = settings
			hostEntry.reliability = MaxReliability
			hostEntry.weight = hdb.hostWeight(*hostEntry)

			// If the host is not already in the database and 'MaxActiveHosts' has not
			// been reached, add the host to the database.
			_, exists1 := hdb.activeHosts[hostEntry.IPAddress]
			_, exists2 := hdb.allHosts[hostEntry.IPAddress]
			if !exists1 && exists2 && len(hdb.activeHosts) < MaxActiveHosts {
				hdb.insertNode(hostEntry)
				hdb.notifySubscribers()
			}
		}
		hdb.mu.Unlock(id)
	}
}

{'fraction_non_alphanumeric': 0.10158910329171396, 'fraction_numerical': 0.010499432463110102, 'mean_word_length': 3.5951760104302477, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '733418', 'n_tokens_mistral': 2513, 'n_tokens_neox': 2256, 'n_words': 709}