<commit_before>package renter

import (
	"errors"
	"io"
	"net"
	"os"
	"sync/atomic"
	"time"

	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
)

var (
	errInsufficientHosts  = errors.New("insufficient hosts to recover file")
	errInsufficientPieces = errors.New("couldn't fetch enough pieces to recover data")
)

// A fetcher fetches pieces from a host. This interface exists to facilitate
// easy testing.
type fetcher interface {
	// pieces returns the set of pieces corresponding to a given chunk.
	pieces(chunk uint64) []pieceData

	// fetch returns the data specified by piece metadata.
	fetch(pieceData) ([]byte, error)
}

// A hostFetcher fetches pieces from a host. It implements the fetcher
// interface.
type hostFetcher struct {
	conn      net.Conn
	pieceMap  map[uint64][]pieceData
	pieceSize uint64
	masterKey crypto.TwofishKey
}

// pieces returns the pieces stored on this host that are part of a given
// chunk.
func (hf *hostFetcher) pieces(chunk uint64) []pieceData {
	return hf.pieceMap[chunk]
}

// fetch downloads the piece specified by p.
func (hf *hostFetcher) fetch(p pieceData) ([]byte, error) {
	// request piece
	err := encoding.WriteObject(hf.conn, modules.DownloadRequest{p.Offset, hf.pieceSize})
	if err != nil {
		return nil, err
	}

	// download piece
	data := make([]byte, hf.pieceSize)
	_, err = io.ReadFull(hf.conn, data)
	if err != nil {
		return nil, err
	}

	// generate decryption key
	key := deriveKey(hf.masterKey, p.Chunk, p.Piece)

	// decrypt and return
	return key.DecryptBytes(data)
}

func (hf *hostFetcher) Close() error {
	// ignore error; we'll need to close conn anyway
	encoding.WriteObject(hf.conn, modules.DownloadRequest{0, 0})
	return hf.conn.Close()
}

// newHostFetcher creates a new hostFetcher by connecting to a host.
// TODO: We may not wind up requesting data from this, which means we will
// connect and then disconnect without making any actual requests (but holding
// the connection open the entire time). This is wasteful of host resources.
// Consider only opening the connection after the first request has been made.
func newHostFetcher(fc fileContract, pieceSize uint64, masterKey crypto.TwofishKey) (*hostFetcher, error) {
	conn, err := net.DialTimeout("tcp", string(fc.IP), 5*time.Second)
	if err != nil {
		return nil, err
	}

	// send RPC
	err = encoding.WriteObject(conn, modules.RPCDownload)
	if err != nil {
		return nil, err<commit_msg>add timeouts to download<commit_after>package renter

import (
	"errors"
	"io"
	"net"
	"os"
	"sync/atomic"
	"time"

	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
)

var (
	errInsufficientHosts  = errors.New("insufficient hosts to recover file")
	errInsufficientPieces = errors.New("couldn't fetch enough pieces to recover data")
)

// A fetcher fetches pieces from a host. This interface exists to facilitate
// easy testing.
type fetcher interface {
	// pieces returns the set of pieces corresponding to a given chunk.
	pieces(chunk uint64) []pieceData

	// fetch returns the data specified by piece metadata.
	fetch(pieceData) ([]byte, error)
}

// A hostFetcher fetches pieces from a host. It implements the fetcher
// interface.
type hostFetcher struct {
	conn      net.Conn
	pieceMap  map[uint64][]pieceData
	pieceSize uint64
	masterKey crypto.TwofishKey
}

// pieces returns the pieces stored on this host that are part of a given
// chunk.
func (hf *hostFetcher) pieces(chunk uint64) []pieceData {
	return hf.pieceMap[chunk]
}

// fetch downloads the piece specified by p.
func (hf *hostFetcher) fetch(p pieceData) ([]byte, error) {
	hf.conn.SetDeadline(time.Now().Add(1 * time.Minute)) // sufficient to transfer 4 MB over 500 kbps
	defer hf.conn.SetDeadline(time.Time{})
	// request piece
	err := encoding.WriteObject(hf.conn, modules.DownloadRequest{p.Offset, hf.pieceSize})
	if err != nil {
		return nil, err
	}

	// download piece
	data := make([]byte, hf.pieceSize)
	_, err = io.ReadFull(hf.conn, data)
	if err != nil {
		return nil, err
	}

	// generate decryption key
	key := deriveKey(hf.masterKey, p.Chunk, p.Piece)

	// decrypt and return
	return key.DecryptBytes(data)
}

func (hf *hostFetcher) Close() error {
	// ignore error; we'll need to close conn anyway
	encoding.WriteObject(hf.conn, modules.DownloadRequest{0, 0})
	return hf.conn.Close()
}

// newHostFetcher creates a new hostFetcher by connecting to a host.
// TODO: We may not wind up requesting data from this, which means we will
// connect and then disconnect without making any actual requests (but holding
// the connection open the entire time). This is wasteful of host resources.
// Consider only opening the connection after the first request has been made.
func newHostFetcher(fc fileContract, pieceSize uint64, masterKey crypto.TwofishKey) (*hostFetcher, error) {
	conn, err := net.DialTimeout("tcp", string(fc.IP), 15*time.Second)
	if err != nil {
		return nil, err
	}
	conn.SetDeadline(time.Now().Add(15 * time.Second))
	defer conn.SetDeadline(time.Time{})

	// send RPC
	err = encoding.WriteObject(conn, modules.RPCDownload)
	if err != nil {
		return nil, err
{'fraction_non_alphanumeric': 0.108678955453149, 'fraction_numerical': 0.006528417818740399, 'mean_word_length': 4.062196307094267, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4120625', 'n_tokens_mistral': 1703, 'n_tokens_neox': 1682, 'n_words': 568}