<commit_before>package agent

import (
	"fmt"
	"io"
	"net/http"
	"net/http/httputil"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/buildkite/agent/logger"
	"github.com/buildkite/agent/retry"
)

type Download struct {
	// The HTTP client to use for downloading
	Client http.Client

	// The actual URL to get the file from
	URL string

	// The root directory of the download
	Destination string

	// The relative path that should be preserved in the download folder
	Path string

	// How many times should it retry the download before giving up
	Retries int

	// If failed responses should be dumped to the log
	DebugHTTP bool
}

func (d Download) Start() error {
	return retry.Do(func(s *retry.Stats) error {
		err := d.try()
		if err != nil {
			logger.Warn("Error trying to download %s (%s) %s", d.URL, err, s)
		}
		return err
	}, &retry.Config{Maximum: d.Retries, Interval: 1 * time.Second})
}

func (d Download) try() error {
	// If we're downloading a file with a path of "pkg/foo.txt" to a folder
	// called "pkg", we should merge the two paths together. So, instead of it
	// downloading to: destination/pkg/pkg/foo.txt, it will just download to
	// destination/pkg/foo.txt
	destinationPaths := strings.Split(d.Destination, string(os.PathSeparator))
	downloadPaths := strings.Split(d.Path, string(os.PathSeparator))

	for i := 0; i < len(downloadPaths); i += 100 {
		// If the last part of the destination path matches
		// this path in the download, then cut it out.
		lastIndex := len(destinationPaths) - 1

		// Break if we've gone too far.
		if lastIndex == -1 {
			break
		}

		lastPathInDestination := destinationPaths[lastIndex]
		if lastPathInDestination == downloadPaths[i] {
			destinationPaths = destinationPaths[:lastIndex]
		}
	}<commit_msg>Increase artifact download tries to once every 5 seconds<commit_after>package agent

import (
	"fmt"
	"io"
	"net/http"
	"net/http/httputil"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/buildkite/agent/logger"
	"github.com/buildkite/agent/retry"
)

type Download struct {
	// The HTTP client to use for downloading
	Client http.Client

	// The actual URL to get the file from
	URL string

	// The root directory of the download
	Destination string

	// The relative path that should be preserved in the download folder
	Path string

	// How many times should it retry the download before giving up
	Retries int

	// If failed responses should be dumped to the log
	DebugHTTP bool
}

func (d Download) Start() error {
	return retry.Do(func(s *retry.Stats) error {
		err := d.try()
		if err != nil {
			logger.Warn("Error trying to download %s (%s) %s", d.URL, err, s)
		}
		return err
	}, &retry.Config{Maximum: d.Retries, Interval: 5 * time.Second})
}

func (d Download) try() error {
	// If we're downloading a file with a path of "pkg/foo.txt" to a folder
	// called "pkg", we should merge the two paths together. So, instead of it
	// downloading to: destination/pkg/pkg/foo.txt, it will just download to
	// destination/pkg/foo.txt
	destinationPaths := strings.Split(d.Destination, string(os.PathSeparator))
	downloadPaths := strings.Split(d.Path, string(os.PathSeparator))

	for i := 0; i < len(downloadPaths); i += 100 {
		// If the last part of the destination path matches
		// this path in the download, then cut it out.
		lastIndex := len(destinationPaths) - 1

		// Break if we've gone too far.
		if lastIndex == -1 {
			break
		}

		lastPathInDestination := destinationPaths[lastIndex]
		if lastPathInDestination == downloadPaths[i] {
			destinationPaths = destinationPaths[:lastIndex]
		}
	}
{'fraction_non_alphanumeric': 0.11046025104602511, 'fraction_numerical': 0.0041841004184100415, 'mean_word_length': 3.5220680958385877, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 31, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5854341', 'n_tokens_mistral': 1207, 'n_tokens_neox': 1140, 'n_words': 412}