<commit_before>// Package database provides an abstraction over getting and writing a
// database file.
package database

import (
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"time"

	"github.com/maxmind/geoipupdate/v4/pkg/geoipupdate"
	"github.com/maxmind/geoipupdate/v4/pkg/geoipupdate/internal"
	"github.com/pkg/errors"
)

// HTTPDatabaseReader is a Reader that uses an HTTP client to retrieve
// databases.
type HTTPDatabaseReader struct {
	client            *http.Client
	retryFor          time.Duration
	url               string
	licenseKey        string
	accountID         int
	preserveFileTimes bool
	verbose           bool
}

// NewHTTPDatabaseReader creates a Reader that downloads database updates via
// HTTP.
func NewHTTPDatabaseReader(client *http.Client, config *geoipupdate.Config) Reader {
	return &HTTPDatabaseReader{
		client:            client,
		retryFor:          config.RetryFor,
		url:               config.URL,
		licenseKey:        config.LicenseKey,
		accountID:         config.AccountID,
		preserveFileTimes: config.PreserveFileTimes,
		verbose:           config.Verbose,
	}
}

// Get retrieves the given edition ID using an HTTP client, writes it to the
// Writer, and validates the hash before committing.
func (reader *HTTPDatabaseReader) Get(destination Writer, editionID string) error {
	defer func() {
		if err := destination.Close(); err != nil {
			log.Println(err)
		}
	}()

	maxMindURL := fmt.Sprintf(
		"%s/geoip/databases/%s/update?db_md5=%s",
		reader.url,
		url.PathEscape(editionID),
		url.QueryEscape(destination.GetHash()),
	)

	if reader.verbose {
		log.Printf("Performing update request to %s", maxMindURL)
	}

	req, err := http.NewRequest(http.MethodGet, maxMindURL, nil) // nolint: noctx
	if err != nil {
		return errors.Wrap(err, "error creating request")
	}
	req.SetBasicAuth(fmt.Sprintf("%d", reader.accountID), reader.licenseKey)

	response, err := internal.MaybeRetryRequest(reader.client, reader.retryFor, req)
	if err != nil {
		return errors.Wrap(err, "error performing HTTP request")
	}
	defer func() {
		if err := response.Body.Close(); err != nil {
			log.Fatalf("Error closing response body: %+v", errors.Wrap(err, "closing body"))
		}
	}()

	if response.StatusCode == http.StatusNotModified {
		if reader.verbose {
			log.Printf("No new updates available for %s", editionID)
		}
		return nil
	}

	if response.StatusCode != http.StatusOK {
		buf, err := ioutil.ReadAll(io.LimitReader(response.Body, 256))
		if err == nil {
			return errors.Errorf("unexpected HTTP status code: %s: %s", response.Status, buf)
		}
		return errors.Errorf("unexpected HTTP status code: %s", response.Status)
	}

	gzReader, err := gzip.NewReader(response.Body)
	if err != nil {
		return errors.Wrap(err, "encountered an error creating GZIP reader")
	}
	defer func() {
		if err := gzReader.Close(); err != nil {
			log.Printf("error closing gzip reader: %s", err)
		}
	}()

	if _, err = io.Copy(destination, gzReader); err != nil { //nolint:gosec
		return errors.Wrap(err, "error writing response")
	}

	newMD5 := response.Header.Get("X-Database-MD5")
	if newMD5 == "" {
		return errors.New("no X-Database-MD5 header found")
	}
	if err := destination.ValidHash(newMD5); err != nil {
		return err
	}

	if err := destination.Commit(); err != nil {
		return errors.Wrap(err, "encountered an issue committing database update")
	}

	if reader.preserveFileTimes {
		modificationTime, err := lastModified(response.Header.Get("Last-Modified"))
		if err != nil {
			return errors.Wrap(err, "unable to get last modified time")
		}
		err = destination.SetFileModificationTime(modificationTime)
		if err != nil {
			return errors.Wrap(err, "unable to set modification time")
		}
	}

	return nil
}

// LastModified retrieves the date that the MaxMind database was last modified.
func lastModified(lastModified string) (time.Time, error) {
	if lastModified == "" {<commit_msg>Retry more of the database download process<commit_after>// Package database provides an abstraction over getting and writing a
// database file.
package database

import (
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"time"

	"github.com/maxmind/geoipupdate/v4/pkg/geoipupdate"
	"github.com/maxmind/geoipupdate/v4/pkg/geoipupdate/internal"
	"github.com/pkg/errors"
)

// HTTPDatabaseReader is a Reader that uses an HTTP client to retrieve
// databases.
type HTTPDatabaseReader struct {
	client            *http.Client
	retryFor          time.Duration
	url               string
	licenseKey        string
	accountID         int
	preserveFileTimes bool
	verbose           bool
}

// NewHTTPDatabaseReader creates a Reader that downloads database updates via
// HTTP.
func NewHTTPDatabaseReader(client *http.Client, config *geoipupdate.Config) Reader {
	return &HTTPDatabaseReader{
		client:            client,
		retryFor:          config.RetryFor,
		url:               config.URL,
		licenseKey:        config.LicenseKey,
		accountID:         config.AccountID,
		preserveFileTimes: config.PreserveFileTimes,
		verbose:           config.Verbose,
	}
}

// Get retrieves the given edition ID using an HTTP client, writes it to the
// Writer, and validates the hash before committing.
func (reader *HTTPDatabaseReader) Get(destination Writer, editionID string) error {
	defer func() {
		if err := destination.Close(); err != nil {
			log.Println(err)
		}
	}()

	maxMindURL := fmt.Sprintf(
		"%s/geoip/databases/%s/update?db_md5=%s",
		reader.url,
		url.PathEscape(editionID),
		url.QueryEscape(destination.GetHash()),
	)

	if reader.verbose {
		log.Printf("Performing update request to %s", maxMindURL)
	}

	var modified bool
	// It'd be nice to not use a temporary file here. However the Writer API does
	// not currently support multiple attempts to download the file (it assumes
	// we'll begin writing once). Using a temporary file here should be a less
	// disruptive alternative to changing the API. If we change that API in the
	// future, adding something like Reset() may be desirable.
	tempFile, err := ioutil.TempFile("", "geoipupdate")
	if err != nil {
		return errors.Wrap(err, "error opening temporary file")
	}
	defer func() {
		if err := tempFile.Close(); err != nil {
			log.Printf("error closing temporary file: %s", err)
		}
		if err := os.Remove(tempFile.Name()); err != nil {
			log.Printf("error removing temporary file: %s", err)
		}
	}()
	var newMD5 string
	var modificationTime time.Time
	err = internal.RetryWithBackoff(
		func() error {
			// Prepare a clean slate for this download attempt.

			modified = false
			if err := tempFile.Truncate(0); err != nil {
				return errors.Wrap(err, "error truncating")
			}
			if _, err := tempFile.Seek(0, 0); err != nil {
				return errors.Wrap(err, "error seeking")
			}
			newMD5 = ""
			modificationTime = time.Time{}

			// Perform the download.

			req, err := http.NewRequest(http.MethodGet, maxMindURL, nil) // nolint: noctx
			if err != nil {
				return errors.Wrap(err, "error creating request")
			}
			req.SetBasicAuth(fmt.Sprintf("%d", reader.accountID), reader.licenseKey)

			response, err := reader.client.Do(req)
			if err != nil {
				return errors.Wrap(err, "error performing HTTP request")
			}
			defer func() {
				if err := response.Body.Close(); err != nil {
					log.Fatalf("Error closing response body: %+v", errors.Wrap(err, "closing body"))
				}
			}()

			if response.StatusCode == http.StatusNotModified {
				if reader.verbose {
					log.Printf("No new updates available for %s", editionID)
				}
				return nil
			}
			modified = true

			if response.StatusCode != http.StatusOK {
				buf, err := ioutil.ReadAll(io.LimitReader(response.Body, 256))
				if err == nil {
					return errors.Errorf("unexpected HTTP status code: %s: %s", response.Status, buf)
				}
				return errors.Errorf("unexpected HTTP status code: %s", response.Status)
			}

			gzReader, err := gzip.NewReader(response.Body)
			if err != nil {
				return errors.Wrap(err, "encountered an error creating GZIP reader")
			}
			defer func() {
				if err := gzReader.Close(); err != nil {
					log.Printf("error closing gzip reader: %s", err)
				}
			}()

			if _, err := io.Copy(tempFile, gzReader); err != nil { //nolint:gosec
				return errors.Wrap(err, "error writing response")
			}

			newMD5 = response.Header.Get("X-Database-MD5")
			if newMD5 == "" {
				return errors.New("no X-Database-MD5 header found")
			}

			modificationTime, err = lastModified(response.Header.Get("Last-Modified"))
			if err != nil {
				return errors.Wrap(err, "unable to get last modified time")
			}

			return nil
		},
		reader.retryFor,
	)
	if err != nil {
		return err
	}

	if !modified {
		return nil
	}

	if _, err := tempFile.Seek(0, 0); err != nil {
		return errors.Wrap(err, "error seeking")
	}

	if _, err = io.Copy(destination, tempFile); err != nil {
		return errors.Wrap(err, "error writing response")
	}

	if err := destination.ValidHash(newMD5); err != nil {
		return err
	}

	if err := destination.Commit(); err != nil {
		return errors.Wrap(err, "encountered an issue committing database update")
	}

	if reader.preserveFileTimes {
		err = destination.SetFileModificationTime(modificationTime)
		if err != nil {
			return errors.Wrap(err, "unable to set modification time")
		}
	}

	return nil
}

// LastModified retrieves the date that the MaxMind database was last modified.
func lastModified(lastModified string) (time.Time, error) {
	if lastModified == "" {
{'fraction_non_alphanumeric': 0.12140373063547265, 'fraction_numerical': 0.0030561703024554747, 'mean_word_length': 3.369244935543278, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1069052', 'n_tokens_mistral': 3289, 'n_tokens_neox': 3031, 'n_words': 869}