<commit_before>package sigupdate

import (
	"errors"
	"fmt"
	"github.com/dekobon/clamav-mirror/utils"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"time"
)

import (
	"github.com/hashicorp/errwrap"
)

// Function that downloads a file from the mirror URL and moves it into the
// data directory if it was successfully downloaded.
func downloadFile(filename string, localFilePath string,
	downloadMirrorURL string) (int, error) {

	unknownStatus := -1
	downloadURL := downloadMirrorURL + "/" + filename

	output, err := ioutil.TempFile(os.TempDir(), filename+"-")

	/* For all non-cvd files, skip downloading the file if our local copy is
	 * newer than the remote copy. For .cvd files, the only authoritative way
	 * to know what is newer is to use sigtool. */
	if utils.Exists(localFilePath) {
		newer, err := checkIfRemoteIsNewer(localFilePath, downloadURL)

		if err != nil {
			return unknownStatus, err
		}

		if !newer {
			logger.Printf("Not downloading [%v] because remote copy is "+
				"older than local copy", filename)
			return unknownStatus, nil
		}
	}

	if verboseMode {
		logger.Printf("Downloading to temporary file: [%v]", output.Name())
	}

	if err != nil {
		msg := fmt.Sprintf("Unable to create file: [%v]. {{err}}", output.Name())
		return unknownStatus, errwrap.Wrapf(msg, err)
	}

	defer output.Close()

	response, err := http.Get(downloadURL)

	if err != nil {
		msg := fmt.Sprintf("Unable to retrieve file from: [%v]. {{err}}", downloadURL)
		return unknownStatus, errwrap.Wrapf(msg, err)
	}

	if response.StatusCode != http.StatusOK {
		msg := fmt.Sprintf("Unable to download file: [%v]", response.Status)
		return response.StatusCode, errors.New(msg)
	}

	lastModified, err := http.ParseTime(response.Header.Get("Last-Modified"))

	if err != nil {
		logger.Printf("Error parsing last-modified header [%v] for file: %v",
			response.Header.Get("Last-Modified"), downloadURL)
		lastModified = time.Now()
	}

	defer response.Body.Close()

	n, err := io.Copy(output, response.Body)

	if err != nil {
		msg := fmt.Sprintf("Error copying data from URL [%v] to local file [%v]. {{err}}",
			downloadURL, localFilePath)
		return response.StatusCode, errwrap.Wrapf(msg, err)
	}

	if isItOkToOverwrite(filename, localFilePath, output.Name()) {
		/* Change the last modified time so that we have a record that corresponds to the
		 * server's timestamps. */
		os.Chtimes(output.Name(), lastModified, lastModified)
		os.Rename(output.Name(), localFilePath)

		logger.Printf("Download complete: %v --> %v [%v bytes]", downloadURL, localFilePath, n)
	} else {
		logger.Println("Downloaded file an older signature version than the current file")

		err := os.Remove(output.Name())

		if err != nil {
			logger.Printf("Unable to delete temporary file: %v", output.Name())
		}
	}

	return response.StatusCode, nil
}

// Function that checks to see if the remote file is newer than the locally stored
// file.
func checkIfRemoteIsNewer(localFilePath string, downloadURL string) (bool, error) {
	localFileStat, err := os.Stat(localFilePath)

	if err != nil {
		return true, errwrap.Wrapf("Unable to stat file. {{err}}", err)
	}

	localModTime := localFileStat.ModTime()
	response, err := http.Head(downloadURL)

	if err != nil {
		msg := fmt.Sprintf("Unable to complete HEAD request: [%v]. {{err}}", downloadURL)
		return true, errwrap.Wrapf(msg, err)
	}

	remoteModTime, err := http.ParseTime(response.Header.Get("Last-Modified"))

	if verboseMode {
		logger.Printf("Local file [%v] last-modified: %v", downloadURL, localModTime)
		logger.Printf("Remote file [%v] last-modified: %v", downloadURL, remoteModTime)
	}

	if err != nil {
		msg := fmt.Sprintf("Error parsing last-modified header [%v] for file [%v]. {{err}}",
			response.Header.Get("Last-Modified"), downloadURL)
		return true, errwrap.Wrapf(msg, err)
	}

	if localModTime.After(remoteModTime) {
		logger.Printf("Skipping download of [%v] because local copy is newer", downloadURL)
		return false, nil
	}

	return true, nil
}

// Function that checks to see if we can overwrite a file with a newly downloaded file
func isItOkToOverwrite(filename string, originalFilePath string, newFileTempPath string) bool {
	if !strings.HasSuffix(filename, ".cvd") {
		return true
	}

	oldVersion, err := findLocalVersion(originalFilePath)

	// If there is a problem with the original file, we just overwrite it
	if err != nil {
		return true
	}

	newVersion, err := findLocalVersion(newFileTempPath)

	// If there is a problem with the new file, we don't overwrite the original
	if err != nil {
		return false
	}

	isNewer := newVersion > oldVersion

	if verboseMode {
		logger.Printf("Current file [%v] version [%v]. New file version [%v]. "+
			"Will overwrite: %v",
			filename, oldVersion, newVersion, isNewer)
	}<commit_msg>Added user-agent and if-modified-since to client headers<commit_after>package sigupdate

import (
	"errors"
	"fmt"
	"github.com/dekobon/clamav-mirror/utils"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"time"
)

import (
	"github.com/hashicorp/errwrap"
)

// Function that downloads a file from the mirror URL and moves it into the
// data directory if it was successfully downloaded.
func downloadFile(filename string, localFilePath string,
	downloadMirrorURL string) (int, error) {

	unknownStatus := -1
	downloadURL := downloadMirrorURL + "/" + filename

	output, err := ioutil.TempFile(os.TempDir(), filename+"-")


	if verboseMode {
		logger.Printf("Downloading to temporary file: [%v]", output.Name())
	}

	if err != nil {
		msg := fmt.Sprintf("Unable to create file: [%v]. {{err}}", output.Name())
		return unknownStatus, errwrap.Wrapf(msg, err)
	}

	defer output.Close()

	request, err := http.NewRequest("GET", downloadURL, nil)

	if err != nil {
		msg := fmt.Sprintf("Unable to create request for: [GEt %v]. {{err}}", downloadURL)
		return unknownStatus, errwrap.Wrapf(msg, err)
	}

	request.Header.Add("User-Agent", "github.com/dekobon/clamav-mirror")

	/* For all non-cvd files, skip downloading the file if our local copy is
	 * newer than the remote copy. For .cvd files, the only authoritative way
	 * to know what is newer is to use sigtool. */
	if utils.Exists(localFilePath) {
		stat, err := os.Stat(localFilePath)

		if err == nil {
			localModTime := stat.ModTime().UTC().Format(http.TimeFormat)
			request.Header.Add("If-Modified-Since", localModTime)
		} else {
			logger.Printf("Unable to stat local file [%v]. %v", localFilePath, err)
		}
	}

	response, err := http.DefaultClient.Do(request)

	if err != nil {
		msg := fmt.Sprintf("Unable to retrieve file from: [%v]. {{err}}", downloadURL)
		return unknownStatus, errwrap.Wrapf(msg, err)
	}

	if response.StatusCode == http.StatusNotModified {
		logger.Printf("Not downloading [%v] because local copy is newer or the same as remote",
			filename)
		return response.StatusCode, nil
	}

	if response.StatusCode != http.StatusOK {
		msg := fmt.Sprintf("Unable to download file: [%v]", response.Status)
		return response.StatusCode, errors.New(msg)
	}

	lastModified, err := http.ParseTime(response.Header.Get("Last-Modified"))

	if err != nil {
		logger.Printf("Error parsing last-modified header [%v] for file: %v",
			response.Header.Get("Last-Modified"), downloadURL)
		lastModified = time.Now()
	}

	defer response.Body.Close()

	n, err := io.Copy(output, response.Body)

	if err != nil {
		msg := fmt.Sprintf("Error copying data from URL [%v] to local file [%v]. {{err}}",
			downloadURL, localFilePath)
		return response.StatusCode, errwrap.Wrapf(msg, err)
	}

	if isItOkToOverwrite(filename, localFilePath, output.Name()) {
		/* Change the last modified time so that we have a record that corresponds to the
		 * server's timestamps. */
		os.Chtimes(output.Name(), lastModified, lastModified)
		os.Rename(output.Name(), localFilePath)

		logger.Printf("Download complete: %v --> %v [%v bytes]", downloadURL, localFilePath, n)
	} else {
		logger.Println("Downloaded file an older signature version than the current file")

		err := os.Remove(output.Name())

		if err != nil {
			logger.Printf("Unable to delete temporary file: %v", output.Name())
		}
	}

	return response.StatusCode, nil
}

// Function that checks to see if we can overwrite a file with a newly downloaded file
func isItOkToOverwrite(filename string, originalFilePath string, newFileTempPath string) bool {
	if !strings.HasSuffix(filename, ".cvd") {
		return true
	}

	oldVersion, err := findLocalVersion(originalFilePath)

	// If there is a problem with the original file, we just overwrite it
	if err != nil {
		return true
	}

	newVersion, err := findLocalVersion(newFileTempPath)

	// If there is a problem with the new file, we don't overwrite the original
	if err != nil {
		return false
	}

	isNewer := newVersion > oldVersion

	if verboseMode {
		logger.Printf("Current file [%v] version [%v]. New file version [%v]. "+
			"Will overwrite: %v",
			filename, oldVersion, newVersion, isNewer)
	}
{'fraction_non_alphanumeric': 0.1307460015658204, 'fraction_numerical': 0.0002236886254333967, 'mean_word_length': 4.127293577981652, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 46, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2366927', 'n_tokens_mistral': 3013, 'n_tokens_neox': 2823, 'n_words': 901}