<commit_before>package main

import (
	"bytes"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"text/template"
)

var (
	repositories = map[string]*template.Template{
		"github.com": template.Must(template.New("github.com").Parse("https://github.com/{{.Vendor}}/{{.Name}}/archive/{{.Version}}.zip")),
	}
)

type Origin struct {
	Host    string
	Vendor  string
	Name    string
	Version string
}

func download(uri, file string) (int64, error) {
	out, err := os.Create(file)
	if err != nil {
		return -1, err
	}
	defer out.Close()

	resp, err := http.Get(uri)
	if err != nil {
		return -1, err
	}
	defer resp.Body.Close()

	size, err := io.Copy(out, resp.Body)
	if err != nil {
		return -1, err
	}

	return size, nil
}

func origin(reference string) Origin {
	pos := strings.LastIndex(reference, ":")
	if pos == -1 {
		parsed := strings.SplitN(reference, "/", 3)
		return Origin{Host: parsed[0], Vendor: parsed[1], Name: parsed[2], Version: "master"}
	} else {
		parsed := strings.SplitN(reference[0:pos], "/", 3)
		return Origin{Host: parsed[0], Vendor: parsed[1], Name: parsed[2], Version: reference[pos+1 : len(reference)]}
	}
}

func fetch(reference string) (string, error) {
	origin := origin(reference)

	if delegate, ok := repositories[origin.Host]; ok {
		var uri bytes.Buffer
		if err := delegate.Execute(&uri, origin); err != nil {
			return "", err
		}

		target := filepath.Join("vendor", origin.Host, origin.Vendor, origin.Name)
		zip := filepath.Join(target, origin.Version+".zip")

		if err := os.MkdirAll(target, 0700); err != nil {
			return "", err
		}

		// TODO: If-Modified-Since!
		_, err := download(uri.String(), zip)
		if err != nil {
			return "", err
		}

		if err := unzip(zip, target, strings.NewReplacer(origin.Name+"-"+origin.Version+"/", "")); err != nil {
			return "", err
		}

		return target, nil
	} else {
		return "", fmt.Errorf("No repository %s", origin.Host)
	}
}<commit_msg>Implement support for caching downloads via If-Modified-Since<commit_after>package main

import (
	"bytes"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"text/template"
)

var (
	repositories = map[string]*template.Template{
		"github.com": template.Must(template.New("github.com").Parse("https://github.com/{{.Vendor}}/{{.Name}}/archive/{{.Version}}.zip")),
	}
)

type Origin struct {
	Host    string
	Vendor  string
	Name    string
	Version string
}

func download(uri, file string) (int64, error) {
	client := &http.Client{}

	req, err := http.NewRequest("GET", uri, nil)
	if err != nil {
		return -1, err
	}

	stat, err := os.Stat(file)
	if err == nil {
		req.Header.Add("If-Modified-Since", stat.ModTime().UTC().Format(http.TimeFormat))
	}

	// DEBUG fmt.Printf(">>> %+v\n", req)

	resp, err := client.Do(req)
	if err != nil {
		return -1, err
	}
	defer resp.Body.Close()

	// DEBUG fmt.Printf("<<< %+v\n", resp)

	switch resp.StatusCode {
	case 200:
		out, err := os.Create(file)
		if err != nil {
			return -1, err
		}
		defer out.Close()

		size, err := io.Copy(out, resp.Body)
		if err != nil {
			return -1, err
		}

		return size, nil
	
	case 304:
		return stat.Size(), nil

	default:
		return -1, fmt.Errorf("Could not download %q, response %+v", uri, resp)
	}
}

func origin(reference string) Origin {
	pos := strings.LastIndex(reference, ":")
	if pos == -1 {
		parsed := strings.SplitN(reference, "/", 3)
		return Origin{Host: parsed[0], Vendor: parsed[1], Name: parsed[2], Version: "master"}
	} else {
		parsed := strings.SplitN(reference[0:pos], "/", 3)
		return Origin{Host: parsed[0], Vendor: parsed[1], Name: parsed[2], Version: reference[pos+1 : len(reference)]}
	}
}

func fetch(reference string) (string, error) {
	origin := origin(reference)

	if delegate, ok := repositories[origin.Host]; ok {
		var uri bytes.Buffer
		if err := delegate.Execute(&uri, origin); err != nil {
			return "", err
		}

		target := filepath.Join("vendor", origin.Host, origin.Vendor, origin.Name)
		zip := filepath.Join(target, origin.Version+".zip")

		if err := os.MkdirAll(target, 0700); err != nil {
			return "", err
		}

		_, err := download(uri.String(), zip)
		if err != nil {
			return "", err
		}

		if err := unzip(zip, target, strings.NewReplacer(origin.Name+"-"+origin.Version+"/", "")); err != nil {
			return "", err
		}

		return target, nil
	} else {
		return "", fmt.Errorf("No repository %s", origin.Host)
	}
}
{'fraction_non_alphanumeric': 0.18443935926773455, 'fraction_numerical': 0.010983981693363844, 'mean_word_length': 3.6010526315789475, 'pattern_counts': {'":': 4, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1005986', 'n_tokens_mistral': 1694, 'n_tokens_neox': 1568, 'n_words': 393}