<commit_before>package splibaux

import (
	"crypto/md5"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"

	"github.com/gregjones/httpcache"
	"github.com/gregjones/httpcache/diskcache"
)

var (
	cachemethod string
	client      *http.Client
)

func init() {
	cachemethod = os.Getenv("CACHEMETHOD")
}

func docaching(cachedir, outfilename, url string) (string, error) {
	// Let's assume cachedir already exists
	var err error
	hashedFilename := fmt.Sprintf("%x", md5.Sum([]byte(outfilename)))
	outpath := filepath.Join(cachedir, hashedFilename)

	if cachemethod == "optimal" {
		if client == nil {
			goimgcache := filepath.Join(cachedir, "sp")
			err = os.MkdirAll(goimgcache, 0755)
			if err != nil {
				return "", err
			}
			dc := diskcache.New(goimgcache)
			dcTP := httpcache.NewTransport(dc)
			client = dcTP.Client()
		}

		resp, err := client.Get(url)
		if err != nil {
			return "", err
		}
		if resp.StatusCode == http.StatusNotFound {
			os.Remove(outpath)
			return "", fmt.Errorf("Resource not found (404): %q", url)
		}
		outf, err := os.Create(outpath)
		if err != nil {
			return "", err
		}
		defer outf.Close()
		io.Copy(outf, resp.Body)
	}
	return outpath, nil
}<commit_msg>Better timing for disk caching<commit_after>package splibaux

import (
	"crypto/md5"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"

	"github.com/gregjones/httpcache"
	"github.com/gregjones/httpcache/diskcache"
)

var (
	cachemethod string
	client      *http.Client
)

func init() {
	cachemethod = os.Getenv("CACHEMETHOD")
}

func docaching(cachedir, outfilename, url string) (string, error) {
	// Let's assume cachedir already exists
	var err error
	hashedFilename := fmt.Sprintf("%x", md5.Sum([]byte(outfilename)))
	outpath := filepath.Join(cachedir, hashedFilename)

	if cachemethod == "optimal" {
		if client == nil {
			goimgcache := filepath.Join(cachedir, "sp")
			err = os.MkdirAll(goimgcache, 0755)
			if err != nil {
				return "", err
			}
			dc := diskcache.New(goimgcache)
			dcTP := httpcache.NewTransport(dc)
			client = dcTP.Client()
		}

		resp, err := client.Get(url)
		if err != nil {
			return "", err
		}
		if resp.StatusCode == http.StatusNotFound {
			os.Remove(outpath)
			return "", fmt.Errorf("Resource not found (404): %q", url)
		}
		outf, err := ioutil.TempFile(cachedir, "download")
		if err != nil {
			return "", err
		}

		if _, err = io.Copy(outf, resp.Body); err != nil {
			return "", err
		}
		resp.Body.Close()

		temfilename := outf.Name()
		if err = outf.Close(); err != nil {
			return "", err
		}
		err = os.Rename(temfilename, outpath)
		return outpath, err

	}
	return outpath, nil
}
{'fraction_non_alphanumeric': 0.1519756838905775, 'fraction_numerical': 0.006838905775075988, 'mean_word_length': 3.4855195911413968, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2861878', 'n_tokens_mistral': 1079, 'n_tokens_neox': 968, 'n_words': 222}