<commit_before>package cache

import (
	"crypto/sha256"
	"fmt"
	"io/ioutil"
	"os"
	"testing"
)

const (
	sampleData = "0123456789"
)

func TestDirectoryCache(t *testing.T) {
	tmp, err := ioutil.TempDir("", "testcache")
	defer os.RemoveAll(tmp)
	if err != nil {
		t.Fatalf("failed to make tempdir: %v", err)
	}

	// with enough memory cache
	c, err := NewDirectoryCache(tmp, 10)
	if err != nil {
		t.Fatalf("failed to make cache: %v", err)
	}
	testCache(t, c)

	// with smaller memory cache
	c, err = NewDirectoryCache(tmp, 1)
	if err != nil {
		t.Fatalf("failed to make cache: %v", err)
	}
	testCache(t, c)
}

func TestMemoryCache(t *testing.T) {
	testCache(t, NewMemoryCache())
}

func testCache(t *testing.T, c BlobCache) {
	tests := []struct {
		name   string
		blobs  []string
		checks []check
	}{
		{
			name: "empty_data",
			blobs: []string{
				"",
			},
			checks: []check{
				hit(""),
				miss(sampleData),
			},
		},
		{
			name: "data",
			blobs: []string{
				sampleData,
			},
			checks: []check{
				hit(sampleData),
				miss("dummy"),
			},
		},
		{
			name: "manydata",
			blobs: []string{
				sampleData,
				"test",
			},
			checks: []check{
				hit(sampleData),
				miss("dummy"),
			},
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			for _, blob := range tt.blobs {
				d := digestFor(blob)
				c.Add(d, []byte(blob))
			}
			for _, check := range tt.checks {
				check(t, c)
			}
		})
	}
}

type check func(*testing.T, BlobCache)

func digestFor(content string) string {
	sum := sha256.Sum256([]byte(content))
	return fmt.Sprintf("%x", sum)
}

func hit(sample string) check {
	return func(t *testing.T, c BlobCache) {
		d := digestFor(sample)
		p, err := c.Fetch(d)
		if err != nil {<commit_msg>Fix test: sync with directory cache<commit_after>package cache

import (
	"crypto/sha256"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"
)

const (
	sampleData = "0123456789"
)

func TestDirectoryCache(t *testing.T) {
	tmp, err := ioutil.TempDir("", "testcache")
	if err != nil {
		t.Fatalf("failed to make tempdir: %v", err)
	}
	defer os.RemoveAll(tmp)

	waitFunc := func(d string) bool {
		// Check if cache exists.
		c := filepath.Join(tmp, d[:2], d)
		if _, err := os.Stat(c); err == nil {
			return true
		}
		return false
	}

	// with enough memory cache
	c, err := NewDirectoryCache(tmp, 10)
	if err != nil {
		t.Fatalf("failed to make cache: %v", err)
	}
	testCache(t, c, tmp, waitFunc)

	// with smaller memory cache
	c, err = NewDirectoryCache(tmp, 1)
	if err != nil {
		t.Fatalf("failed to make cache: %v", err)
	}
	testCache(t, c, tmp, waitFunc)
}

func TestMemoryCache(t *testing.T) {
	testCache(t, NewMemoryCache(), "", func(d string) bool { return true })
}

func testCache(t *testing.T, c BlobCache, cleanDir string, waitFunc func(string) bool) {
	tests := []struct {
		name   string
		blobs  []string
		checks []check
	}{
		{
			name: "empty_data",
			blobs: []string{
				"",
			},
			checks: []check{
				hit(""),
				miss(sampleData),
			},
		},
		{
			name: "data",
			blobs: []string{
				sampleData,
			},
			checks: []check{
				hit(sampleData),
				miss("dummy"),
			},
		},
		{
			name: "manydata",
			blobs: []string{
				sampleData,
				"test",
			},
			checks: []check{
				hit(sampleData),
				miss("dummy"),
			},
		},
		{
			name: "dup_data",
			blobs: []string{
				sampleData,
				sampleData,
			},
			checks: []check{
				hit(sampleData),
			},
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			for _, blob := range tt.blobs {
				d := digestFor(blob)
				c.Add(d, []byte(blob))
				for {
					if complete := waitFunc(d); complete {
						break
					}
				}
			}
			for _, check := range tt.checks {
				check(t, c)
			}

			if cleanDir != "" {
				os.RemoveAll(cleanDir)
				err := os.MkdirAll(cleanDir, os.ModePerm)
				if err != nil {
					t.Fatalf("failed to recreate cachedir: %v", err)
				}
			}
		})
	}
}

type check func(*testing.T, BlobCache)

func digestFor(content string) string {
	sum := sha256.Sum256([]byte(content))
	return fmt.Sprintf("%x", sum)
}

func hit(sample string) check {
	return func(t *testing.T, c BlobCache) {
		d := digestFor(sample)
		p, err := c.Fetch(d)
		if err != nil {
{'fraction_non_alphanumeric': 0.17211174242424243, 'fraction_numerical': 0.01065340909090909, 'mean_word_length': 2.876146788990826, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5352569', 'n_tokens_mistral': 1939, 'n_tokens_neox': 1638, 'n_words': 397}