<commit_before>// Copyright 2018 <NAME>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package backend

import (
	"bytes"
	"fmt"
	"os"
	"reflect"
	"runtime"
	"testing"
	"time"
)

func tempDir() string {
	if runtime.GOOS == "linux" {
		return "/var/tmp"
	} else {
		return os.TempDir()
	}
}

func makeStorageBackends(t *testing.T) (map[string]Storage, func()) {
	tempDir, err := os.MkdirTemp(tempDir(), "fakegcstest")
	if err != nil {
		t.Fatal(err)
	}
	storageFS, err := NewStorageFS(nil, tempDir)
	if err != nil {
		t.Fatal(err)
	}
	return map[string]Storage{
			"memory":     NewStorageMemory(nil),
			"filesystem": storageFS,
		}, func() {
			err := os.RemoveAll(tempDir)
			if err != nil {
				t.Fatal(err)
			}
		}
}

func testForStorageBackends(t *testing.T, test func(t *testing.T, storage Storage)) {
	backends, cleanup := makeStorageBackends(t)
	defer cleanup()
	for backendName, storage := range backends {
		storage := storage
		t.Run(fmt.Sprintf("storage backend %s", backendName), func(t *testing.T) {
			test(t, storage)
		})
	}
}

func noError(t *testing.T, err error) {
	t.Helper()
	if err != nil {
		t.Fatalf("should not error, but got: %v", err)
	}
}

func shouldError(t *testing.T, err error) {
	t.Helper()
	if err == nil {
		t.Fatalf("should error, but error is nil")
	}
}

func uploadAndCompare(t *testing.T, storage Storage, obj Object) int64 {
	isFSStorage := reflect.TypeOf(storage) == reflect.TypeOf(&storageFS{})
	_, err := storage.CreateObject(obj)
	if isFSStorage && obj.Generation != 0 {
		t.Log("FS should not support objects generation")
		shouldError(t, err)
		obj.Generation = 0
		_, err = storage.CreateObject(obj)
	}
	noError(t, err)
	activeObj, err := storage.GetObject(obj.BucketName, obj.Name)
	noError(t, err)
	if activeObj.Generation == 0 {
		t.Errorf("generation is empty, but we expect a unique int")
	}
	if err := compareObjects(activeObj, obj); err != nil {
		t.Errorf("object retrieved differs from the created one. Descr: %v", err)
	}
	objFromGeneration, err := storage.GetObjectWithGeneration(obj.BucketName, obj.Name, activeObj.Generation)
	noError(t, err)
	if err := compareObjects(objFromGeneration, obj); err != nil {
		t.Errorf("object retrieved differs from the created one. Descr: %v", err)
	}
	return activeObj.Generation
}

func TestObjectCRUD(t *testing.T) {
	const bucketName = "prod-bucket"
	const objectName = "video/hi-res/best_video_1080p.mp4"
	content1 := []byte("content1")
	const crc1 = "crc1"
	const md51 = "md51"
	content2 := []byte("content2")
	for _, versioningEnabled := range []bool{true, false} {
		versioningEnabled := versioningEnabled
		testForStorageBackends(t, func(t *testing.T, storage Storage) {
			// Get in non-existent case
			_, err := storage.GetObject(bucketName, objectName)
			shouldError(t, err)
			// Delete in non-existent case
			err = storage.DeleteObject(bucketName, objectName)
			shouldError(t, err)
			err = storage.CreateBucket(bucketName, versioningEnabled)
			if reflect.TypeOf(storage) == reflect.TypeOf(&storageFS{}) && versioningEnabled {
				t.Log("FS storage type should not implement versioning")
				shouldError(t, err)
				return
			}

			initialObject := Object{
				ObjectAttrs: ObjectAttrs{
					BucketName: bucketName,
					Name:       objectName,
					Crc32c:     crc1,
					Md5Hash:    md51,
				},
				Content: content1,
			}<commit_msg>internal/backend: skip TestObjectCRUD on Windows<commit_after>// Copyright 2018 <NAME>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package backend

import (
	"bytes"
	"fmt"
	"os"
	"reflect"
	"runtime"
	"testing"
	"time"
)

func tempDir() string {
	if runtime.GOOS == "linux" {
		return "/var/tmp"
	} else {
		return os.TempDir()
	}
}

func makeStorageBackends(t *testing.T) (map[string]Storage, func()) {
	tempDir, err := os.MkdirTemp(tempDir(), "fakegcstest")
	if err != nil {
		t.Fatal(err)
	}
	storageFS, err := NewStorageFS(nil, tempDir)
	if err != nil {
		t.Fatal(err)
	}
	return map[string]Storage{
			"memory":     NewStorageMemory(nil),
			"filesystem": storageFS,
		}, func() {
			err := os.RemoveAll(tempDir)
			if err != nil {
				t.Fatal(err)
			}
		}
}

func testForStorageBackends(t *testing.T, test func(t *testing.T, storage Storage)) {
	backends, cleanup := makeStorageBackends(t)
	defer cleanup()
	for backendName, storage := range backends {
		storage := storage
		t.Run(fmt.Sprintf("storage backend %s", backendName), func(t *testing.T) {
			test(t, storage)
		})
	}
}

func noError(t *testing.T, err error) {
	t.Helper()
	if err != nil {
		t.Fatalf("should not error, but got: %v", err)
	}
}

func shouldError(t *testing.T, err error) {
	t.Helper()
	if err == nil {
		t.Fatal("should error, but error is nil")
	}
}

func uploadAndCompare(t *testing.T, storage Storage, obj Object) int64 {
	isFSStorage := reflect.TypeOf(storage) == reflect.TypeOf(&storageFS{})
	_, err := storage.CreateObject(obj)
	if isFSStorage && obj.Generation != 0 {
		t.Log("FS should not support objects generation")
		shouldError(t, err)
		obj.Generation = 0
		_, err = storage.CreateObject(obj)
	}
	noError(t, err)
	activeObj, err := storage.GetObject(obj.BucketName, obj.Name)
	noError(t, err)
	if activeObj.Generation == 0 {
		t.Errorf("generation is empty, but we expect a unique int")
	}
	if err := compareObjects(activeObj, obj); err != nil {
		t.Errorf("object retrieved differs from the created one. Descr: %v", err)
	}
	objFromGeneration, err := storage.GetObjectWithGeneration(obj.BucketName, obj.Name, activeObj.Generation)
	noError(t, err)
	if err := compareObjects(objFromGeneration, obj); err != nil {
		t.Errorf("object retrieved differs from the created one. Descr: %v", err)
	}
	return activeObj.Generation
}

func TestObjectCRUD(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Skip("time resolution on Windows makes this test flaky on that platform")
	}

	const bucketName = "prod-bucket"
	const objectName = "video/hi-res/best_video_1080p.mp4"
	content1 := []byte("content1")
	const crc1 = "crc1"
	const md51 = "md51"
	content2 := []byte("content2")
	for _, versioningEnabled := range []bool{true, false} {
		versioningEnabled := versioningEnabled
		testForStorageBackends(t, func(t *testing.T, storage Storage) {
			// Get in non-existent case
			_, err := storage.GetObject(bucketName, objectName)
			shouldError(t, err)
			// Delete in non-existent case
			err = storage.DeleteObject(bucketName, objectName)
			shouldError(t, err)
			err = storage.CreateBucket(bucketName, versioningEnabled)
			if reflect.TypeOf(storage) == reflect.TypeOf(&storageFS{}) && versioningEnabled {
				t.Log("FS storage type should not implement versioning")
				shouldError(t, err)
				return
			}

			initialObject := Object{
				ObjectAttrs: ObjectAttrs{
					BucketName: bucketName,
					Name:       objectName,
					Crc32c:     crc1,
					Md5Hash:    md51,
				},
				Content: content1,
			}
{'fraction_non_alphanumeric': 0.13605248146035367, 'fraction_numerical': 0.008841985168282943, 'mean_word_length': 3.9387323943661974, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7464757', 'n_tokens_mistral': 2628, 'n_tokens_neox': 2392, 'n_words': 638}