<commit_before>package srvr

import (
	"encoding/json"
	"fmt"
	"math/rand"
	"os"
	"strings"
	"sync"
	"testing"
	"time"
)

var tmpFileCount int
var randVal int64

var dirLock = &sync.RWMutex{}

func GenerateFileName() string {
	tmpFileCount++
	return fmt.Sprintf("tmp%d_%d.json", randVal, tmpFileCount)
}

type TestPersistable struct {
	TimestampHolder
	Data     string `json:"data"`
	Filename string `json:"-"`
}

func (tp *TestPersistable) Filepath() string {
	return tp.Filename
}

func LockfileName(filename string) string {
	return filename + ".lk"
}

func NewTestPersistable() *TestPersistable {
	tmpFileCount++
	if randVal == 0 {
		randVal = time.Now().UnixNano()
	}
	tp := &TestPersistable{
		Data:     "Some information here",
		Filename: GenerateFileName(),
	}
	return tp
}

func TestDelete(t *testing.T) {
	tp := NewTestPersistable()
	filename := tp.Filepath()
	create(filename, t)
	write(filename, tp, t)
	defer remove(filename)

	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err := fd.Delete()
	AssertSuccess(t, err)
	FileDoesNotExist(t, filename)
}

func TestUpdateFileFailsWhenLkFileExists(t *testing.T) {
	tp := NewTestPersistable()
	filename := tp.Filepath()
	create(filename, t)
	write(filename, tp, t)
	defer remove(filename)
	create(LockfileName(filename), t)
	defer remove(LockfileName(filename))

	tpr := &TestPersistable{
		Data: "Some new data here",
	}
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err := fd.Update(tpr)
	IsTrue(t, err != nil, "Update should fail since lk file exists")
}

func TestUpdateFileFailsWhenFileDoesNotExists(t *testing.T) {
	tpr := NewTestPersistable()
	filename := tpr.Filepath()
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err := fd.Update(tpr)
	IsTrue(t, err != nil, "Update should fail since target file does not exist")
}

func TestUpdateFileFailsWhenPersistableOutOfDate(t *testing.T) {
	tp := NewTestPersistable()
	filename := tp.Filepath()
	create(filename, t)
	write(filename, tp, t)
	defer remove(filename)

	tpr := &TestPersistable{
		Data: "Some new data here",
	}
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err := fd.Update(tpr)
	IsTrue(t, err != nil, "Update should fail since timestamp is out of date")
}

func TestUpdateFileSucceeds(t *testing.T) {
	tp := NewTestPersistable()
	filename := tp.Filepath()
	create(filename, t)
	write(filename, tp, t)
	defer remove(filename)

	tpr := &TestPersistable{}
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	fd.Read(tpr)
	stamp := tpr.Timestamp()
	time.Sleep(50 * time.Nanosecond)

	tpr.Data = "Some new data here"
	err := fd.Update(tpr)
	AssertSuccess(t, err)
	NotEqual(t, stamp, tpr.Timestamp(), "Update should have changed the timestamp")
}

func TestConcurrentUpdateToFileSucceedsOrFailsWithLockOrStaleErrors(t *testing.T) {
	tp := NewTestPersistable()
	filename := tp.Filepath()
	create(filename, t)
	write(filename, tp, t)
	defer remove(filename)

	hold := make(chan bool)
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	result := make(chan int)

	for i := 0; i < 100; i++ {
		go func() {
			<-hold
			r := rand.Intn(10)
			time.Sleep(time.Duration(r) * time.Nanosecond)
<commit_msg>Delay 0.5s before updating file.<commit_after>package srvr

import (
	"encoding/json"
	"fmt"
	"math/rand"
	"os"
	"strings"
	"sync"
	"testing"
	"time"
)

var tmpFileCount int
var randVal int64

var dirLock = &sync.RWMutex{}

func GenerateFileName() string {
	tmpFileCount++
	return fmt.Sprintf("tmp%d_%d.json", randVal, tmpFileCount)
}

type TestPersistable struct {
	TimestampHolder
	Data     string `json:"data"`
	Filename string `json:"-"`
}

func (tp *TestPersistable) Filepath() string {
	return tp.Filename
}

func LockfileName(filename string) string {
	return filename + ".lk"
}

func NewTestPersistable() *TestPersistable {
	tmpFileCount++
	if randVal == 0 {
		randVal = time.Now().UnixNano()
	}
	tp := &TestPersistable{
		Data:     "Some information here",
		Filename: GenerateFileName(),
	}
	return tp
}

func TestDelete(t *testing.T) {
	tp := NewTestPersistable()
	filename := tp.Filepath()
	create(filename, t)
	write(filename, tp, t)
	defer remove(filename)

	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err := fd.Delete()
	AssertSuccess(t, err)
	FileDoesNotExist(t, filename)
}

func TestUpdateFileFailsWhenLkFileExists(t *testing.T) {
	tp := NewTestPersistable()
	filename := tp.Filepath()
	create(filename, t)
	write(filename, tp, t)
	defer remove(filename)
	create(LockfileName(filename), t)
	defer remove(LockfileName(filename))

	tpr := &TestPersistable{
		Data: "Some new data here",
	}
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err := fd.Update(tpr)
	IsTrue(t, err != nil, "Update should fail since lk file exists")
}

func TestUpdateFileFailsWhenFileDoesNotExists(t *testing.T) {
	tpr := NewTestPersistable()
	filename := tpr.Filepath()
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err := fd.Update(tpr)
	IsTrue(t, err != nil, "Update should fail since target file does not exist")
}

func TestUpdateFileFailsWhenPersistableOutOfDate(t *testing.T) {
	tp := NewTestPersistable()
	filename := tp.Filepath()
	create(filename, t)
	write(filename, tp, t)
	defer remove(filename)

	tpr := &TestPersistable{
		Data: "Some new data here",
	}
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err := fd.Update(tpr)
	IsTrue(t, err != nil, "Update should fail since timestamp is out of date")
}

func TestUpdateFileSucceeds(t *testing.T) {
	tp := NewTestPersistable()
	filename := tp.Filepath()
	create(filename, t)
	write(filename, tp, t)
	defer remove(filename)

	tpr := &TestPersistable{}
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err := fd.Read(tpr)
	AssertSuccess(t, err)
	stamp := tpr.Timestamp()

	time.Sleep(500 * time.Millisecond)

	tpr.Data = "Some new data here"
	err = fd.Update(tpr)
	AssertSuccess(t, err)
	NotEqual(t, stamp, tpr.Timestamp(), "Update should have changed the timestamp")
}

func TestConcurrentUpdateToFileSucceedsOrFailsWithLockOrStaleErrors(t *testing.T) {
	tp := NewTestPersistable()
	filename := tp.Filepath()
	create(filename, t)
	write(filename, tp, t)
	defer remove(filename)

	hold := make(chan bool)
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	result := make(chan int)

	for i := 0; i < 100; i++ {
		go func() {
			<-hold
			r := rand.Intn(10)
			time.Sleep(time.Duration(r) * time.Nanosecond)

{'fraction_non_alphanumeric': 0.13695685551972978, 'fraction_numerical': 0.003838476892369108, 'mean_word_length': 4.601031814273431, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3067691', 'n_tokens_mistral': 2498, 'n_tokens_neox': 2436, 'n_words': 591}