<commit_before>package sql_test

import (
	"database/sql"
	"fmt"
	"os"
	"sync"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"

	_ "github.com/mattn/go-sqlite3"
	"github.com/stretchr/testify/require"
	sqln "nimona.io/internal/store/sql"
	"nimona.io/pkg/errors"
	"nimona.io/pkg/hash"
	"nimona.io/pkg/object"
	"nimona.io/pkg/stream"
)

const dbFilepath string = "./nimona.db"

func TestNewDatabase(t *testing.T) {
	dblite, err := sql.Open("sqlite3", dbFilepath)
	defer func() {
		os.Remove(dbFilepath) // nolint
	}()
	require.NoError(t, err)

	store, err := sqln.New(dblite)
	require.NoError(t, err)
	require.NotNil(t, store)

	err = store.Close()
	require.NoError(t, err)
}

func TestStoreRetrieveUpdate(t *testing.T) {
	dblite, err := sql.Open("sqlite3", dbFilepath)
	defer func() {
		os.Remove(dbFilepath) // nolint
	}()
	require.NoError(t, err)

	store, err := sqln.New(dblite)
	require.NoError(t, err)
	require.NotNil(t, store)

	p := stream.Created{
		Nonce: "asdf",
	}
	c := stream.PolicyAttached{
		Stream: hash.New(p.ToObject()),
	}
	obj := c.ToObject()
	obj.SetType("foo")
	obj.Set("key:s", "value")

	err = store.Put(
		obj,
		sqln.WithTTL(0),
	)
	require.NoError(t, err)

	err = store.Put(
		obj,
		sqln.WithTTL(10),
	)

	require.NoError(t, err)
	retrievedObj, err := store.Get(hash.New(obj))
	require.NoError(t, err)

	val := retrievedObj.Get("key:s")
	require.NotNil(t, val)
	assert.Equal(t, "value", val.(string))

	stHash := stream.Stream(obj)
	require.NotEmpty(t, stHash)

	err = store.UpdateTTL(hash.New(obj), 10)
	require.NoError(t, err)

	hashList, err := store.GetRelations(hash.New(p.ToObject()))
	require.NoError(t, err)
	assert.NotEmpty(t, hashList)

	err = store.Remove(hash.New(p.ToObject()))
	require.NoError(t, err)

	retrievedObj2, err := store.Get(hash.New(p.ToObject()))
	require.True(t, errors.CausedBy(err, sqln.ErrNotFound))
	require.Nil(t, retrievedObj2)

	err = store.Close()
	require.NoError(t, err)
}

func TestSubscribe(t *testing.T) {
	// create db
	dblite, err := sql.Open("sqlite3", dbFilepath)
	defer func() {
		os.Remove(dbFilepath) // nolint
	}()
	require.NoError(t, err)

	store, err := sqln.New(dblite)
	require.NoError(t, err)
	require.NotNil(t, store)

	// setup data
	p := stream.Created{
		Nonce: "asdf",
	}
	streamHash := hash.New(p.ToObject())
	c := stream.PolicyAttached{
		Stream: streamHash,
	}
	obj := c.ToObject()
	obj.SetType("foo")
	obj.Set("key:s", "value")

	var wg sync.WaitGroup

	for i := 1; i <= 5; i++ {
		wg.Add(1)
		// subscribe
		subscription := store.Subscribe(
			sqln.FilterByStreamHash(streamHash),
		)

		go func() {
			hs, err := subscription.Next()
			require.NoError(t, err)
			require.NotEmpty(t, hs)
			wg.Done()
		}()
	}

	// store data
	err = store.Put(
		obj,
		sqln.WithTTL(10),
	)
	require.NoError(t, err)

	done := make(chan bool)
	go func() {
		wg.Wait()
		done <- true
	}()

	select {
	case <-done:
		break
	case <-time.After(1 * time.Second):
		t.Fatalf("failed to get update")
	}
}

func TestAll(t *testing.T) {
	dblite, err := sql.Open("sqlite3", dbFilepath)
	defer func() {
		os.Remove(dbFilepath) // nolint
	}()
	require.NoError(t, err)

	store, err := sqln.New(dblite)
	require.NoError(t, err)
	require.NotNil(t, store)

	p := stream.Created{
		Nonce: "asdf",
	}
	c := stream.PolicyAttached{
		Stream: hash.New(p.ToObject()),
	}

	hashes := []object.Hash{}

	for i := 0; i < 5; i++ {
		obj := c.ToObject()
		obj.Set("key:s", fmt.Sprintf("value_%d", i))

		err = store.Put(
			obj,
			sqln.WithTTL(0),
		)
		require.NoError(t, err)

		hashes = append(hashes, hash.New(obj))
	}

	objects, err := store.All(hashes...)
	require.NoError(t, err)
	require.Len(t, objects, len(hashes))

	err = store.Close()
	require.NoError(t, err)
}<commit_msg>refactor(store/sql): use fixtures for tests, use new file on every test to fix io disk errors in github actions<commit_after>package sql_test

import (
	ssql "database/sql"
	"fmt"
	"io/ioutil"
	"path"
	"sync"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"nimona.io/internal/fixtures"
	"nimona.io/internal/store/sql"
	"nimona.io/pkg/errors"
	"nimona.io/pkg/hash"
	"nimona.io/pkg/object"
	"nimona.io/pkg/stream"

	_ "github.com/mattn/go-sqlite3"
)

func tempSqlite3(t *testing.T) *ssql.DB {
	dirPath, err := ioutil.TempDir("", "nimona-store-sql")
	require.NoError(t, err)
	db, err := ssql.Open("sqlite3", path.Join(dirPath, "sqlite3.db"))
	require.NoError(t, err)
	return db
}

func TestNewDatabase(t *testing.T) {
	dblite := tempSqlite3(t)
	store, err := sql.New(dblite)
	require.NoError(t, err)
	require.NotNil(t, store)

	err = store.Close()
	require.NoError(t, err)
}

func TestStoreRetrieveUpdate(t *testing.T) {
	dblite := tempSqlite3(t)
	store, err := sql.New(dblite)
	require.NoError(t, err)
	require.NotNil(t, store)

	p := fixtures.TestStream{
		Nonce: "asdf",
	}
	c := fixtures.TestSubscribed{
		Stream: hash.New(p.ToObject()),
	}
	obj := c.ToObject()
	obj.SetType("foo")
	obj.Set("key:s", "value")

	err = store.Put(
		obj,
		sql.WithTTL(0),
	)
	require.NoError(t, err)

	err = store.Put(
		obj,
		sql.WithTTL(10),
	)

	require.NoError(t, err)
	retrievedObj, err := store.Get(hash.New(obj))
	require.NoError(t, err)

	val := retrievedObj.Get("key:s")
	require.NotNil(t, val)
	assert.Equal(t, "value", val.(string))

	stHash := stream.Stream(obj)
	require.NotEmpty(t, stHash)

	err = store.UpdateTTL(hash.New(obj), 10)
	require.NoError(t, err)

	hashList, err := store.GetRelations(hash.New(p.ToObject()))
	require.NoError(t, err)
	assert.NotEmpty(t, hashList)

	err = store.Remove(hash.New(p.ToObject()))
	require.NoError(t, err)

	retrievedObj2, err := store.Get(hash.New(p.ToObject()))
	require.True(t, errors.CausedBy(err, sql.ErrNotFound))
	require.Nil(t, retrievedObj2)

	err = store.Close()
	require.NoError(t, err)
}

func TestSubscribe(t *testing.T) {
	// create db
	dblite := tempSqlite3(t)
	store, err := sql.New(dblite)
	require.NoError(t, err)
	require.NotNil(t, store)

	// setup data
	p := fixtures.TestStream{
		Nonce: "asdf",
	}
	streamHash := hash.New(p.ToObject())
	c := fixtures.TestSubscribed{
		Stream: streamHash,
	}
	obj := c.ToObject()
	obj.SetType("foo")
	obj.Set("key:s", "value")

	var wg sync.WaitGroup

	for i := 1; i <= 5; i++ {
		wg.Add(1)
		// subscribe
		subscription := store.Subscribe(
			sql.FilterByStreamHash(streamHash),
		)

		go func() {
			hs, err := subscription.Next()
			require.NoError(t, err)
			require.NotEmpty(t, hs)
			wg.Done()
		}()
	}

	// store data
	err = store.Put(
		obj,
		sql.WithTTL(10),
	)
	require.NoError(t, err)

	done := make(chan bool)
	go func() {
		wg.Wait()
		done <- true
	}()

	select {
	case <-done:
		break
	case <-time.After(1 * time.Second):
		t.Fatalf("failed to get update")
	}
}

func TestAll(t *testing.T) {
	dblite := tempSqlite3(t)
	store, err := sql.New(dblite)
	require.NoError(t, err)
	require.NotNil(t, store)

	p := fixtures.TestStream{
		Nonce: "asdf",
	}
	c := fixtures.TestSubscribed{
		Stream: hash.New(p.ToObject()),
	}

	hashes := []object.Hash{}

	for i := 0; i < 5; i++ {
		obj := c.ToObject()
		obj.Set("key:s", fmt.Sprintf("value_%d", i))

		err = store.Put(
			obj,
			sql.WithTTL(0),
		)
		require.NoError(t, err)

		hashes = append(hashes, hash.New(obj))
	}

	objects, err := store.All(hashes...)
	require.NoError(t, err)
	require.Len(t, objects, len(hashes))

	err = store.Close()
	require.NoError(t, err)
}
{'fraction_non_alphanumeric': 0.18240690771721532, 'fraction_numerical': 0.006071235833783054, 'mean_word_length': 3.8419333768778574, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6629094', 'n_tokens_mistral': 3318, 'n_tokens_neox': 3132, 'n_words': 577}