<commit_before>// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package queue

import (
	"context"
	"io/ioutil"
	"os"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func TestLevelQueue(t *testing.T) {
	handleChan := make(chan *testData)
	handle := func(data ...Data) {
		assert.True(t, len(data) == 2)
		for _, datum := range data {
			testDatum := datum.(*testData)
			handleChan <- testDatum
		}
	}

	queueShutdown := []func(){}
	queueTerminate := []func(){}

	tmpDir, err := ioutil.TempDir("", "level-queue-test-data")
	assert.NoError(t, err)
	defer os.RemoveAll(tmpDir)

	queue, err := NewLevelQueue(handle, LevelQueueConfiguration{
		WorkerPoolConfiguration: WorkerPoolConfiguration{
			QueueLength:  20,
			BatchLength:  2,
			BlockTimeout: 1 * time.Second,
			BoostTimeout: 5 * time.Minute,
			BoostWorkers: 5,
			MaxWorkers:   10,
		},
		DataDir: tmpDir,
		Workers: 1,
	}, &testData{})
	assert.NoError(t, err)

	go queue.Run(func(_ context.Context, shutdown func()) {
		queueShutdown = append(queueShutdown, shutdown)
	}, func(_ context.Context, terminate func()) {
		queueTerminate = append(queueTerminate, terminate)
	})

	test1 := testData{"A", 1}
	test2 := testData{"B", 2}

	err = queue.Push(&test1)
	assert.NoError(t, err)
	go func() {
		err = queue.Push(&test2)
		assert.NoError(t, err)
	}()

	result1 := <-handleChan
	assert.Equal(t, test1.TestString, result1.TestString)
	assert.Equal(t, test1.TestInt, result1.TestInt)

	result2 := <-handleChan
	assert.Equal(t, test2.TestString, result2.TestString)
	assert.Equal(t, test2.TestInt, result2.TestInt)

	err = queue.Push(test1)
	assert.Error(t, err)

	for _, callback := range queueShutdown {
		callback()
	}
	time.Sleep(200 * time.Millisecond)
	err = queue.Push(&test1)
	assert.NoError(t, err)
	err = queue.Push(&test2)
	assert.NoError(t, err)
	select {
	case <-handleChan:
		assert.Fail(t, "Handler processing should have stopped")
	default:
	}
	for _, callback := range queueTerminate {
		callback()
	}

	// Reopen queue
	queue, err = NewWrappedQueue(handle,
		WrappedQueueConfiguration{
			Underlying: LevelQueueType,
			Config: LevelQueueConfiguration{
				WorkerPoolConfiguration: WorkerPoolConfiguration{
					QueueLength:  20,
					BatchLength:  2,
					BlockTimeout: 1 * time.Second,
					BoostTimeout: 5 * time.Minute,
					BoostWorkers: 5,
					MaxWorkers:   10,
				},
				DataDir: tmpDir,
				Workers: 1,
			},
		}, &testData{})
	assert.NoError(t, err)

	go queue.Run(func(_ context.Context, shutdown func()) {
		queueShutdown = append(queueShutdown, shutdown)
	}, func(_ context.Context, terminate func()) {
		queueTerminate = append(queueTerminate, terminate)
	})

	result3 := <-handleChan
	assert.Equal(t, test1.TestString, result3.TestString)
	assert.Equal(t, test1.TestInt, result3.TestInt)

	result4 := <-handleChan
	assert.Equal(t, test2.TestString, result4.TestString)
	assert.Equal(t, test2.TestInt, result4.TestInt)
	for _, callback := range queueShutdown {
		callback()
	}
	for _, callback := range queueTerminate {
		callback()
	}
}<commit_msg>Fix leveldb test race (#10054)<commit_after>// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package queue

import (
	"context"
	"io/ioutil"
	"os"
	"sync"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func TestLevelQueue(t *testing.T) {
	handleChan := make(chan *testData)
	handle := func(data ...Data) {
		assert.True(t, len(data) == 2)
		for _, datum := range data {
			testDatum := datum.(*testData)
			handleChan <- testDatum
		}
	}

	var lock sync.Mutex
	queueShutdown := []func(){}
	queueTerminate := []func(){}

	tmpDir, err := ioutil.TempDir("", "level-queue-test-data")
	assert.NoError(t, err)
	defer os.RemoveAll(tmpDir)

	queue, err := NewLevelQueue(handle, LevelQueueConfiguration{
		WorkerPoolConfiguration: WorkerPoolConfiguration{
			QueueLength:  20,
			BatchLength:  2,
			BlockTimeout: 1 * time.Second,
			BoostTimeout: 5 * time.Minute,
			BoostWorkers: 5,
			MaxWorkers:   10,
		},
		DataDir: tmpDir,
		Workers: 1,
	}, &testData{})
	assert.NoError(t, err)

	go queue.Run(func(_ context.Context, shutdown func()) {
		lock.Lock()
		queueShutdown = append(queueShutdown, shutdown)
		lock.Unlock()
	}, func(_ context.Context, terminate func()) {
		lock.Lock()
		queueTerminate = append(queueTerminate, terminate)
		lock.Unlock()
	})

	test1 := testData{"A", 1}
	test2 := testData{"B", 2}

	err = queue.Push(&test1)
	assert.NoError(t, err)
	go func() {
		err = queue.Push(&test2)
		assert.NoError(t, err)
	}()

	result1 := <-handleChan
	assert.Equal(t, test1.TestString, result1.TestString)
	assert.Equal(t, test1.TestInt, result1.TestInt)

	result2 := <-handleChan
	assert.Equal(t, test2.TestString, result2.TestString)
	assert.Equal(t, test2.TestInt, result2.TestInt)

	err = queue.Push(test1)
	assert.Error(t, err)

	lock.Lock()
	for _, callback := range queueShutdown {
		callback()
	}
	lock.Unlock()

	time.Sleep(200 * time.Millisecond)
	err = queue.Push(&test1)
	assert.NoError(t, err)
	err = queue.Push(&test2)
	assert.NoError(t, err)
	select {
	case <-handleChan:
		assert.Fail(t, "Handler processing should have stopped")
	default:
	}
	lock.Lock()
	for _, callback := range queueTerminate {
		callback()
	}
	lock.Unlock()

	// Reopen queue
	queue, err = NewWrappedQueue(handle,
		WrappedQueueConfiguration{
			Underlying: LevelQueueType,
			Config: LevelQueueConfiguration{
				WorkerPoolConfiguration: WorkerPoolConfiguration{
					QueueLength:  20,
					BatchLength:  2,
					BlockTimeout: 1 * time.Second,
					BoostTimeout: 5 * time.Minute,
					BoostWorkers: 5,
					MaxWorkers:   10,
				},
				DataDir: tmpDir,
				Workers: 1,
			},
		}, &testData{})
	assert.NoError(t, err)

	go queue.Run(func(_ context.Context, shutdown func()) {
		lock.Lock()
		queueShutdown = append(queueShutdown, shutdown)
		lock.Unlock()
	}, func(_ context.Context, terminate func()) {
		lock.Lock()
		queueTerminate = append(queueTerminate, terminate)
		lock.Unlock()
	})

	result3 := <-handleChan
	assert.Equal(t, test1.TestString, result3.TestString)
	assert.Equal(t, test1.TestInt, result3.TestInt)

	result4 := <-handleChan
	assert.Equal(t, test2.TestString, result4.TestString)
	assert.Equal(t, test2.TestInt, result4.TestInt)

	lock.Lock()
	for _, callback := range queueShutdown {
		callback()
	}
	for _, callback := range queueTerminate {
		callback()
	}
	lock.Unlock()
}
{'fraction_non_alphanumeric': 0.13929440389294404, 'fraction_numerical': 0.017487834549878345, 'mean_word_length': 4.059230769230769, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2741255', 'n_tokens_mistral': 2601, 'n_tokens_neox': 2315, 'n_words': 524}