<commit_before>package pool

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"net/url"
	"testing"
	"time"
)

var (
	numWorkers int64 = 4
	t10ms            = time.Duration(10) * time.Millisecond
)

func testHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprint(w, "Test page")
}

func testHandlerWithTimeout(w http.ResponseWriter, r *http.Request) {
	time.Sleep(t10ms)
	fmt.Fprint(w, "Test page with timeout")
}

func TestQueue(t *testing.T) {
	queue := new(taskQueue)
	testTask := &Task{
		Hostname: "",
	}
	queue.put(testTask)
	if queue.len != 1 {
		t.Errorf("Got %v error, want %v", queue.len, 1)
	}
	queue.put(testTask)
	_, ok := queue.get()
	if !ok {
		t.Errorf("Got %v error, want %v", ok, true)
	}
}

// func TestClosedInputTaskChan(t *testing.T) {
// 	ts := httptest.NewServer(http.HandlerFunc(testHandlerWithTimeout))
// 	defer ts.Close()

// 	p := New(1)
// 	p.SetTimeout(100)
// 	_ = p.Add(ts.URL, nil)
// 	// close(p.inputTaskChan)
// 	err := p.Add(ts.URL, nil)
// 	if err == nil {
// 		t.Errorf("Got %v error, want %v", nil, "error")
// 	}
// 	log.Println(err)
// }

func TestNoServer(t *testing.T) {
	p := New(numWorkers)
	if p.numWorkers != numWorkers {
		t.Errorf("Got %v numWorkers, want %v", p.numWorkers, numWorkers)
	}
	err := p.Add("", nil)
	if err != errEmptyTarget {
		t.Errorf("Got %v error, want %v", err, errEmptyTarget)
	}
	err = p.Add(":", nil)
	if err == nil {
		t.Errorf("Got %v error, want hostname error", err)
	}
	if !p.poolIsRunning() {
		t.Errorf("Got %v error, want true", p.poolIsRunning())
	}
	err = p.Add("http://127.0.0.1:80/", nil)
	if err != nil {
		t.Errorf("Got %v error, want nil error", err)
	}
	task := <-p.ResultChan
	if task.Error == nil {
		t.Error("Got nil error, want net error")
	}
	if p.GetAddedTasks() != 1 {
		t.Errorf("Wrong input jobs. Want 1, got %v", p.GetAddedTasks())
	}
	if p.getFreeWorkers() != numWorkers {
		t.Errorf("Wrong free workers. Want %v, got %v", numWorkers, p.getFreeWorkers())
	}
	proxy, _ := url.Parse("http://127.0.0.1:80/")
	err = p.Add("http://127.0.0.1:80/", proxy)
	if err != nil {
		t.Errorf("Got %v error, want nil error", err)
	}
	task = <-p.ResultChan
	if task.Error == nil {
		t.Error("Got nil error, want net error")
	}
	p.Quit()
	if p.poolIsRunning() {
		t.Errorf("Got %v error, want false", p.poolIsRunning())
	}
}

func TestWithServer(t *testing.T) {
	ts := httptest.NewServer(http.HandlerFunc(testHandler))
	defer ts.Close()

	p := New(numWorkers)
	_ = p.Add(ts.URL, nil)
	task, ok := <-p.ResultChan
	if !ok {
		t.Error("Channel is closed'")
	}
	if string(task.Body) != "Test page" {
		t.Errorf("Got %v error, want 'Test page'", string(task.Body))
	}
	p.EndWaitingTasks()
	err := p.Add(ts.URL, nil)
	if err != errNotWait {
		t.Errorf("Got %v error, want %v", err, errNotWait)
	}
	if !p.isCompleteJobs() {
		t.Errorf("Got %v error, want %v", false, true)<commit_msg>more test<commit_after>package pool

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"net/url"
	"testing"
	"time"
)

var (
	numWorkers int64 = 4
	t10ms            = time.Duration(10) * time.Millisecond
)

func testHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprint(w, "Test page")
}

func testHandlerWithTimeout(w http.ResponseWriter, r *http.Request) {
	time.Sleep(t10ms)
	fmt.Fprint(w, "Test page with timeout")
}

func TestQueue(t *testing.T) {
	queue := new(taskQueue)
	testTask := &Task{
		Hostname: "",
	}
	queue.put(testTask)
	if queue.len != 1 {
		t.Errorf("Got %v error, want %v", queue.len, 1)
	}
	queue.put(testTask)
	_, ok := queue.get()
	if !ok {
		t.Errorf("Got %v error, want %v", ok, true)
	}
}

// func TestClosedInputTaskChan(t *testing.T) {
// 	ts := httptest.NewServer(http.HandlerFunc(testHandlerWithTimeout))
// 	defer ts.Close()

// 	p := New(1)
// 	p.SetTimeout(100)
// 	_ = p.Add(ts.URL, nil)
// 	// close(p.inputTaskChan)
// 	err := p.Add(ts.URL, nil)
// 	if err == nil {
// 		t.Errorf("Got %v error, want %v", nil, "error")
// 	}
// 	log.Println(err)
// }

func TestNoServer(t *testing.T) {
	p := New(numWorkers)
	if p.numWorkers != numWorkers {
		t.Errorf("Got %v numWorkers, want %v", p.numWorkers, numWorkers)
	}
	err := p.Add("", nil)
	if err != errEmptyTarget {
		t.Errorf("Got %v error, want %v", err, errEmptyTarget)
	}
	err = p.Add(":", nil)
	if err == nil {
		t.Errorf("Got %v error, want hostname error", err)
	}
	if !p.poolIsRunning() {
		t.Errorf("Got %v error, want true", p.poolIsRunning())
	}
	err = p.Add("http://127.0.0.1:80/", nil)
	if err != nil {
		t.Errorf("Got %v error, want nil error", err)
	}
	task := <-p.ResultChan
	if task.Error == nil {
		t.Error("Got nil error, want net error")
	}
	if p.GetAddedTasks() != 1 {
		t.Errorf("Wrong input jobs. Want 1, got %v", p.GetAddedTasks())
	}
	if p.getFreeWorkers() != numWorkers {
		t.Errorf("Wrong free workers. Want %v, got %v", numWorkers, p.getFreeWorkers())
	}
	proxy, _ := url.Parse("http://127.0.0.1:80/")
	err = p.Add("http://127.0.0.1:80/", proxy)
	if err != nil {
		t.Errorf("Got %v error, want nil error", err)
	}
	task = <-p.ResultChan
	if task.Error == nil {
		t.Error("Got nil error, want net error")
	}
	p.Quit()
	if p.poolIsRunning() {
		t.Errorf("Got %v error, want false", p.poolIsRunning())
	}
}

func TestWithServer(t *testing.T) {
	ts := httptest.NewServer(http.HandlerFunc(testHandler))
	defer ts.Close()

	p := New(numWorkers)
	_ = p.Add(ts.URL, nil)
	task, ok := <-p.ResultChan
	if !ok {
		t.Error("Channel is closed'")
	}
	if string(task.Body) != "Test page" {
		t.Errorf("Got %v error, want 'Test page'", string(task.Body))
	}
	p.EndWaitingTasks()
	err := p.Add(ts.URL, nil)
	if err != errNotWait {
		t.Errorf("Got %v error, want %v", err, errNotWait)
	}
	if p.GetCompletedTasks() == 0 {
		t.Errorf("Got %v error, want %v", 0, 1)
	}
	if !p.isCompleteJobs() {
		t.Errorf("Got %v error, want %v", false, true)
{'fraction_non_alphanumeric': 0.1940426997245179, 'fraction_numerical': 0.014634986225895317, 'mean_word_length': 3.397426192278577, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 6, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1817665', 'n_tokens_mistral': 2559, 'n_tokens_neox': 2426, 'n_words': 564}