<commit_before>package main

import (
	"bytes"
	"context"
	"encoding/hex"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"strconv"
	"sync/atomic"
	"testing"
)

const NoValidURLSInTest = "no valid urls"

const NonExistingURLInTest = "http://some.non.existing.domain.at.nowhere/pesho"

func parseRangeInTest(s string) (start, end int) {
	fmt.Sscanf(s, "bytes=%d-%d", &start, &end)
	return
}

func responseRangeHeaderValueInTest(start, end, size int) string {
	return fmt.Sprintf("bytes %d-%d/%d", start, end, size)
}

func checkResponseInTest(t *testing.T, expected, got []byte) {
	if !bytes.Equal(expected, got) {
		t.Errorf("Expected result was '%s' but got '%s'", hex.EncodeToString(expected), hex.EncodeToString(got))
	}
}

// Test nothing to return
func TestNothingIsReturned(t *testing.T) {
	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
	}))
	defer s.Close()

	r := DownloadFile(context.TODO(), []string{s.URL + "/pesho"})
	var buf [512]byte
	n, err := io.ReadFull(r, buf[:])
	if n != 0 {
		t.Errorf("Expected to read 0 bytes from empty download but got %d", n)
	}

	if err != io.EOF {
		t.Errorf("Expected to get error '%s', but got '%s'", io.EOF, err)
	}
}

// Test simple case with one url<commit_msg>task04: Return 0 Content-Length in empty download case<commit_after>package main

import (
	"bytes"
	"context"
	"encoding/hex"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"strconv"
	"sync/atomic"
	"testing"
)

const NoValidURLSInTest = "no valid urls"

const NonExistingURLInTest = "http://some.non.existing.domain.at.nowhere/pesho"

func parseRangeInTest(s string) (start, end int) {
	fmt.Sscanf(s, "bytes=%d-%d", &start, &end)
	return
}

func responseRangeHeaderValueInTest(start, end, size int) string {
	return fmt.Sprintf("bytes %d-%d/%d", start, end, size)
}

func checkResponseInTest(t *testing.T, expected, got []byte) {
	if !bytes.Equal(expected, got) {
		t.Errorf("Expected result was '%s' but got '%s'", hex.EncodeToString(expected), hex.EncodeToString(got))
	}
}

// Test nothing to return
func TestNothingIsReturned(t *testing.T) {
	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
		w.Header().Add("Content-Length", "0")
		w.WriteHeader(200)
	}))
	defer s.Close()

	r := DownloadFile(context.TODO(), []string{s.URL + "/pesho"})
	var buf [512]byte
	n, err := io.ReadFull(r, buf[:])
	if n != 0 {
		t.Errorf("Expected to read 0 bytes from empty download but got %d", n)
	}

	if err != io.EOF {
		t.Errorf("Expected to get error '%s', but got '%s'", io.EOF, err)
	}
}

// Test simple case with one url
{'fraction_non_alphanumeric': 0.16781874039938557, 'fraction_numerical': 0.006528417818740399, 'mean_word_length': 4.427083333333333, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6480036', 'n_tokens_mistral': 1002, 'n_tokens_neox': 979, 'n_words': 249}