<commit_before>package lily

import (
	"fmt"
	"io/ioutil"
	"os"
	"testing"
)

//
// Author <NAME>.
//
// <EMAIL>
//

func TestLineIterator(t *testing.T) {
	tmp := "/tmp/line_iterator.txt"
	content := "line1\n1\n"
	//defer os.Remove(tmp)
	err := ioutil.WriteFile(tmp, []byte(content), 0644)
	if err != nil {
		fmt.Printf("Tmp file couldn't be writen becauser error %s", err.Error())
		os.Exit(1)
	}
	iter, err := NewLineIterator(tmp)
	switch {
	case err != nil:
		t.Error(err.Error())
	case iter.Next() != "line1" && iter.HasNext():
		t.("Line 1 is not expected(line1).")
	case iter.Next() != "1" && iter.HasNext():
		t.Error("Line 2 is not expected(1).")
	case iter.Next() != "" && !iter.HasNext():
		t.Error("Line 3 is not expected().")
	case iter.Next() != "":
		t.Error("Line 4 is not expected at all.")
	}

}

func TestGenerateBase64StringSize(t *testing.T) {
	for i, n := range []int{0, 1, 2, 5, 10, 25, 30, 50, 75, 100} {
		hex := GenerateBase64String(n)
		if len(hex) != n {
			t.Errorf("%d - Size of GenerateBase64String(%d) sould be %d, got %d instead.", i, n, n, len(hex))
			return
		}
	}
}

func BGenerateBase64StringN(n int, b *testing.B) {
	for i := 0; i < b.N; i++ {
		GenerateBase64String(n)
	}
}

func BenchmarkGenerateBase64String1(b *testing.B) {
	BGenerateBase64StringN(1, b)
}<commit_msg>Fix util bug<commit_after>package lily

import (
	"fmt"
	"io/ioutil"
	"os"
	"testing"
)

//
// Author <NAME>.
//
// <EMAIL>
//

func TestLineIterator(t *testing.T) {
	tmp := "/tmp/line_iterator.txt"
	content := "line1\n1\n"
	//defer os.Remove(tmp)
	err := ioutil.WriteFile(tmp, []byte(content), 0644)
	if err != nil {
		fmt.Printf("Tmp file couldn't be writen becauser error %s", err.Error())
		os.Exit(1)
	}
	iter, err := NewLineIterator(tmp)
	switch {
	case err != nil:
		t.Error(err.Error())
	case iter.Next() != "line1" && iter.HasNext():
		t.Error("Line 1 is not expected(line1).")
	case iter.Next() != "1" && iter.HasNext():
		t.Error("Line 2 is not expected(1).")
	case iter.Next() != "" && !iter.HasNext():
		t.Error("Line 3 is not expected().")
	case iter.Next() != "":
		t.Error("Line 4 is not expected at all.")
	}
}

func TestGenerateBase64StringSize(t *testing.T) {
	for i, n := range []int{0, 1, 2, 5, 10, 25, 30, 50, 75, 100} {
		hex := GenerateBase64String(n)
		if len(hex) != n {
			t.Errorf("%d - Size of GenerateBase64String(%d) sould be %d, got %d instead.", i, n, n, len(hex))
			return
		}
	}
}

func BGenerateBase64StringN(n int, b *testing.B) {
	for i := 0; i < b.N; i++ {
		GenerateBase64String(n)
	}
}

func BenchmarkGenerateBase64String1(b *testing.B) {
	BGenerateBase64StringN(1, b)
}
{'fraction_non_alphanumeric': 0.19687142312094622, 'fraction_numerical': 0.037390309042350245, 'mean_word_length': 3.5839160839160837, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1248064', 'n_tokens_mistral': 1165, 'n_tokens_neox': 1069, 'n_words': 268}