<commit_before>// TODO(dev) add test overview
//
package task_test

import (
	"archive/tar"
	"bytes"
	"errors"
	"fmt"
	"reflect"
	"testing"

	"cloud.google.com/go/bigquery"

	"time"

	"github.com/m-lab/etl/parser"
	"github.com/m-lab/etl/storage" // TODO - would be better not to have this.
	"github.com/m-lab/etl/task"
)

// Just test call to NullParser.Parse
func TestPlumbing(t *testing.T) {
	foo := [10]byte{1, 2, 3, 4, 5, 1, 2, 3, 4, 5}
	p := parser.NullParser{}
	err := p.ParseAndInsert(nil, "foo", foo[:])
	if err != nil {
		fmt.Println(err)
	}
}

type NullCloser struct{}

func (nc NullCloser) Close() error {
	return nil
}

// Create a TarReader with simple test contents.
// TODO - could we break the dependency on storage here?
func MakeTestSource(t *testing.T) *storage.ETLSource {
	b := new(bytes.Buffer)
	tw := tar.NewWriter(b)
	hdr := tar.Header{Name: "foo", Mode: 0666, Typeflag: tar.TypeReg, Size: int64(8)}
	tw.WriteHeader(&hdr)
	_, err := tw.Write([]byte("biscuits"))
	if err != nil {
		t.Fatal(err)
	}

	// Put a large file in the middle to test skipping.
	hdr = tar.Header{Name: "big_file", Mode: 0666, Typeflag: tar.TypeReg, Size: int64(101)}
	tw.WriteHeader(&hdr)
	_, err = tw.Write(make([]byte, 101))
	if err != nil {
		t.Fatal(err)
	}

	hdr = tar.Header{Name: "bar", Mode: 0666, Typeflag: tar.TypeReg, Size: int64(11)}
	tw.WriteHeader(&hdr)
	_, err = tw.Write([]byte("butter milk"))
	if err = tw.Close(); err != nil {
		t.Fatal(err)
	}

	return &storage.ETLSource{TarReader: tar.NewReader(b), Closer: NullCloser{}, RetryBaseTime: time.Millisecond}
}

type TestParser struct {
	parser.FakeRowStats
	files []string
}

func (tp *TestParser) TableName() string {
	return "test-table"
}
func (tp *TestParser) FullTableName() string {
	return "test-table"
}
func (tp *TestParser) Flush() error {
	return nil
}

// TODO - pass testName through to BQ inserter?
func (tp *TestParser) ParseAndInsert(meta map[string]bigquery.Value, testName string, test []byte) error {
	tp.files = append(tp.files, testName)
	return nil
}

type badSource struct{}

func (bs *badSource) Next() (*tar.Header, error) {
	return nil, errors.New("Random Error")
}
func (bs *badSource) Read(b []byte) (int, error) {
	return 0, errors.New("Read error")
}

// TODO - this test is very slow, because it triggers the backoff and retry mechanism.
func TestBadTarFileInput(t *testing.T) {
	rdr := &storage.ETLSource{TarReader: &badSource{}, Closer: NullCloser{}, RetryBaseTime: time.Millisecond}

	tp := &TestParser{}

	// Among other things, this requires that tp implements etl.Parser.
	tt := task.NewTask("filename", rdr, tp)
	fc, err := tt.ProcessAllTests()
	if err.Error() != "Random Error" {
		t.Error("Expected Random Error, but got " + err.Error())<commit_msg>Fix TestParser<commit_after>// TODO(dev) add test overview
//
package task_test

import (
	"archive/tar"
	"bytes"
	"errors"
	"fmt"
	"reflect"
	"testing"

	"cloud.google.com/go/bigquery"

	"time"

	"github.com/m-lab/etl/parser"
	"github.com/m-lab/etl/storage" // TODO - would be better not to have this.
	"github.com/m-lab/etl/task"
)

// Just test call to NullParser.Parse
func TestPlumbing(t *testing.T) {
	foo := [10]byte{1, 2, 3, 4, 5, 1, 2, 3, 4, 5}
	p := parser.NullParser{}
	err := p.ParseAndInsert(nil, "foo", foo[:])
	if err != nil {
		fmt.Println(err)
	}
}

type NullCloser struct{}

func (nc NullCloser) Close() error {
	return nil
}

// Create a TarReader with simple test contents.
// TODO - could we break the dependency on storage here?
func MakeTestSource(t *testing.T) *storage.ETLSource {
	b := new(bytes.Buffer)
	tw := tar.NewWriter(b)
	hdr := tar.Header{Name: "foo", Mode: 0666, Typeflag: tar.TypeReg, Size: int64(8)}
	tw.WriteHeader(&hdr)
	_, err := tw.Write([]byte("biscuits"))
	if err != nil {
		t.Fatal(err)
	}

	// Put a large file in the middle to test skipping.
	hdr = tar.Header{Name: "big_file", Mode: 0666, Typeflag: tar.TypeReg, Size: int64(101)}
	tw.WriteHeader(&hdr)
	_, err = tw.Write(make([]byte, 101))
	if err != nil {
		t.Fatal(err)
	}

	hdr = tar.Header{Name: "bar", Mode: 0666, Typeflag: tar.TypeReg, Size: int64(11)}
	tw.WriteHeader(&hdr)
	_, err = tw.Write([]byte("butter milk"))
	if err = tw.Close(); err != nil {
		t.Fatal(err)
	}

	return &storage.ETLSource{TarReader: tar.NewReader(b), Closer: NullCloser{}, RetryBaseTime: time.Millisecond}
}

type TestParser struct {
	parser.FakeRowStats
	files []string
}

func (tp *TestParser) TableName() string {
	return "test-table"
}
func (tp *TestParser) FullTableName() string {
	return "test-table"
}
func (tp *TestParser) Flush() error {
	return nil
}
func (tp *TestParser) TaskError() error {
	return nil
}

// TODO - pass testName through to BQ inserter?
func (tp *TestParser) ParseAndInsert(meta map[string]bigquery.Value, testName string, test []byte) error {
	tp.files = append(tp.files, testName)
	return nil
}

type badSource struct{}

func (bs *badSource) Next() (*tar.Header, error) {
	return nil, errors.New("Random Error")
}
func (bs *badSource) Read(b []byte) (int, error) {
	return 0, errors.New("Read error")
}

// TODO - this test is very slow, because it triggers the backoff and retry mechanism.
func TestBadTarFileInput(t *testing.T) {
	rdr := &storage.ETLSource{TarReader: &badSource{}, Closer: NullCloser{}, RetryBaseTime: time.Millisecond}

	tp := &TestParser{}

	// Among other things, this requires that tp implements etl.Parser.
	tt := task.NewTask("filename", rdr, tp)
	fc, err := tt.ProcessAllTests()
	if err.Error() != "Random Error" {
		t.Error("Expected Random Error, but got " + err.Error())
{'fraction_non_alphanumeric': 0.15944634190185153, 'fraction_numerical': 0.014380729822038469, 'mean_word_length': 4.190298507462686, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6465748', 'n_tokens_mistral': 2132, 'n_tokens_neox': 2041, 'n_words': 575}