<commit_before>// Copyright 2017 <NAME>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package felix

import (
	"bytes"
	"context"
	"errors"
	"io"
	"strings"
	"sync"
	"testing"
	"time"
)

type mockScanSource struct {
	sourceCalls int // number of times Source.Get was called
	scanCalls   int // number of times Scanner.Scan was called
}

func (ss *mockScanSource) Get(ctx context.Context, url string) (io.Reader, error) {
	ss.sourceCalls++

	switch ss.sourceCalls {
	case 1:
		return &bytes.Buffer{}, nil
	case 2:
		return nil, &tempNetError{"tempSourceError"}
	case 3:
		return nil, errors.New("otherSourceError")
	case 4, 5:
		return &bytes.Buffer{}, nil
	}

	return nil, errors.New("unexpectedError")
}

func (ss *mockScanSource) Scan(ctx context.Context, r io.Reader, e Emitter) error {
	ss.scanCalls++

	switch ss.scanCalls {
	case 1:
		e.EmitFollow("followurl1")
		e.EmitFollow("followurl2")
		e.EmitFollow("followurl3")
		e.EmitFollow("followurl4")
		return nil
	case 2:
		return errors.New("scanError")
	case 3:
		e.EmitItem(Item{Title: "emittedItem"})
		return nil
	}

	return errors.New("unexpectedError")
}

type tempNetError struct {
	msg string
}

func (e *tempNetError) Timeout() bool {
	panic("implement me")
}

func (e *tempNetError) Temporary() bool {
	return true
}

func (e *tempNetError) Error() string {
	return e.msg
}

func TestFetcher(t *testing.T) {
	items := make(chan Item)
	links := make(chan Link)

	scanSource := &mockScanSource{}

	nextFetch := func(n int) func() time.Duration {
		i := 0
		return func() time.Duration {
			defer func() { i++ }()
			if i < n {
				return 0
			} else {
				return 1000 * time.Hour
			}
		}
	}(1)

	logBuf := &bytes.Buffer{}
	log := NewLogger()<commit_msg>Fixed Fetcher tests<commit_after>// Copyright 2017 <NAME>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package felix

import (
	"bytes"
	"context"
	"errors"
	"io"
	"strings"
	"sync"
	"testing"
	"time"
)

type mockScanSource struct {
	sourceCalls int // number of times Source.Get was called
	scanCalls   int // number of times Scanner.Scan was called
}

func (ss *mockScanSource) Get(ctx context.Context, url string) (io.Reader, error) {
	ss.sourceCalls++

	switch ss.sourceCalls {
	case 1:
		return &bytes.Buffer{}, nil
	case 2:
		return nil, &tempNetError{"tempSourceError"}
	case 3:
		return nil, errors.New("otherSourceError")
	case 4, 5:
		return &bytes.Buffer{}, nil
	}

	return nil, errors.New("unexpectedError")
}

func (ss *mockScanSource) Scan(ctx context.Context, r io.Reader, e Emitter) error {
	ss.scanCalls++

	switch ss.scanCalls {
	case 1:
		e.EmitFollow("followurl1")
		e.EmitFollow("followurl2")
		e.EmitFollow("followurl3")
		e.EmitFollow("followurl4")
		return nil
	case 2:
		return errors.New("scanError")
	case 3:
		e.EmitItem(Item{Title: "emittedItem"})
		return nil
	}

	return errors.New("unexpectedError")
}

type tempNetError struct {
	msg string
}

func (e *tempNetError) Timeout() bool {
	panic("implement me")
}

func (e *tempNetError) Temporary() bool {
	return true
}

func (e *tempNetError) Error() string {
	return e.msg
}

func TestFetcher(t *testing.T) {
	items := make(chan Item)
	links := make(chan Link)

	scanSource := &mockScanSource{}

	nextFetch := func(n int) func(url string) (bool, time.Duration) {
		i := 0
		return func(url string) (bool, time.Duration) {
			defer func() { i++ }()
			if i < n {
				return true, 0
			} else {
				return true, 1000 * time.Hour
			}
		}
	}(1)

	logBuf := &bytes.Buffer{}
	log := NewLogger()
{'fraction_non_alphanumeric': 0.13362541073384446, 'fraction_numerical': 0.012595837897042717, 'mean_word_length': 3.7876802096985585, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3688246', 'n_tokens_mistral': 1411, 'n_tokens_neox': 1310, 'n_words': 385}