<commit_before>package fetch

import (
	"net/http"
	"testing"
	"time"

	"github.com/slotix/dataflowkit/splash"
	"github.com/slotix/dataflowkit/storage"
	"github.com/spf13/viper"
	"github.com/stretchr/testify/assert"
)

var (
	mw storageMiddleware
	s  storage.Store
)

func init() {
	var svc Service
	svc = FetchService{}
	s = storage.NewStore(storage.Diskv)
	mw = storageMiddleware{
		storage: s,
		Service: svc,
	}
}

func Test_storageMiddleware(t *testing.T) {
	var cookies []*http.Cookie
	req := splash.Request{
		URL:      "http://example.com",
		FormData: "", Cookies: cookies, LUA: "",
	}
	//Loading from remote server
	start := time.Now()
	resp, err := mw.Response(req)
	assert.Nil(t, err, "Expected no error")
	assert.Equal(t, 200, resp.(*splash.Response).Response.Status, "Expected Splash server returns 200 status code")
	elapsed1 := time.Since(start)
	t.Log("Loading from remote server... ", elapsed1)

	//Loading from cached storage
	start = time.Now()
	resp, err = mw.Response(req)
	assert.Nil(t, err, "Expected no error")
	assert.Equal(t, 200, resp.(*splash.Response).Response.Status, "Expected Splash server returns 200 status code")
	elapsed2 := time.Since(start)
	t.Log("Loading from remote server... ", elapsed2)
	assert.Equal(t, true, elapsed1 > elapsed2, "it takes longer to load a webpage from remote server")

	err = s.DeleteAll()
	assert.Nil(t, err, "Expected no error")
}

func Test_IGNORE_CACHE_INFO(t *testing.T) {
	viper.Set("IGNORE_CACHE_INFO", true)
	viper.Set("STORAGE_TYPE","Diskv")
	req := splash.Request{
		URL:    "http://google.com",
		FormData: "",  LUA: "",
		UserToken: "<PASSWORD>",
	}
	//Loading from remote server
	start := time.Now()
	resp, err := mw.Response(req)
	assert.Nil(t, err, "Expected no error")
	assert.Equal(t, 200, resp.(*splash.Response).Response.Status, "Expected Splash server returns 200 status code")
	elapsed1 := time.Since(start)
	t.Log("Loading from remote server... ", elapsed1)

	//Loading from cached storage
	start = time.Now()
	resp, err = mw.Response(req)
	assert.Nil(t, err, "Expected no error")
	assert.Equal(t, 200, resp.(*splash.Response).Response.Status, "Expected Splash server returns 200 status code")
	elapsed2 := time.Since(start)
	t.Log("Loading from remote server... ", elapsed2)
	assert.Equal(t, true, elapsed1 > elapsed2, "it takes longer to load a webpage from remote server")

	err = s.DeleteAll()
	assert.Nil(t, err, "Expected no error")
}<commit_msg>change fetcher type from splash to base<commit_after>package fetch

import (
	"testing"
	"time"

	"github.com/slotix/dataflowkit/storage"
	"github.com/spf13/viper"
	"github.com/stretchr/testify/assert"
)

var (
	mw storageMiddleware
	s  storage.Store
)

func init() {
	var svc Service
	svc = FetchService{}
	s = storage.NewStore(storage.Diskv)
	mw = storageMiddleware{
		storage: s,
		Service: svc,
	}
}

func Test_storageMiddleware(t *testing.T) {
	req := BaseFetcherRequest{
		//URL:      "http://example.com",
		URL:      "http://" + addr,
		FormData: "",
	}
	//Loading from remote server
	start := time.Now()
	resp, err := mw.Response(req)
	assert.Nil(t, err, "Expected no error")
	assert.Equal(t, 200, resp.(*BaseFetcherResponse).StatusCode, "Expected Fetcher returns 200 status code")
	elapsed1 := time.Since(start)
	t.Log("Loading from remote server... ", elapsed1)

	//Loading from cached storage
	start = time.Now()
	resp, err = mw.Response(req)
	assert.Nil(t, err, "Expected no error")
	assert.Equal(t, 200, resp.(*BaseFetcherResponse).StatusCode, "Expected Splash server returns 200 status code")
	elapsed2 := time.Since(start)
	t.Log("Loading from remote server... ", elapsed2)
	assert.Equal(t, true, elapsed1 > elapsed2, "it takes longer to load a webpage from remote server")

	err = s.DeleteAll()
	assert.Nil(t, err, "Expected no error")
}

func Test_IGNORE_CACHE_INFO(t *testing.T) {
	viper.Set("IGNORE_CACHE_INFO", true)
	viper.Set("STORAGE_TYPE", "Diskv")
	req := BaseFetcherRequest{
		//URL:      "http://google.com",
		URL:       "http://" + addr,
		FormData:  "",
		UserToken: "<PASSWORD>",
	}
	//Loading from remote server
	start := time.Now()
	resp, err := mw.Response(req)
	assert.Nil(t, err, "Expected no error")
	assert.Equal(t, 200, resp.(*BaseFetcherResponse).StatusCode, "Expected Splash server returns 200 status code")
	elapsed1 := time.Since(start)
	t.Log("Loading from remote server... ", elapsed1)

	//Loading from cached storage
	start = time.Now()
	resp, err = mw.Response(req)
	assert.Nil(t, err, "Expected no error")
	assert.Equal(t, 200, resp.(*BaseFetcherResponse).StatusCode, "Expected Splash server returns 200 status code")
	elapsed2 := time.Since(start)
	t.Log("Loading from remote server... ", elapsed2)
	assert.Equal(t, true, elapsed1 > elapsed2, "it takes longer to load a webpage from remote server")

	err = s.DeleteAll()
	assert.Nil(t, err, "Expected no error")
}
{'fraction_non_alphanumeric': 0.14948453608247422, 'fraction_numerical': 0.015670103092783504, 'mean_word_length': 4.307439824945296, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7214580', 'n_tokens_mistral': 1787, 'n_tokens_neox': 1689, 'n_words': 451}