<commit_before>package plinq

import (
	"errors"
	"fmt"
	c "github.com/smartystreets/goconvey/convey"
	"math"
	"math/rand"
	"reflect"
	"runtime"
	"strconv"
	"strings"
	"testing"
)

const (
	count     int = 20
	rptCount  int = 22
	countP    int = 300
	rptCountP int = 300
)

var (
	maxProcs int

	taUsers    = [][]interface{}{make([]interface{}, count), make([]interface{}, countP)}
	taRptUsers = [][]interface{}{make([]interface{}, rptCount), make([]interface{}, rptCountP)}
	taUsers2   = [][]interface{}{make([]interface{}, count), make([]interface{}, countP)}
	taInts     = [][]int{make([]int, count), make([]int, countP)}
	taRoles    = [][]interface{}{make([]interface{}, count), make([]interface{}, countP)}
	taEmptys   = [][]int{[]int{}, []int{}}

	tUsers    = taUsers[0]
	tRptUsers = taRptUsers[0]
	tUsers2   = taUsers2[0]
	tInts     = taInts[0]
	tRoles    = taRoles[0]

	tMap map[int]interface{} = make(map[int]interface{}, count)

	sequentialChunkSize int = count
	parallelChunkSize   int = count / 7
)

func init() {
	maxProcs = numCPU
	runtime.GOMAXPROCS(maxProcs)

	fullTestDatas := func(seq int) {
		size := len(taInts[seq])
		rptSize := len(taRptUsers[seq])
		for i := 0; i < size; i++ {
			taInts[seq][i] = i
			taUsers[seq][i] = user{i, "user" + strconv.Itoa(i)}
			taRptUsers[seq][i] = user{i, "user" + strconv.Itoa(i)}
			taUsers2[seq][i] = user{i + size/2, "user" + strconv.Itoa(i+size/2)}
			//tMap[seq][i] = user{i, "user" + strconv.Itoa(i)}
		}
		for i := 0; i < rptSize-size; i++ {
			taRptUsers[seq][size+i] = user{i, "user" + strconv.Itoa(size+i)}
		}
		for i := 0; i < size/2; i++ {
			taRoles[seq][i*2] = role{i, "role" + strconv.Itoa(i)}
			taRoles[seq][i*2+1] = role{i, "role" + strconv.Itoa(i+1)}
		}
	}

	//full datas for testing sequential
	fullTestDatas(0)

	//full datas for testing parallel
	fullTestDatas(1)

}

// The structs for testing----------------------------------------------------
type user struct {
	id   int
	name string
}
type role struct {
	uid  int
	role string
}
type userRoles struct {
	user
	roles []role
}

// The functions used in testing----------------------------------------------
var (
	//用来给测试的操作增加计算量
	computerTask = func() {
		for i := 0; i < 2; i++ {
			_ = strconv.Itoa(i)
		}
	}

	//随机运算，用于打乱次序
	outOfOrder = func() {
		rand.Seed(10)
		j := rand.Intn(50000)
		var sum float64 = 0
		for k := 0; k < j; k++ {
			sum += math.Cos(float64(k)) * math.Pi
		}
		_ = sum
	}

	getChan = func(src interface{}) interface{} {
		switch s := src.(type) {
		case []interface{}:
			chanSrc := make(chan interface{})
			go func() {
				for _, v := range s {
					chanSrc <- v
				}
				close(chanSrc)
			}()
			return chanSrc
		case []int:
			chanSrc := make(chan int)
			go func() {
				for _, v := range s {
					chanSrc <- v
				}
				close(chanSrc)
			}()
			return chanSrc
		default:
			return nil
		}
	}

	//将在最后一个数据抛出错误的过滤函数，用于测试错误处理
	filterWithPanic = func(v interface{}) bool {
		var s []interface{}
		switch val := v.(type) {
		case int:
			if val == count-1 {
				_ = s[2]
			}
		case user:
			if val.id == count-1 {
				_ = s[2]
			}
		}
		return true
	}

	//测试用的过滤函数
	isEven = func(v interface{}) bool {
		computerTask()
		return v.(int)%2 == 0
	}

	//测试用的过滤函数
	idIsEven = func(v interface{}) bool {
		computerTask()
		return strconv.Itoa(v.(user).id%2) == "0"
	}

	//测试用的过滤函数，加入随机运算来打乱返回的顺序
	isEvenWithOutOfOrder = func(v interface{}) bool {
		i := v.(int)
		outOfOrder()
		return i%2 == 0
	}

	//将抛出错误的转换函数，用于测试错误处理
	projectWithPanic = func(v interface{}) interface{} {
		switch val := v.(type) {
		case int:
			if val == count-1 {
				var s []interface{}
				_ = s[2]
			}
		case user:
			if val.id == count-1 {
				var s []interface{}
				_ = s[2]
			}
		}
		return v
	}

	//测试用的转换函数
	multiply10 = func(v interface{}) interface{} {
		computerTask()
		return v.(int) * 10
	}

	userToStr = func(v interface{}) interface{} {
		computerTask()
		val := v.(user)
		return strconv.Itoa(val.id) + "/" + val.name
	}

	distinctUser = func(v interface{}) interface{} {
		u := v.(user)
		return u.id
	}

	orderUserById = func(v1 interface{}, v2 interface{}) int {
		u1, u2 := v1.(user), v2.(user)
		if u1.id < u2.id {
			return -1
		} else if u1.id == u2.id {
			return 0
		} else {
			return 1
		}
	}

	orderUserByIdPanic = func(v1 interface{}, v2 interface{}) int {
		panic(errors.New("panic"))
	}

	getUserId = func(v interface{}) interface{} {
		return v.(user).id
	}

	getRoleUid = func(v interface{}) interface{} {
		r := v.(role)
		return r.uid
	}

	getUserIdAndRole = func(u interface{}, v interface{}) interface{} {
		return strconv.Itoa(u.(user).id) + "-" + v.(role).role
	}
)

// Testing functions----------------------------------------------------------
func TestFrom(t *testing.T) {
	c.Convey("Nil as data source", t, func() {<commit_msg>update testing code<commit_after>package plinq

import (
	"errors"
	"fmt"
	c "github.com/smartystreets/goconvey/convey"
	"math"
	"math/rand"
	"reflect"
	"runtime"
	"strconv"
	"strings"
	"testing"
)

const (
	count     int = 20
	rptCount  int = 22
	countP    int = 300
	rptCountP int = 300
)

var (
	maxProcs int

	taUsers    = [][]interface{}{make([]interface{}, count), make([]interface{}, countP)}
	taRptUsers = [][]interface{}{make([]interface{}, rptCount), make([]interface{}, rptCountP)}
	taUsers2   = [][]interface{}{make([]interface{}, count), make([]interface{}, countP)}
	taInts     = [][]int{make([]int, count), make([]int, countP)}
	taRoles    = [][]interface{}{make([]interface{}, count), make([]interface{}, countP)}
	taEmptys   = [][]int{[]int{}, []int{}}

	tUsers    = taUsers[0]
	tRptUsers = taRptUsers[0]
	tUsers2   = taUsers2[0]
	tInts     = taInts[0]
	tRoles    = taRoles[0]

	tMap map[int]interface{} = make(map[int]interface{}, count)

	sequentialChunkSize int = count
	parallelChunkSize   int = count / 7
)

func init() {
	maxProcs = numCPU
	runtime.GOMAXPROCS(maxProcs)

	fullTestDatas := func(seq int) {
		size := len(taInts[seq])
		rptSize := len(taRptUsers[seq])
		for i := 0; i < size; i++ {
			taInts[seq][i] = i
			taUsers[seq][i] = user{i, "user" + strconv.Itoa(i)}
			taRptUsers[seq][i] = user{i, "user" + strconv.Itoa(i)}
			taUsers2[seq][i] = user{i + size/2, "user" + strconv.Itoa(i+size/2)}
			//tMap[seq][i] = user{i, "user" + strconv.Itoa(i)}
		}
		for i := 0; i < rptSize-size; i++ {
			taRptUsers[seq][size+i] = user{i, "user" + strconv.Itoa(size+i)}
		}
		for i := 0; i < size/2; i++ {
			taRoles[seq][i*2] = role{i, "role" + strconv.Itoa(i)}
			taRoles[seq][i*2+1] = role{i, "role" + strconv.Itoa(i+1)}
		}
	}

	//full datas for testing sequential
	fullTestDatas(0)

	//full datas for testing parallel
	fullTestDatas(1)

}

// The structs for testing----------------------------------------------------
type user struct {
	id   int
	name string
}
type role struct {
	uid  int
	role string
}
type userRoles struct {
	user
	roles []role
}

// The functions used in testing----------------------------------------------
var (
	//用来给测试的操作增加计算量
	computerTask = func() {
		for i := 0; i < 2; i++ {
			_ = strconv.Itoa(i)
		}
	}

	//随机运算，用于打乱次序
	outOfOrder = func() {
		rand.Seed(10)
		j := rand.Intn(50000)
		var sum float64 = 0
		for k := 0; k < j; k++ {
			sum += math.Cos(float64(k)) * math.Pi
		}
		_ = sum
	}

	getChan = func(src interface{}) interface{} {
		switch s := src.(type) {
		case []interface{}:
			chanSrc := make(chan interface{})
			go func() {
				for _, v := range s {
					chanSrc <- v
				}
				close(chanSrc)
			}()
			return chanSrc
		case []int:
			chanSrc := make(chan int)
			go func() {
				for _, v := range s {
					chanSrc <- v
				}
				close(chanSrc)
			}()
			return chanSrc
		default:
			return nil
		}
	}

	//panic when is last item
	panicInLast = func(v interface{}) {
		var s []interface{}
		switch val := v.(type) {
		case int:
			if val == count-1 {
				_ = s[2]
			}
		case user:
			if val.id == count-1 {
				_ = s[2]
			}
		}
	}

	//将在最后一个数据抛出错误的过滤函数，用于测试错误处理
	filterWithPanic = func(v interface{}) bool {
		panicInLast()
		return true
	}

	//测试用的过滤函数
	isEven = func(v interface{}) bool {
		computerTask()
		return v.(int)%2 == 0
	}

	//测试用的过滤函数
	idIsEven = func(v interface{}) bool {
		computerTask()
		return strconv.Itoa(v.(user).id%2) == "0"
	}

	//测试用的过滤函数，加入随机运算来打乱返回的顺序
	isEvenWithOutOfOrder = func(v interface{}) bool {
		outOfOrder()
		return v.(int)%2 == 0
	}

	//将抛出错误的转换函数，用于测试错误处理
	projectWithPanic = func(v interface{}) interface{} {
		panicInLast()
		return v
	}

	//测试用的转换函数
	multiply10 = func(v interface{}) interface{} {
		computerTask()
		return v.(int) * 10
	}

	userToStr = func(v interface{}) interface{} {
		computerTask()
		val := v.(user)
		return strconv.Itoa(val.id) + "/" + val.name
	}

	distinctUser = func(v interface{}) interface{} {
		return v.(user).id
	}

	orderUserById = func(v1 interface{}, v2 interface{}) int {
		u1, u2 := v1.(user), v2.(user)
		if u1.id < u2.id {
			return -1
		} else if u1.id == u2.id {
			return 0
		} else {
			return 1
		}
	}

	orderUserByIdPanic = func(v1 interface{}, v2 interface{}) int {
		panic(errors.New("panic"))
	}

	getUserId = func(v interface{}) interface{} {
		return v.(user).id
	}

	getRoleUid = func(v interface{}) interface{} {
		return v.(role).uid
	}

	getUserIdAndRole = func(u interface{}, v interface{}) interface{} {
		return strconv.Itoa(u.(user).id) + "-" + v.(role).role
	}
)

// Testing functions----------------------------------------------------------
func TestFrom(t *testing.T) {
	c.Convey("Nil as data source", t, func() {
{'fraction_non_alphanumeric': 0.20642105263157895, 'fraction_numerical': 0.016421052631578947, 'mean_word_length': 2.7538522323192414, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1618770', 'n_tokens_mistral': 4252, 'n_tokens_neox': 3943, 'n_words': 841}