<commit_before>package yabf

import (
	"bufio"
	"bytes"
	"fmt"
	"github.com/hhkbp2/go-strftime"
	g "github.com/hhkbp2/yabf/generator"
	"math"
	"os"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"
)

type Client interface {
	Main()
}

type ClientBase struct {
	Args           *Arguemnts
	DoTransactions bool
}

func NewClientBase(args *Arguemnts) *ClientBase {
	return &ClientBase{
		Args: args,
	}
}

func checkRequiredProperties(props Properties) bool {
	workload, ok := props[PropertyWorkload]
	if (!ok) || (len(workload) == 0) {
		EPrintf("Missing property: %s", PropertyWorkload)
		return false
	}
	return true
}

func (self *ClientBase) CheckProperties() {
	if !checkRequiredProperties(self.Args.Properties) {
		os.Exit(2)
	}
	propStr, ok := self.Args.Properties[PropertyTransactions]
	if ok {
		doTransactions, err := strconv.ParseBool(propStr)
		if err != nil {
			ExitOnError("invalid property %s=%s, should be bool", PropertyTransactions)
		}
		if doTransactions != self.DoTransactions {
			ExitOnError("property %s=%s conflicts with command %s", PropertyTransactions, propStr, self.Args.Command)
		}
	}
}

func (self *ClientBase) Main() {
	self.CheckProperties()

	props := self.Args.Properties
	propStr := props.GetDefault(PropertyMaxExecutionTime, PropertyMaxExecutionTimeDefault)
	maxExecutionTime, err := strconv.ParseInt(propStr, 0, 64)
	if err != nil {
		ExitOnError("invalid property %s=%s, should be integer", PropertyMaxExecutionTime, propStr)
	}
	// get number of threads, target and db
	propStr = props.GetDefault(PropertyThreadCount, PropertyThreadCountDefault)
	threadCount, err := strconv.ParseInt(propStr, 0, 64)
	if err != nil {
		ExitOnError("invalid property %s=%s, should be integer", PropertyThreadCount, propStr)
	}
	dbName := props.GetDefault(PropertyDB, PropertyDBDefault)
	propStr = props.GetDefault(PropertyTarget, PropertyTargetDefault)
	target, err := strconv.ParseInt(propStr, 0, 64)
	if err != nil {
		ExitOnError("invalid property %s=%s, should be integer", PropertyTarget, propStr)
	}
	targetPerThreadPerMS := float64(-1)
	if target > 0 {
		targetPerThread := float64(target) / float64(threadCount)
		targetPerThreadPerMS = targetPerThread / 1000.0
	}

	Printf("YCSB Client 0.1")
	Infof("Command line: \n%s", strings.Join(os.Args, " "))
	Printf("Loading workload...")

	// show a warning message that creating the workload is taking a while
	// but only do so if it is taking longer than 2 seconds
	// (showing the message right away if the setup wasn't taking very long
	// was confusing people)
	warningCh := make(chan int, 1)
	go func() {
		waitTime := time.Second * 2
		select {
		case <-warningCh:
			return
		case <-time.After(waitTime):
			Printf(" (might take a minutes for large data sets)")
		}
	}()
	// set up measurements
	SetMeasurementProperties(props)
	// load the workload
	workloadName := props.Get(PropertyWorkload)
	workload, err := NewWorkload(workloadName)
	if err != nil {
		ExitOnError("%s", err)
	}
	if err = workload.Init(props); err != nil {
		ExitOnError("%s", err)
	}
	warningCh <- 1

	// run the workload
	Printf("Starting test.")
	var opCount int64
	if self.Args.Command == "run" {
		propStr = props.GetDefault(PropertyOperationCount, PropertyOperationCountDefault)
		opCount, err = strconv.ParseInt(propStr, 0, 64)
		if err != nil {
			ExitOnError("invalid property %s=%s, should be integer", PropertyOperationCount, propStr)
		}
	} else {
		propStr, ok := props[PropertyInsertCount]
		if ok {
			opCount, err = strconv.ParseInt(propStr, 0, 64)
		} else {
			opCount, _ = strconv.ParseInt(PropertyRecordCountDefault, 0, 64)
		}
	}

	resultCh := make(chan int64, threadCount)
	// init all worker routines
	workerCh := make(chan int, threadCount)
	workers := make([]*Worker, 0, threadCount)
	startTime := NowNS()
	for i := int64(0); i < threadCount; i++ {
		db, err := NewDB(dbName, props)
		if err != nil {
			ExitOnError("fail to create db, error: %s", err)
		}
		threadOpCount := opCount / threadCount
		// ensure correct number of operations, in case opCount is not a multiple of threadCount
		if i < (opCount % threadCount) {
			threadOpCount++
		}
		worker := NewWorker(db, workload, props, self.DoTransactions, opCount, targetPerThreadPerMS, workerCh, resultCh)
		workers = append(workers, worker)
		go worker.run()
	}

	stopCh := make(chan int, 1)
	waitGroup := &sync.WaitGroup{}
	label := ""<commit_msg>Fix worker operation count should use the correct count<commit_after>package yabf

import (
	"bufio"
	"bytes"
	"fmt"
	"github.com/hhkbp2/go-strftime"
	g "github.com/hhkbp2/yabf/generator"
	"math"
	"os"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"
)

type Client interface {
	Main()
}

type ClientBase struct {
	Args           *Arguemnts
	DoTransactions bool
}

func NewClientBase(args *Arguemnts) *ClientBase {
	return &ClientBase{
		Args: args,
	}
}

func checkRequiredProperties(props Properties) bool {
	workload, ok := props[PropertyWorkload]
	if (!ok) || (len(workload) == 0) {
		EPrintf("Missing property: %s", PropertyWorkload)
		return false
	}
	return true
}

func (self *ClientBase) CheckProperties() {
	if !checkRequiredProperties(self.Args.Properties) {
		os.Exit(2)
	}
	propStr, ok := self.Args.Properties[PropertyTransactions]
	if ok {
		doTransactions, err := strconv.ParseBool(propStr)
		if err != nil {
			ExitOnError("invalid property %s=%s, should be bool", PropertyTransactions)
		}
		if doTransactions != self.DoTransactions {
			ExitOnError("property %s=%s conflicts with command %s", PropertyTransactions, propStr, self.Args.Command)
		}
	}
}

func (self *ClientBase) Main() {
	self.CheckProperties()

	props := self.Args.Properties
	propStr := props.GetDefault(PropertyMaxExecutionTime, PropertyMaxExecutionTimeDefault)
	maxExecutionTime, err := strconv.ParseInt(propStr, 0, 64)
	if err != nil {
		ExitOnError("invalid property %s=%s, should be integer", PropertyMaxExecutionTime, propStr)
	}
	// get number of threads, target and db
	propStr = props.GetDefault(PropertyThreadCount, PropertyThreadCountDefault)
	threadCount, err := strconv.ParseInt(propStr, 0, 64)
	if err != nil {
		ExitOnError("invalid property %s=%s, should be integer", PropertyThreadCount, propStr)
	}
	dbName := props.GetDefault(PropertyDB, PropertyDBDefault)
	propStr = props.GetDefault(PropertyTarget, PropertyTargetDefault)
	target, err := strconv.ParseInt(propStr, 0, 64)
	if err != nil {
		ExitOnError("invalid property %s=%s, should be integer", PropertyTarget, propStr)
	}
	targetPerThreadPerMS := float64(-1)
	if target > 0 {
		targetPerThread := float64(target) / float64(threadCount)
		targetPerThreadPerMS = targetPerThread / 1000.0
	}

	Printf("YCSB Client 0.1")
	Infof("Command line: \n%s", strings.Join(os.Args, " "))
	Printf("Loading workload...")

	// show a warning message that creating the workload is taking a while
	// but only do so if it is taking longer than 2 seconds
	// (showing the message right away if the setup wasn't taking very long
	// was confusing people)
	warningCh := make(chan int, 1)
	go func() {
		waitTime := time.Second * 2
		select {
		case <-warningCh:
			return
		case <-time.After(waitTime):
			Printf(" (might take a minutes for large data sets)")
		}
	}()
	// set up measurements
	SetMeasurementProperties(props)
	// load the workload
	workloadName := props.Get(PropertyWorkload)
	workload, err := NewWorkload(workloadName)
	if err != nil {
		ExitOnError("%s", err)
	}
	if err = workload.Init(props); err != nil {
		ExitOnError("%s", err)
	}
	warningCh <- 1

	// run the workload
	Printf("Starting test.")
	var opCount int64
	if self.Args.Command == "run" {
		propStr = props.GetDefault(PropertyOperationCount, PropertyOperationCountDefault)
		opCount, err = strconv.ParseInt(propStr, 0, 64)
		if err != nil {
			ExitOnError("invalid property %s=%s, should be integer", PropertyOperationCount, propStr)
		}
	} else {
		propStr, ok := props[PropertyInsertCount]
		if ok {
			opCount, err = strconv.ParseInt(propStr, 0, 64)
		} else {
			opCount, _ = strconv.ParseInt(PropertyRecordCountDefault, 0, 64)
		}
	}

	resultCh := make(chan int64, threadCount)
	// init all worker routines
	workerCh := make(chan int, threadCount)
	workers := make([]*Worker, 0, threadCount)
	startTime := NowNS()
	for i := int64(0); i < threadCount; i++ {
		db, err := NewDB(dbName, props)
		if err != nil {
			ExitOnError("fail to create db, error: %s", err)
		}
		threadOpCount := opCount / threadCount
		// ensure correct number of operations, in case opCount is not a multiple of threadCount
		if i < (opCount % threadCount) {
			threadOpCount++
		}
		worker := NewWorker(db, workload, props, self.DoTransactions, threadOpCount, targetPerThreadPerMS, workerCh, resultCh)
		workers = append(workers, worker)
		go worker.run()
	}

	stopCh := make(chan int, 1)
	waitGroup := &sync.WaitGroup{}
	label := ""
{'fraction_non_alphanumeric': 0.12482977757603268, 'fraction_numerical': 0.011348161597821153, 'mean_word_length': 4.396815676668708, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7261807', 'n_tokens_mistral': 3144, 'n_tokens_neox': 2921, 'n_words': 810}