<commit_before>package load

import (
	"errors"
	"fmt"
	"time"

	"github.com/golang/glog"
)

const MAX_RUNS = 50

type SLO struct {
	LatencyMs  int64 `json:"latencyMs"`
	Percentile int   `json:"percentile"`
}

type CalibrationRecord struct {
	Concurrency int    `json:"concurrency"`
	LatencyMs   int64  `json:"latencyMs"`
	Failures    uint64 `json:"failures"`
}

type LatencyCalibrationRun struct {
	RunId       string
	Config      *LatencyCalibration
	Results     []*CalibrationRecord
	FinalResult *CalibrationRecord
}

func NewLatencyCalibrationRun(id string, calibrate *LatencyCalibration) *LatencyCalibrationRun {
	return &LatencyCalibrationRun{
		RunId:   id,
		Config:  calibrate,
		Results: make([]*CalibrationRecord, 0),
	}
}

// RunCalibrationParams : latency struct
type LatencyCalibration struct {
	SLO       SLO     `json:"slo"`
	Load      AppLoad `json:"appLoad"`
	Calibrate struct {
		InitialConcurrency int    `json:"initialConcurrency"`
		Step               int    `json:"step"`
		RunsPerIntensity   int    `json:"runsPerIntensity"`
		LoadTime           string `json:"loadTime"`
	} `json:"calibrate"`
}

func (load *LatencyCalibrationRun) getSummaryLatency() int64 {
	var sum int64
	length := len(load.Results)
	for i := 0; i < load.Config.Calibrate.RunsPerIntensity; i++ {
		sum += load.Results[length-1-i].LatencyMs
	}

	return sum / int64(load.Config.Calibrate.RunsPerIntensity)
}

func (load *LatencyCalibrationRun) Run() error {
	glog.Infof("Start running latency calibration for id: " + load.RunId)
	appLoad := &load.Config.Load
	// Override defaults for calibration
	appLoad.NoLatencySummary = true
	// Setting a long default interval as we don't need periodic metrics
	appLoad.Interval = time.Hour * 24
	loadDuration, err := time.ParseDuration(load.Config.Calibrate.LoadTime)
	if err != nil {
		glog.Errorf("Unable to parse load time '%s': ", load.Config.Calibrate.LoadTime, err.Error())
		return fmt.Errorf("Unable to parse load time %s: %s", load.Config.Calibrate.LoadTime, err.Error())
	}

	load.Results = make([]*CalibrationRecord, 0)
	concurrency := load.Config.Calibrate.InitialConcurrency
	runs := 1
	for runs < MAX_RUNS {
		appLoad.Concurrency = concurrency
		var failedCount uint64
		for i := 0; i < load.Config.Calibrate.RunsPerIntensity || failedCount > 0; i++ {
			failedCount = 0
			go func() {
				glog.Infof("Starting calibration run #%d with concurrency %d", i+1, concurrency)
				appLoad.Run()
			}()
			<-time.After(loadDuration)
			appLoad.Stop()

			if appLoad.HandlerParams.Failed+appLoad.HandlerParams.Bad > 0 {
				failedCount = appLoad.HandlerParams.Failed + appLoad.HandlerParams.Bad
			}
<commit_msg>Move load time to top level<commit_after>package load

import (
	"errors"
	"fmt"
	"time"

	"github.com/golang/glog"
)

const MAX_RUNS = 50

type SLO struct {
	LatencyMs  int64 `json:"latencyMs"`
	Percentile int   `json:"percentile"`
}

type CalibrationRecord struct {
	Concurrency int    `json:"concurrency"`
	LatencyMs   int64  `json:"latencyMs"`
	Failures    uint64 `json:"failures"`
}

type LatencyCalibrationRun struct {
	RunId       string
	Config      *LatencyCalibration
	Results     []*CalibrationRecord
	FinalResult *CalibrationRecord
}

func NewLatencyCalibrationRun(id string, calibrate *LatencyCalibration) *LatencyCalibrationRun {
	return &LatencyCalibrationRun{
		RunId:   id,
		Config:  calibrate,
		Results: make([]*CalibrationRecord, 0),
	}
}

// RunCalibrationParams : latency struct
type LatencyCalibration struct {
	SLO       SLO     `json:"slo"`
	Load      AppLoad `json:"appLoad"`
	Calibrate struct {
		InitialConcurrency int `json:"initialConcurrency"`
		Step               int `json:"step"`
		RunsPerIntensity   int `json:"runsPerIntensity"`
	} `json:"calibrate"`
	LoadTime string `json:"loadTime"`
}

func (load *LatencyCalibrationRun) getSummaryLatency() int64 {
	var sum int64
	length := len(load.Results)
	for i := 0; i < load.Config.Calibrate.RunsPerIntensity; i++ {
		sum += load.Results[length-1-i].LatencyMs
	}

	return sum / int64(load.Config.Calibrate.RunsPerIntensity)
}

func (load *LatencyCalibrationRun) Run() error {
	glog.Infof("Start running latency calibration for id: " + load.RunId)
	appLoad := &load.Config.Load
	// Override defaults for calibration
	appLoad.NoLatencySummary = true
	// Setting a long default interval as we don't need periodic metrics
	appLoad.Interval = time.Hour * 24
	loadDuration, err := time.ParseDuration(load.Config.LoadTime)
	if err != nil {
		glog.Errorf("Unable to parse load time '%s': ", load.Config.LoadTime, err.Error())
		return fmt.Errorf("Unable to parse load time '%s': %s", load.Config.LoadTime, err.Error())
	}

	load.Results = make([]*CalibrationRecord, 0)
	concurrency := load.Config.Calibrate.InitialConcurrency
	runs := 1
	for runs < MAX_RUNS {
		appLoad.Concurrency = concurrency
		var failedCount uint64
		for i := 0; i < load.Config.Calibrate.RunsPerIntensity || failedCount > 0; i++ {
			failedCount = 0
			go func() {
				glog.Infof("Starting calibration run #%d with concurrency %d", i+1, concurrency)
				appLoad.Run()
			}()
			<-time.After(loadDuration)
			appLoad.Stop()

			if appLoad.HandlerParams.Failed+appLoad.HandlerParams.Bad > 0 {
				failedCount = appLoad.HandlerParams.Failed + appLoad.HandlerParams.Bad
			}

{'fraction_non_alphanumeric': 0.11640714014432206, 'fraction_numerical': 0.010634257500949488, 'mean_word_length': 4.148582600195503, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6791899', 'n_tokens_mistral': 1891, 'n_tokens_neox': 1749, 'n_words': 421}