<commit_before>//
// Copyright 2016 <NAME>. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package receiver

import (
	"fmt"
	"log"
	"time"
)

var workerPeriodicFlush = func(ident string, dsf dsFlusherBlocking, recent map[int64]*receiverDs, minCacheDur, maxCacheDur time.Duration, maxPoints, maxFlushes int) map[int64]*receiverDs {
	leftover := make(map[int64]*receiverDs)
	n := 0
	for id, rds := range recent {
		if rds.shouldBeFlushed(maxPoints, minCacheDur, minCacheDur) {
			if debug {
				log.Printf("%s: Requesting (periodic) flush of ds id: %d", ident, id)
			}
			if !dsf.flushDs(rds, false) {
				leftover[id] = rds
			}
			delete(recent, id)
		}
		n++
		if n > maxFlushes {
			break
		}
	}
	return leftover
}

func reportWorkerChannelFillPercent(workerCh chan *incomingDpWithDs, sr statReporter, ident string, nap time.Duration) {
	fillStatName := fmt.Sprintf("receiver.workers.%s.channel.fill_percent", ident)
	lenStatName := fmt.Sprintf("receiver.workers.%s.channel.len", ident)
	cp := float64(cap(workerCh))
	for {
		time.Sleep(nap)
		ln := float64(len(workerCh))
		if cp > 0 {
			fillPct := (ln / cp) * 100
			sr.reportStatGauge(fillStatName, fillPct)
		}
		sr.reportStatGauge(lenStatName, ln)
	}
}

var worker = func(wc wController, dsf dsFlusherBlocking, workerCh chan *incomingDpWithDs,
	minCacheDur, maxCacheDur time.Duration, maxPoints int, flushInt time.Duration, sr statReporter) {
	wc.onEnter()
	defer wc.onExit()

	var recent = make(map[int64]*receiverDs)
	var leftover map[int64]*receiverDs

	periodicFlushTicker := time.NewTicker(flushInt)

	go reportWorkerChannelFillPercent(workerCh, sr, wc.ident(), time.Second)

	log.Printf("  - %s started.", wc.ident())
	wc.onStarted()

	maxFlushes := len(workerCh) / 2
	for {
		select {
		case <-periodicFlushTicker.C:
			if len(leftover) > 0 {
				leftover = workerPeriodicFlush(wc.ident(), dsf, leftover, minCacheDur, maxCacheDur, maxPoints, maxFlushes)
			} else {<commit_msg>maxFlushes ought to be based on cap, not len<commit_after>//
// Copyright 2016 <NAME>. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package receiver

import (
	"fmt"
	"log"
	"time"
)

var workerPeriodicFlush = func(ident string, dsf dsFlusherBlocking, recent map[int64]*receiverDs, minCacheDur, maxCacheDur time.Duration, maxPoints, maxFlushes int) map[int64]*receiverDs {
	leftover := make(map[int64]*receiverDs)
	n := 0
	for id, rds := range recent {
		if rds.shouldBeFlushed(maxPoints, minCacheDur, minCacheDur) {
			if debug {
				log.Printf("%s: Requesting (periodic) flush of ds id: %d", ident, id)
			}
			if !dsf.flushDs(rds, false) {
				leftover[id] = rds
			}
			delete(recent, id)
		}
		n++
		if n > maxFlushes {
			break
		}
	}
	return leftover
}

func reportWorkerChannelFillPercent(workerCh chan *incomingDpWithDs, sr statReporter, ident string, nap time.Duration) {
	fillStatName := fmt.Sprintf("receiver.workers.%s.channel.fill_percent", ident)
	lenStatName := fmt.Sprintf("receiver.workers.%s.channel.len", ident)
	cp := float64(cap(workerCh))
	for {
		time.Sleep(nap)
		ln := float64(len(workerCh))
		if cp > 0 {
			fillPct := (ln / cp) * 100
			sr.reportStatGauge(fillStatName, fillPct)
		}
		sr.reportStatGauge(lenStatName, ln)
	}
}

var worker = func(wc wController, dsf dsFlusherBlocking, workerCh chan *incomingDpWithDs,
	minCacheDur, maxCacheDur time.Duration, maxPoints int, flushInt time.Duration, sr statReporter) {
	wc.onEnter()
	defer wc.onExit()

	var recent = make(map[int64]*receiverDs)
	var leftover map[int64]*receiverDs

	periodicFlushTicker := time.NewTicker(flushInt)

	go reportWorkerChannelFillPercent(workerCh, sr, wc.ident(), time.Second)

	log.Printf("  - %s started.", wc.ident())
	wc.onStarted()

	maxFlushes := cap(workerCh) / 2
	for {
		select {
		case <-periodicFlushTicker.C:
			if len(leftover) > 0 {
				leftover = workerPeriodicFlush(wc.ident(), dsf, leftover, minCacheDur, maxCacheDur, maxPoints, maxFlushes)
			} else {
{'fraction_non_alphanumeric': 0.118555667001003, 'fraction_numerical': 0.011634904714142427, 'mean_word_length': 4.166839378238342, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3555358', 'n_tokens_mistral': 1882, 'n_tokens_neox': 1669, 'n_words': 480}