<commit_before>package blocks

import (
	"container/heap"
	"time"
)

func Histogram(b *Block) {

	type histogramRule struct {
		Window int
		Key    string
	}

	type histogramBucket struct {
		Count int
		Label string
	}

	type histogramData struct {
		Histogram []histogramBucket
	}

	data := &histogramData{}

	var rule *histogramRule

	waitTimer := time.NewTimer(100 * time.Millisecond)
	window := time.Duration(0)

	histogram := map[string]*PriorityQueue{}
	emptyByte := make([]byte, 0)

	for {
		select {
		case query := <-b.Routes["histogram"]:
			data.Histogram = make([]histogramBucket, len(histogram))
			i := 0
			for k, pq := range histogram {
				bucket := histogramBucket{
					Count: len(*pq),
					Label: k,
				}
				data.Histogram[i] = bucket
				i++
			}
			marshal(query, data)
		case msg := <- b.Routes["get_rule"]:
			if rule == nil {
				marshal(msg, &histogramRule{})
			} else {
				marshal(msg, rule)
			}
		case ruleUpdate := <-b.Routes["set_rule"]:
			if rule == nil {
				rule = &histogramRule{}
			}

			unmarshal(ruleUpdate, rule)
			window = time.Duration(rule.Window) * time.Second
		case msg := <-b.InChan:
			if rule == nil {
				break
			}

			value := getKeyValues(msg, rule.Key)[0]
			valueString := value.(string)

			if pq, ok := histogram[valueString]; ok {<commit_msg>caught panic in histogram if requested key is set to nil<commit_after>package blocks

import (
	"container/heap"
	"log"
	"time"
)

func Histogram(b *Block) {

	type histogramRule struct {
		Window int
		Key    string
	}

	type histogramBucket struct {
		Count int
		Label string
	}

	type histogramData struct {
		Histogram []histogramBucket
	}

	data := &histogramData{}

	var rule *histogramRule

	waitTimer := time.NewTimer(100 * time.Millisecond)
	window := time.Duration(0)

	histogram := map[string]*PriorityQueue{}
	emptyByte := make([]byte, 0)

	for {
		select {
		case query := <-b.Routes["histogram"]:
			data.Histogram = make([]histogramBucket, len(histogram))
			i := 0
			for k, pq := range histogram {
				bucket := histogramBucket{
					Count: len(*pq),
					Label: k,
				}
				data.Histogram[i] = bucket
				i++
			}
			marshal(query, data)
		case msg := <-b.Routes["get_rule"]:
			if rule == nil {
				marshal(msg, &histogramRule{})
			} else {
				marshal(msg, rule)
			}
		case ruleUpdate := <-b.Routes["set_rule"]:
			if rule == nil {
				rule = &histogramRule{}
			}

			unmarshal(ruleUpdate, rule)
			window = time.Duration(rule.Window) * time.Second
		case msg := <-b.InChan:
			if rule == nil {
				break
			}

			value := getKeyValues(msg, rule.Key)[0]
			valueString, ok := value.(string)
			if !ok {
				log.Println("nil value against", rule.Key, " - ignoring")
				break
			}

			if pq, ok := histogram[valueString]; ok {
{'fraction_non_alphanumeric': 0.14031907179115302, 'fraction_numerical': 0.005076142131979695, 'mean_word_length': 3.0573529411764704, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '765170', 'n_tokens_mistral': 1163, 'n_tokens_neox': 937, 'n_words': 238}