<commit_before>package main

type MessageQueue struct {
	outputCh chan interface{}
	inputCh  chan interface{}
	selector chan *MessageQueue
}

func (mq *MessageQueue) Put(m interface{}) {
	mq.inputCh <- m
}

func (mq *MessageQueue) Get() interface{} {
	return <-mq.outputCh
}

func (mq *MessageQueue) Shutdown() {
	close(mq.inputCh)
}

func (mq *MessageQueue) loop() {
	buffer := make([]interface{}, 0, 10)
	for {
		var selector chan *MessageQueue
		var outputChan chan interface{}
		var outputValue interface{}
		if len(buffer) > 0 {
			selector = mq.selector
			outputChan = mq.outputCh
			outputValue = buffer[0]
		}
		select {
		case value, ok := <-mq.inputCh:
			if !ok {
				return
			}
			buffer = append(buffer, value)
		case outputChan <- outputValue:
			buffer = buffer[1:]
		case selector <- mq:
		}
	}
}

func NewMessageQueue(selector chan *MessageQueue) *MessageQueue {
	mq := &MessageQueue{
		outputCh: make(chan interface{}),
		inputCh:  make(chan interface{}, 1),
		selector: selector,
	}<commit_msg>MessageQueue: dump all the remaining messages on shutdown<commit_after>package main

type MessageQueue struct {
	outputCh chan interface{}
	inputCh  chan interface{}
	dumpCh   chan []interface{}
	selector chan *MessageQueue
}

func (mq *MessageQueue) Put(m interface{}) {
	mq.inputCh <- m
}

func (mq *MessageQueue) Get() interface{} {
	return <-mq.outputCh
}

func (mq *MessageQueue) Shutdown() []interface{} {
	close(mq.inputCh)
	return <-mq.dumpCh
}

func (mq *MessageQueue) loop() {
	buffer := make([]interface{}, 0, 10)
	for {
		var selector chan *MessageQueue
		var outputChan chan interface{}
		var outputValue interface{}
		if len(buffer) > 0 {
			selector = mq.selector
			outputChan = mq.outputCh
			outputValue = buffer[0]
		}
		select {
		case value, ok := <-mq.inputCh:
			if !ok {
				mq.dumpCh <- buffer
				return
			}
			buffer = append(buffer, value)
		case outputChan <- outputValue:
			buffer = buffer[1:]
		case selector <- mq:
		}
	}
}

func NewMessageQueue(selector chan *MessageQueue) *MessageQueue {
	mq := &MessageQueue{
		outputCh: make(chan interface{}),
		inputCh:  make(chan interface{}, 1),
		dumpCh:   make(chan []interface{}, 1),
		selector: selector,
	}
{'fraction_non_alphanumeric': 0.1298819255222525, 'fraction_numerical': 0.006811989100817439, 'mean_word_length': 3.841758241758242, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1575329', 'n_tokens_mistral': 831, 'n_tokens_neox': 720, 'n_words': 213}