<commit_before>package protocol

type MessageBlock struct {
	Offset int64
	Msg    *Message
}

func (msb *MessageBlock) encode(pe packetEncoder) {
	pe.putInt64(msb.Offset)
	pe.pushLength32()
	msb.Msg.encode(pe)
	pe.pop()
}

func (msb *MessageBlock) decode(pd packetDecoder) (err error) {
	msb.Offset, err = pd.getInt64()
	if err != nil {
		return err
	}

	err = pd.pushLength32()
	if err != nil {
		return err
	}

	msb.Msg = new(Message)
	err = msb.Msg.decode(pd)
	if err != nil {
		return err
	}

	err = pd.pop()
	if err != nil {
		return err
	}

	return nil
}

type MessageSet struct {
	Messages []*MessageBlock
}

func (ms *MessageSet) encode(pe packetEncoder) {
	for i := range ms.Messages {
		ms.Messages[i].encode(pe)
	}
}

func (ms *MessageSet) decode(pd packetDecoder) (err error) {
	ms.Messages = nil

	for pd.remaining() > 0 {
		msb := new(MessageBlock)
		err = msb.decode(pd)
		if err != nil {
			return err
		}
		ms.Messages = append(ms.Messages, msb)
	}

	return nil
}

func (ms *MessageSet) addMessage(msg *Message) {
	block := new(MessageBlock)
	block.Msg = msg<commit_msg>Correctly deal with partial messages ending message sets<commit_after>package protocol

type MessageBlock struct {
	Offset int64
	Msg    *Message
}

func (msb *MessageBlock) encode(pe packetEncoder) {
	pe.putInt64(msb.Offset)
	pe.pushLength32()
	msb.Msg.encode(pe)
	pe.pop()
}

func (msb *MessageBlock) decode(pd packetDecoder) (err error) {
	msb.Offset, err = pd.getInt64()
	if err != nil {
		return err
	}

	err = pd.pushLength32()
	if err != nil {
		return err
	}

	msb.Msg = new(Message)
	err = msb.Msg.decode(pd)
	if err != nil {
		return err
	}

	err = pd.pop()
	if err != nil {
		return err
	}

	return nil
}

type MessageSet struct {
	Messages []*MessageBlock
}

func (ms *MessageSet) encode(pe packetEncoder) {
	for i := range ms.Messages {
		ms.Messages[i].encode(pe)
	}
}

func (ms *MessageSet) decode(pd packetDecoder) (err error) {
	ms.Messages = nil

	for pd.remaining() > 0 {
		msb := new(MessageBlock)
		err = msb.decode(pd)
		if err == nil {
			ms.Messages = append(ms.Messages, msb)
		} else {
			// As an optimization the server is allowed to return a partial message at the
			// end of the message set. Clients should handle this case. So we just ignore such things.
			break
		}
	}

	return nil
}

func (ms *MessageSet) addMessage(msg *Message) {
	block := new(MessageBlock)
	block.Msg = msg
{'fraction_non_alphanumeric': 0.12541666666666668, 'fraction_numerical': 0.009166666666666667, 'mean_word_length': 3.4298892988929888, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5684596', 'n_tokens_mistral': 938, 'n_tokens_neox': 884, 'n_words': 252}