<commit_before>package ase

import (
	"bytes"
	"encoding/binary"
	"io"
	"unicode/utf16"
)

type Group struct {
	nameLen uint16
	Name    string
	Colors  []Color
}

func (group *Group) read(r io.Reader) (err error) {
	if err = group.readNameLen(r); err != nil {
		return
	}

	return group.readName(r)
}

func (group *Group) readNameLen(r io.Reader) error {
	return binary.Read(r, binary.BigEndian, &group.nameLen)
}

func (group *Group) readName(r io.Reader) (err error) {
	//	make array for our color name based on block length
	name := make([]uint16, group.nameLen)
	if err = binary.Read(r, binary.BigEndian, &name); err != nil {
		return
	}

	//	decode our name. we trim off the last byte since it's zero terminated
	group.Name = string(utf16.Decode(name[:len(name)-1]))

	return
}

func (group *Group) write(w io.Writer) (err error) {

	// Write group start headers (block entry, block length,  nameLen, name)
	if err = group.writeBlockStart(w); err != nil {
		return
	}

	if err = group.writeBlockLength(w); err != nil {
		return
	}

	if err = group.writeNameLen(w); err != nil {
		return
	}
	if err = group.writeName(w); err != nil {
		return
	}

	// Write group's colors
	for _, color := range group.Colors {
		if err = color.write(w); err != nil {
			return err
		}
	}

	// Write the group's closing headers
	if err = group.writeBlockEnd(w); err != nil {
		return
	}

	return nil
}

func (group *Group) writeBlockStart(w io.Writer) (err error) {
	return binary.Write(w, binary.BigEndian, groupStart)
}

func (group *Group) writeBlockEnd(w io.Writer) (err error) {
	return binary.Write(w, binary.BigEndian, groupEnd)
}
<commit_msg>(encode)include zero termination for writing a group end block header<commit_after>package ase

import (
	"bytes"
	"encoding/binary"
	"io"
	"unicode/utf16"
)

type Group struct {
	nameLen uint16
	Name    string
	Colors  []Color
}

func (group *Group) read(r io.Reader) (err error) {
	if err = group.readNameLen(r); err != nil {
		return
	}

	return group.readName(r)
}

func (group *Group) readNameLen(r io.Reader) error {
	return binary.Read(r, binary.BigEndian, &group.nameLen)
}

func (group *Group) readName(r io.Reader) (err error) {
	//	make array for our color name based on block length
	name := make([]uint16, group.nameLen)
	if err = binary.Read(r, binary.BigEndian, &name); err != nil {
		return
	}

	//	decode our name. we trim off the last byte since it's zero terminated
	group.Name = string(utf16.Decode(name[:len(name)-1]))

	return
}

func (group *Group) write(w io.Writer) (err error) {

	// Write group start headers (block entry, block length,  nameLen, name)
	if err = group.writeBlockStart(w); err != nil {
		return
	}

	if err = group.writeBlockLength(w); err != nil {
		return
	}

	if err = group.writeNameLen(w); err != nil {
		return
	}
	if err = group.writeName(w); err != nil {
		return
	}

	// Write group's colors
	for _, color := range group.Colors {
		if err = color.write(w); err != nil {
			return err
		}
	}

	// Write the group's closing headers
	if err = group.writeBlockEnd(w); err != nil {
		return
	}

	return nil
}

// Wrapper around writing a group start header.
func (group *Group) writeBlockStart(w io.Writer) (err error) {
	return binary.Write(w, binary.BigEndian, groupStart)
}

// Wrapper around writing a group end header.
func (group *Group) writeBlockEnd(w io.Writer) (err error) {
	// First writes the groupEnd block followed by a terminating zero.
	binary.Write(w, binary.BigEndian, groupEnd)
	return binary.Write(w, binary.BigEndian, uint16(0x0000))
}

{'fraction_non_alphanumeric': 0.1307540242869246, 'fraction_numerical': 0.007060152499293985, 'mean_word_length': 3.654402102496715, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6751453', 'n_tokens_mistral': 1304, 'n_tokens_neox': 1243, 'n_words': 386}