<commit_before>package fabric

import (
	"bufio"
	"encoding/binary"
	"errors"
	"strings"
	"io"
)

func ReadToken(r io.Reader) ([]byte, error) {
	br := bufio.NewReader(r)
	l, err := binary.ReadUvarint(br)
	if err != nil {
		return nil, err
	}

	b := make([]byte, l)
	n, err := io.ReadFull(br, b)
	if err != nil {
		return nil, err
	}

	if uint64(n) != l {
		return nil, errors.New("Invalid token length")
	}

	return b[:n-1], nil

}

func WriteToken(w io.Writer, bs []byte) error {
	bw := bufio.NewWriter(w)
	vb := make([]byte, 16)
	n := binary.PutUvarint(vb, uint64(len(bs)+1))
	wb := append(vb[:n], bs...)
	wb = append(wb, '\n')
	if _, err := w.Write(wb); err != nil {
		return err
	}

	return bw.Flush()
)

func addrSplit(addr string) [][]string {
	parts := strings.Split(addr, "/")
	res := make([][]string, len(parts))
	for i, part := range parts {
		res[i] = strings.Split(part, ":")
	}
	return res
}

func addrJoin(res [][]string) string {
	parts := make([]string, len(res))
	for i, part := range res {
		parts[i] = strings.Join(part, ":")
	}
	return strings.Join(parts, "/")
}
<commit_msg>Fix syntax from rebase<commit_after>package fabric

import (
	"bufio"
	"encoding/binary"
	"errors"
	"io"
	"strings"
)

func ReadToken(r io.Reader) ([]byte, error) {
	br := bufio.NewReader(r)
	l, err := binary.ReadUvarint(br)
	if err != nil {
		return nil, err
	}

	b := make([]byte, l)
	n, err := io.ReadFull(br, b)
	if err != nil {
		return nil, err
	}

	if uint64(n) != l {
		return nil, errors.New("Invalid token length")
	}

	return b[:n-1], nil

}

func WriteToken(w io.Writer, bs []byte) error {
	bw := bufio.NewWriter(w)
	vb := make([]byte, 16)
	n := binary.PutUvarint(vb, uint64(len(bs)+1))
	wb := append(vb[:n], bs...)
	wb = append(wb, '\n')
	if _, err := w.Write(wb); err != nil {
		return err
	}

	return bw.Flush()
}

func addrSplit(addr string) [][]string {
	parts := strings.Split(addr, "/")
	res := make([][]string, len(parts))
	for i, part := range parts {
		res[i] = strings.Split(part, ":")
	}
	return res
}

func addrJoin(res [][]string) string {
	parts := make([]string, len(res))
	for i, part := range res {
		parts[i] = strings.Join(part, ":")
	}
	return strings.Join(parts, "/")
}

{'fraction_non_alphanumeric': 0.2028194633924511, 'fraction_numerical': 0.007276034561164165, 'mean_word_length': 3.6218487394957983, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6016495', 'n_tokens_mistral': 923, 'n_tokens_neox': 890, 'n_words': 225}