<commit_before>package nbfx

import (
	"bytes"
	"encoding/xml"
	"io"
	"math"
)

type decoder struct {
	codec codec
	elementStack Stack
}

func NewDecoder() Decoder {
	return NewDecoderWithStrings(nil)
}

func NewDecoderWithStrings(dictionaryStrings map[uint32]string) Decoder {
	decoder := &decoder{codec{make(map[uint32]string), make(map[string]uint32)},Stack{}}
	if dictionaryStrings != nil {
		for k, v := range dictionaryStrings {
			decoder.codec.addDictionaryString(k, v)
		}
	}
	return decoder
}

func (d *decoder) Decode(bin []byte) (string, error) {
	reader := bytes.NewReader(bin)
	xmlBuf := &bytes.Buffer{}
	xmlEncoder := xml.NewEncoder(xmlBuf)
	rec, err := decodeRecord(d, reader)
	for err == nil && rec != nil {
		if rec.isElement() {
			elementReader := rec.(elementRecordDecoder)
			rec, err = elementReader.decodeElement(xmlEncoder, reader)
		} else { // text record
			textReader := rec.(textRecordDecoder)
			_, err = textReader.decodeText(xmlEncoder, reader)
			rec = nil
		}
		if err == nil && rec == nil {
			rec, err = decodeRecord(d, reader)
		}
	}
	//fmt.Println("Exiting main decoder loop...")
	xmlEncoder.Flush()
	if err != nil && err != io.EOF {
		return xmlBuf.String(), err
	}
	return xmlBuf.String(), nil
}

func readMultiByteInt31(reader *bytes.Reader) (uint32, error) {
	buf := new([5]byte)
	keepReading := true
	i := -1
	for keepReading {
		i++
		b, err := reader.ReadByte()
		if err != nil {
			return 0, err
		}
		if b >= MASK_MBI31 {
			b -= MASK_MBI31
			keepReading = true
		} else {
			keepReading = false
		}
		buf[i] = b
	}
	var val uint32
	for ; i >= 0; i-- {
		val += uint32(buf[i]) * uint32(math.Pow(128, float64(i)))
	}
	return val, nil
}

func readStringBytes(reader *bytes.Reader, readLenFunc func(r *bytes.Reader) (uint32, error)) (string, error) {
	len, err := readLenFunc(reader)
	if err != nil {
		return "", err
	}
	strBuffer := bytes.Buffer{}
	for i := uint32(0); i < len; {
		b, err := reader.ReadByte()
		if err != nil {
			return strBuffer.String(), err
		}
		strBuffer.WriteByte(b)
		i++
	}
	return strBuffer.String(), nil
}

func readString(reader *bytes.Reader) (string, error) {
	return readStringBytes(reader, func(r *bytes.Reader) (uint32, error) {
		return readMultiByteInt31(r)<commit_msg>readMultiByteInt31 recursively<commit_after>package nbfx

import (
	"bytes"
	"encoding/xml"
	"io"
)

type decoder struct {
	codec codec
	elementStack Stack
}

func NewDecoder() Decoder {
	return NewDecoderWithStrings(nil)
}

func NewDecoderWithStrings(dictionaryStrings map[uint32]string) Decoder {
	decoder := &decoder{codec{make(map[uint32]string), make(map[string]uint32)},Stack{}}
	if dictionaryStrings != nil {
		for k, v := range dictionaryStrings {
			decoder.codec.addDictionaryString(k, v)
		}
	}
	return decoder
}

func (d *decoder) Decode(bin []byte) (string, error) {
	reader := bytes.NewReader(bin)
	xmlBuf := &bytes.Buffer{}
	xmlEncoder := xml.NewEncoder(xmlBuf)
	rec, err := decodeRecord(d, reader)
	for err == nil && rec != nil {
		if rec.isElement() {
			elementReader := rec.(elementRecordDecoder)
			rec, err = elementReader.decodeElement(xmlEncoder, reader)
		} else { // text record
			textReader := rec.(textRecordDecoder)
			_, err = textReader.decodeText(xmlEncoder, reader)
			rec = nil
		}
		if err == nil && rec == nil {
			rec, err = decodeRecord(d, reader)
		}
	}
	//fmt.Println("Exiting main decoder loop...")
	xmlEncoder.Flush()
	if err != nil && err != io.EOF {
		return xmlBuf.String(), err
	}
	return xmlBuf.String(), nil
}

func readMultiByteInt31(reader *bytes.Reader) (uint32, error) {
	b, err := reader.ReadByte()
	if b < byte(128) {
		return uint32(b), err
	}
	nextB, err := readMultiByteInt31(reader)
	return uint32(128*nextB + uint32(b-128)), err
}

func readStringBytes(reader *bytes.Reader, readLenFunc func(r *bytes.Reader) (uint32, error)) (string, error) {
	len, err := readLenFunc(reader)
	if err != nil {
		return "", err
	}
	strBuffer := bytes.Buffer{}
	for i := uint32(0); i < len; {
		b, err := reader.ReadByte()
		if err != nil {
			return strBuffer.String(), err
		}
		strBuffer.WriteByte(b)
		i++
	}
	return strBuffer.String(), nil
}

func readString(reader *bytes.Reader) (string, error) {
	return readStringBytes(reader, func(r *bytes.Reader) (uint32, error) {
		return readMultiByteInt31(r)
{'fraction_non_alphanumeric': 0.1474418604651163, 'fraction_numerical': 0.017674418604651163, 'mean_word_length': 4.071933962264151, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 20}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6765270', 'n_tokens_mistral': 1661, 'n_tokens_neox': 1517, 'n_words': 379}