<commit_before>package gson

import "strconv"
import "math"
import "unicode/utf8"
import "encoding/binary"

var nullStr = "null"
var trueStr = "true"
var falseStr = "false"

func json2cbor(txt string, out []byte, config *Config) (string, int) {
	txt = skipWS(txt, config.ws)

	if len(txt) < 1 {
		panic("cbor scanner empty json text")
	}

	if numCheck[txt[0]] == 1 {
		return jsonNumToCbor(txt, out, config)
	}

	switch txt[0] {
	case 'n':
		if len(txt) >= 4 && txt[:4] == nullStr {
			n := cborNull(out)
			return txt[4:], n
		}
		panic("cbor scanner expected null")

	case 't':
		if len(txt) >= 4 && txt[:4] == trueStr {
			n := cborTrue(out)
			return txt[4:], n
		}
		panic("cbor scanner expected true")

	case 'f':
		if len(txt) >= 5 && txt[:5] == falseStr {
			n := cborFalse(out)
			return txt[5:], n
		}
		panic("cbor scanner expected false")

	case '"':
		if config.jsonString {
			return jsonStringToCbor(txt, out)
		}
		n := 0
		txt, x := scanString(txt, out[n+16:]) // 16 reserved for cbor hdr
		n += valtext2cbor(bytes2str(out[n+16:n+16+x]), out[n:])
		return txt, n

	case '[':
		n, m, n_, n__ := 0, 0, 0, 0
		switch config.ct {
		case LengthPrefix:
			n_, n__ = n+32, n+32
		case Stream:
			n__ += arrayStart(out[n__:])
		}

		var ln int
		if txt = skipWS(txt[1:], config.ws); len(txt) == 0 {
			panic("cbor scanner expected ']'")
		} else if txt[0] != ']' {
			for {
				txt, m = json2cbor(txt, out[n__:], config)
				n__ += m
				ln++
				if txt = skipWS(txt, config.ws); len(txt) == 0 {
					panic("cbor scanner expected ']'")
				} else if txt[0] == ',' {
					txt = skipWS(txt[1:], config.ws)
				} else if txt[0] == ']' {
					break
				} else {
					panic("cbor scanner expected ']'")
				}
			}
		}
		n__ += breakStop(out[n__:])
		switch config.ct {
		case LengthPrefix:
			x := valuint642cbor(uint64(ln), out[n:])
			out[n] = (out[n] & 0x1f) | cborType4 // fix type from type0->type4
			n += x
			n += copy(out[n:], out[n_:n__])
		case Stream:
			n = n__
		}
		return txt[1:], n

	case '{':
		n, m, n_, n__ := 0, 0, 0, 0
		switch config.ct {
		case LengthPrefix:
			n_, n__ = n+32, n+32
		case Stream:
			n__ += mapStart(out[n__:])
		}

		var ln int
		txt = skipWS(txt[1:], config.ws)
		if txt[0] == '}' {
			// pass
		} else if txt[0] != '"' {
			panic("cbor scanner expected property key")
		} else {
			for {
				// 16 reserved for cbor hdr
				txt, m = scanString(txt, out[n__+16:])
				n__ += valtext2cbor(bytes2str(out[n__+16:n__+16+m]), out[n__:])

				if txt = skipWS(txt, config.ws); len(txt) == 0 || txt[0] != ':' {
					panic("cbor scanner expected property colon")
				}
				txt, m = json2cbor(skipWS(txt[1:], config.ws), out[n__:], config)
				n__ += m
				ln++

				if txt = skipWS(txt, config.ws); len(txt) == 0 {
					panic("cbor scanner expected '}'")
				} else if txt[0] == ',' {
					txt = skipWS(txt[1:], config.ws)
				} else if txt[0] == '}' {
					break
				} else {
					panic("cbor scanner expected '}'")
				}
			}
		}
		n__ += breakStop(out[n__:])
		switch config.ct {
		case LengthPrefix:
			x := valuint642cbor(uint64(ln), out[n:])
			out[n] = (out[n] & 0x1f) | cborType5 // fix type from type0->type5
			n += x
			n += copy(out[n:], out[n_:n__])
		case Stream:
			n = n__
		}
		return txt[1:], n

	default:
		panic("cbor scanner expected token")
	}
}
<commit_msg>bugfix: json2cbor() break-stop is not required for LengthPrefix.<commit_after>package gson

import "strconv"
import "math"
import "unicode/utf8"
import "encoding/binary"

var nullStr = "null"
var trueStr = "true"
var falseStr = "false"

func json2cbor(txt string, out []byte, config *Config) (string, int) {
	txt = skipWS(txt, config.ws)

	if len(txt) < 1 {
		panic("cbor scanner empty json text")
	}

	if numCheck[txt[0]] == 1 {
		return jsonNumToCbor(txt, out, config)
	}

	switch txt[0] {
	case 'n':
		if len(txt) >= 4 && txt[:4] == nullStr {
			n := cborNull(out)
			return txt[4:], n
		}
		panic("cbor scanner expected null")

	case 't':
		if len(txt) >= 4 && txt[:4] == trueStr {
			n := cborTrue(out)
			return txt[4:], n
		}
		panic("cbor scanner expected true")

	case 'f':
		if len(txt) >= 5 && txt[:5] == falseStr {
			n := cborFalse(out)
			return txt[5:], n
		}
		panic("cbor scanner expected false")

	case '"':
		if config.jsonString {
			return jsonStringToCbor(txt, out)
		}
		n := 0
		txt, x := scanString(txt, out[n+16:]) // 16 reserved for cbor hdr
		n += valtext2cbor(bytes2str(out[n+16:n+16+x]), out[n:])
		return txt, n

	case '[':
		n, m, n_, n__ := 0, 0, 0, 0
		switch config.ct {
		case LengthPrefix:
			n_, n__ = n+32, n+32
		case Stream:
			n__ += arrayStart(out[n__:])
		}

		var ln int
		if txt = skipWS(txt[1:], config.ws); len(txt) == 0 {
			panic("cbor scanner expected ']'")
		} else if txt[0] != ']' {
			for {
				txt, m = json2cbor(txt, out[n__:], config)
				n__ += m
				ln++
				if txt = skipWS(txt, config.ws); len(txt) == 0 {
					panic("cbor scanner expected ']'")
				} else if txt[0] == ',' {
					txt = skipWS(txt[1:], config.ws)
				} else if txt[0] == ']' {
					break
				} else {
					panic("cbor scanner expected ']'")
				}
			}
		}
		switch config.ct {
		case LengthPrefix:
			x := valuint642cbor(uint64(ln), out[n:])
			out[n] = (out[n] & 0x1f) | cborType4 // fix type from type0->type4
			n += x
			n += copy(out[n:], out[n_:n__])
		case Stream:
			n__ += breakStop(out[n__:])
			n = n__
		}
		return txt[1:], n

	case '{':
		n, m, n_, n__ := 0, 0, 0, 0
		switch config.ct {
		case LengthPrefix:
			n_, n__ = n+32, n+32
		case Stream:
			n__ += mapStart(out[n__:])
		}

		var ln int
		txt = skipWS(txt[1:], config.ws)
		if txt[0] == '}' {
			// pass
		} else if txt[0] != '"' {
			panic("cbor scanner expected property key")
		} else {
			for {
				// 16 reserved for cbor hdr
				txt, m = scanString(txt, out[n__+16:])
				n__ += valtext2cbor(bytes2str(out[n__+16:n__+16+m]), out[n__:])

				if txt = skipWS(txt, config.ws); len(txt) == 0 || txt[0] != ':' {
					panic("cbor scanner expected property colon")
				}
				txt, m = json2cbor(skipWS(txt[1:], config.ws), out[n__:], config)
				n__ += m
				ln++

				if txt = skipWS(txt, config.ws); len(txt) == 0 {
					panic("cbor scanner expected '}'")
				} else if txt[0] == ',' {
					txt = skipWS(txt[1:], config.ws)
				} else if txt[0] == '}' {
					break
				} else {
					panic("cbor scanner expected '}'")
				}
			}
		}
		switch config.ct {
		case LengthPrefix:
			x := valuint642cbor(uint64(ln), out[n:])
			out[n] = (out[n] & 0x1f) | cborType5 // fix type from type0->type5
			n += x
			n += copy(out[n:], out[n_:n__])
		case Stream:
			n__ += breakStop(out[n__:])
			n = n__
		}
		return txt[1:], n

	default:
		panic("cbor scanner expected token")
	}
}

{'fraction_non_alphanumeric': 0.18612647630438034, 'fraction_numerical': 0.02795634624009568, 'mean_word_length': 2.8492520138089756, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7407430', 'n_tokens_mistral': 3285, 'n_tokens_neox': 2722, 'n_words': 699}