<commit_before>package jsoniter

func (iter *Iterator) ReadObject() (ret string) {
	c := iter.nextToken()
	switch c {
	case 'n':
		iter.skipFixedBytes(3)
		return "" // null
	case '{':
		c = iter.nextToken()
		if c == '"' {
			iter.unreadByte()
			return string(iter.readObjectFieldAsBytes())
		}
		if c == '}' {
			return "" // end of object
		}
		iter.reportError("ReadObject", `expect " after {`)
		return
	case ',':
		return string(iter.readObjectFieldAsBytes())
	case '}':
		return "" // end of object
	default:
		iter.reportError("ReadObject", `expect { or , or } or n`)
		return
	}
}

func (iter *Iterator) readFieldHash() int32 {
	hash := 0x811c9dc5
	c := iter.nextToken()
	if c == '"' {
		for {
			for i := iter.head; i < iter.tail; i++ {
				// require ascii string and no escape
				b := iter.buf[i]
				if b == '"' {
					iter.head = i+1
					c = iter.nextToken()
					if c != ':' {
						iter.reportError("readFieldHash", `expect :, but found ` + string([]byte{c}))
					}
					return int32(hash)
				}
				hash ^= int(b)
				hash *= 0x1000193
			}
			if !iter.loadMore() {
				iter.reportError("readFieldHash", `incomplete field name`)
				return 0
			}
		}
	}
	iter.reportError("readFieldHash", `expect ", but found ` + string([]byte{c}))
	return 0
}

func calcHash(str string) int32 {
	hash := 0x811c9dc5
	for _, b := range str {
		hash ^= int(b)
		hash *= 0x1000193
	}
	return int32(hash)
}

func (iter *Iterator) ReadObjectCB(callback func(*Iterator, string) bool) bool {
	c := iter.nextToken()
	if c == '{' {
		c = iter.nextToken()
		if c == '"' {
			iter.unreadByte()
			field := string(iter.readObjectFieldAsBytes())
			if !callback(iter, field) {
				return false
			}
			for iter.nextToken() == ',' {
				field := string(iter.readObjectFieldAsBytes())<commit_msg>fix issue on 32bit platform<commit_after>package jsoniter

func (iter *Iterator) ReadObject() (ret string) {
	c := iter.nextToken()
	switch c {
	case 'n':
		iter.skipFixedBytes(3)
		return "" // null
	case '{':
		c = iter.nextToken()
		if c == '"' {
			iter.unreadByte()
			return string(iter.readObjectFieldAsBytes())
		}
		if c == '}' {
			return "" // end of object
		}
		iter.reportError("ReadObject", `expect " after {`)
		return
	case ',':
		return string(iter.readObjectFieldAsBytes())
	case '}':
		return "" // end of object
	default:
		iter.reportError("ReadObject", `expect { or , or } or n`)
		return
	}
}

func (iter *Iterator) readFieldHash() int32 {
	hash := int64(0x811c9dc5)
	c := iter.nextToken()
	if c == '"' {
		for {
			for i := iter.head; i < iter.tail; i++ {
				// require ascii string and no escape
				b := iter.buf[i]
				if b == '"' {
					iter.head = i+1
					c = iter.nextToken()
					if c != ':' {
						iter.reportError("readFieldHash", `expect :, but found ` + string([]byte{c}))
					}
					return int32(hash)
				}
				hash ^= int(b)
				hash *= 0x1000193
			}
			if !iter.loadMore() {
				iter.reportError("readFieldHash", `incomplete field name`)
				return 0
			}
		}
	}
	iter.reportError("readFieldHash", `expect ", but found ` + string([]byte{c}))
	return 0
}

func calcHash(str string) int32 {
	hash := int64(0x811c9dc5)
	for _, b := range str {
		hash ^= int(b)
		hash *= 0x1000193
	}
	return int32(hash)
}

func (iter *Iterator) ReadObjectCB(callback func(*Iterator, string) bool) bool {
	c := iter.nextToken()
	if c == '{' {
		c = iter.nextToken()
		if c == '"' {
			iter.unreadByte()
			field := string(iter.readObjectFieldAsBytes())
			if !callback(iter, field) {
				return false
			}
			for iter.nextToken() == ',' {
				field := string(iter.readObjectFieldAsBytes())
{'fraction_non_alphanumeric': 0.1688888888888889, 'fraction_numerical': 0.02388888888888889, 'mean_word_length': 3.032474804031355, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3476964', 'n_tokens_mistral': 1550, 'n_tokens_neox': 1311, 'n_words': 313}