<commit_before>package jsonparser

import (
	"bytes"
	"errors"
	"fmt"
	d "runtime/debug"
	"strconv"
)

// Find position of next character which is not ' ', ',', '}' or ']'
func nextValue(data []byte) (offset int) {
	for true {
		if len(data) == offset {
			return -1
		}

		if data[offset] != ' ' && data[offset] != '\n' && data[offset] != ',' && data[offset] != '}' && data[offset] != ']' {
			return
		}

		offset++
	}

	return -1
}

// Tries to find the end of string
// Support if string contains escaped quote symbols.
func stringEnd(data []byte) int {
	i := 0

	for true {
		sIdx := bytes.IndexByte(data[i:], '"')

		if sIdx == -1 {
			return -1
		}

		i += sIdx + 1

		// If it just escaped \", continue
		if i > 2 && data[i-2] == '\\' {
			continue
		}

		break
	}

	return i
}

// Find end of the data structure, array or object.
// For array openSym and closeSym will be '[' and ']', for object '{' and '}'
// Know about nested structures
func trailingBracket(data []byte, openSym byte, closeSym byte) int {
	level := 0
	i := 0
	ln := len(data)

	for true {
		if i >= ln {
			return -1
		}

		c := data[i]

		// If inside string, skip it
		if c == '"' {
			//sFrom := i
			i++

			se := stringEnd(data[i:])
			if se == -1 {
				return -1
			}
			i += se - 1
		}

		if c == openSym {
			level++
		} else if c == closeSym {
			level--
		}

		i++

		if level == 0 {
			break
		}
	}

	return i
}

// Data types available in valid JSON data.
const (
	NotExist = iota
	String
	Number
	Object
	Array
	Boolean
	Null
)

/*
Get - Receives data structure, and key path to extract value from.

Returns:
`value` - Pointer to original data structure containing key value, or just empty slice if nothing found or error
`dataType` -    Can be: `NotExist`, `String`, `Number`, `Object`, `Array`, `Boolean` or `Null`
`offset` - Offset from provided data structure where key value ends. Used mostly internally, for example for `ArrayEach` helper.
`err` - If key not found or any other parsing issue it should return error. If key not found it also sets `dataType` to `NotExist`

Accept multiple keys to specify path to JSON value (in case of quering nested structures).
If no keys provided it will try to extract closest JSON value (simple ones or object/array), useful for reading streams or arrays, see `ArrayEach` implementation.
*/
func Get(data []byte, keys ...string) (value []byte, dataType int, offset int, err error) {
	defer func() {
		if r := recover(); r != nil {
			err = fmt.Errorf("Unhandler JSON parsing error: %v, %s", r, string(d.Stack()))
		}
	}()

	ln := len(data)

	if len(keys) > 0 {
		for ki, k := range keys {
			lk := len(k)

			if ki > 0 {
				// Only objects can have nested keys
				if data[offset] == '{' {
					// Limiting scope for the next key search
					endOffset := trailingBracket(data[offset:], '{', '}')
					data = data[offset : offset+endOffset]
					offset = 0
				} else {
					return []byte{}, NotExist, -1, errors.New("Key path not found")
				}
			}

			for true {
				if idx := bytes.Index(data[offset:], []byte(k)); idx != -1 && (ln-(offset+idx+lk+2)) > 0 {
					offset += idx

					if data[offset+lk] == '"' && data[offset-1] == '"' && data[offset+lk+1] == ':' {
						offset += lk + 2
						nO := nextValue(data[offset:])

						if nO == -1 {
							return []byte{}, NotExist, -1, errors.New("Malformed JSON error")
						}
<commit_msg>Support escaped string in key names<commit_after>package jsonparser

import (
	"bytes"
	"errors"
	"fmt"
	d "runtime/debug"
	"strconv"
)

// Find position of next character which is not ' ', ',', '}' or ']'
func nextValue(data []byte) (offset int) {
	for true {
		if len(data) == offset {
			return -1
		}

		if data[offset] != ' ' && data[offset] != '\n' && data[offset] != ',' && data[offset] != '}' && data[offset] != ']' {
			return
		}

		offset++
	}

	return -1
}

// Tries to find the end of string
// Support if string contains escaped quote symbols.
func stringEnd(data []byte) int {
	i := 0

	for true {
		sIdx := bytes.IndexByte(data[i:], '"')

		if sIdx == -1 {
			return -1
		}

		i += sIdx + 1

		// If it just escaped \", continue
		if i > 2 && data[i-2] == '\\' {
			continue
		}

		break
	}

	return i
}

// Find end of the data structure, array or object.
// For array openSym and closeSym will be '[' and ']', for object '{' and '}'
// Know about nested structures
func trailingBracket(data []byte, openSym byte, closeSym byte) int {
	level := 0
	i := 0
	ln := len(data)

	for true {
		if i >= ln {
			return -1
		}

		c := data[i]

		// If inside string, skip it
		if c == '"' {
			//sFrom := i
			i++

			se := stringEnd(data[i:])
			if se == -1 {
				return -1
			}
			i += se - 1
		}

		if c == openSym {
			level++
		} else if c == closeSym {
			level--
		}

		i++

		if level == 0 {
			break
		}
	}

	return i
}

// Data types available in valid JSON data.
const (
	NotExist = iota
	String
	Number
	Object
	Array
	Boolean
	Null
)

/*
Get - Receives data structure, and key path to extract value from.

Returns:
`value` - Pointer to original data structure containing key value, or just empty slice if nothing found or error
`dataType` -    Can be: `NotExist`, `String`, `Number`, `Object`, `Array`, `Boolean` or `Null`
`offset` - Offset from provided data structure where key value ends. Used mostly internally, for example for `ArrayEach` helper.
`err` - If key not found or any other parsing issue it should return error. If key not found it also sets `dataType` to `NotExist`

Accept multiple keys to specify path to JSON value (in case of quering nested structures).
If no keys provided it will try to extract closest JSON value (simple ones or object/array), useful for reading streams or arrays, see `ArrayEach` implementation.
*/
func Get(data []byte, keys ...string) (value []byte, dataType int, offset int, err error) {
	defer func() {
		if r := recover(); r != nil {
			err = fmt.Errorf("Unhandler JSON parsing error: %v, %s", r, string(d.Stack()))
		}
	}()

	ln := len(data)

	if len(keys) > 0 {
		for ki, k := range keys {
			lk := len(k)

			if ki > 0 {
				// Only objects can have nested keys
				if data[offset] == '{' {
					// Limiting scope for the next key search
					endOffset := trailingBracket(data[offset:], '{', '}')
					data = data[offset : offset+endOffset]
					offset = 0
				} else {
					return []byte{}, NotExist, -1, errors.New("Key path not found")
				}
			}

			for true {
				if idx := bytes.Index(data[offset:], []byte(k)); idx != -1 && (ln-(offset+idx+lk+2)) > 0 {
					offset += idx

					if data[offset+lk] == '"' && data[offset-2] != '\\' && data[offset-1] == '"' && data[offset+lk+1] == ':' {
						offset += lk + 2
						nO := nextValue(data[offset:])

						if nO == -1 {
							return []byte{}, NotExist, -1, errors.New("Malformed JSON error")
						}

{'fraction_non_alphanumeric': 0.15514705882352942, 'fraction_numerical': 0.008088235294117648, 'mean_word_length': 2.718425369054128, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '587352', 'n_tokens_mistral': 2655, 'n_tokens_neox': 2288, 'n_words': 834}