<commit_before>/*
 * Copyright (c) 2017 Intel Corporation
 *
 * SPDX-License-Identifier: Apache-2.0
 */

#include <sys/__assert.h>
#include <ctype.h>
#include <errno.h>
#include <limits.h>
#include <sys/printk.h>
#include <sys/util.h>
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
#include <zephyr/types.h>

#include <data/json.h>

struct token {
	enum json_tokens type;
	char *start;
	char *end;
};

struct lexer {
	void *(*state)(struct lexer *lexer);
	char *start;
	char *pos;
	char *end;
	struct token token;
};

struct json_obj {
	struct lexer lexer;
};

struct json_obj_key_value {
	const char *key;
	size_t key_len;
	struct token value;
};

static bool lexer_consume(struct lexer *lexer, struct token *token,
			  enum json_tokens empty_token)
{
	if (lexer->token.type == empty_token) {
		return false;
	}

	*token = lexer->token;
	lexer->token.type = empty_token;

	return true;
}

static bool lexer_next(struct lexer *lexer, struct token *token)
{
	while (lexer->state) {
		if (lexer_consume(lexer, token, JSON_TOK_NONE)) {
			return true;
		}

		lexer->state = lexer->state(lexer);
	}

	return lexer_consume(lexer, token, JSON_TOK_EOF);
}

static void *lexer_json(struct lexer *lexer);

static void emit(struct lexer *lexer, enum json_tokens token)
{
	lexer->token.type = token;
	lexer->token.start = lexer->start;
	lexer->token.end = lexer->pos;
	lexer->start = lexer->pos;
}

static int next(struct lexer *lexer)
{
	if (lexer->pos >= lexer->end) {
		lexer->pos = lexer->end + 1;

		return '\0';
	}

	return *lexer->pos++;
}

static void ignore(struct lexer *lexer)
{
	lexer->start = lexer->pos;
}

static void backup(struct lexer *lexer)
{
	lexer->pos--;
}

static int peek(struct lexer *lexer)
{
	int chr = next(lexer);

	backup(lexer);

	return chr;
}

static void *lexer_string(struct lexer *lexer)
{
	ignore(lexer);

	while (true) {
		int chr = next(lexer);

		if (chr == '\0') {
			emit(lexer, JSON_TOK_ERROR);
			return NULL;
		}

		if (chr == '\\') {
			switch (next(lexer)) {
			case '"':
			case '\\':
			case '/':
			case 'b':
			case 'f':
			case 'n':
			case 'r':
			case 't':
				continue;
			case 'u':
				if (!isxdigit(next(lexer))) {
					goto error;
				}

				if (!isxdigit(next(lexer))) {
					goto error;
				}

				if (!isxdigit(next(lexer))) {
					goto error;
				}

				if (!isxdigit(next(lexer))) {
					goto error;
				}

				break;
			default:
				goto error;
			}
		}

		if (chr == '"') {
			backup(lexer);
			emit(lexer, JSON_TOK_STRING);

			next(lexer);
			ignore(lexer);

			return lexer_json;
		}
	}

error:
	emit(lexer, JSON_TOK_ERROR);
	return NULL;
}

static int accept_run(struct lexer *lexer, const char *run)
{
	for (; *run; run++) {
		if (next(lexer) != *run) {
			return -EINVAL;
		}
	}

	return 0;
}

static void *lexer_boolean(struct lexer *lexer)
{
	backup(lexer);

	switch (next(lexer)) {
	case 't':
		if (!accept_run(lexer, "rue")) {
			emit(lexer, JSON_TOK_TRUE);
			return lexer_json;
		}
		break;
	case 'f':
		if (!accept_run(lexer, "alse")) {
			emit(lexer, JSON_TOK_FALSE);
			return lexer_json;
		}
		break;
	}

	emit(lexer, JSON_TOK_ERROR);
	return NULL;
}

static void *lexer_null(struct lexer *lexer)
{
	if (accept_run(lexer, "ull") < 0) {
		emit(lexer, JSON_TOK_ERROR);
		return NULL;
	}

	emit(lexer, JSON_TOK_NULL);
	return lexer_json;
}

static void *lexer_number(struct lexer *lexer)
{
	while (true) {
		int chr = next(lexer);

		if (isdigit(chr) || chr == '.') {
			continue;
		}

		backup(lexer);
		emit(lexer, JSON_TOK_NUMBER);

		return lexer_json;
	}
}

static void *lexer_json(struct lexer *lexer)
{
	while (true) {
		int chr = next(lexer);

		switch (chr) {
		case '\0':
			emit(lexer, JSON_TOK_EOF);
			return NULL;
		case '}':
		case '{':
		case '[':
		case ']':
		case ',':
		case ':':
			emit(lexer, (enum json_tokens)chr);
			return lexer_json;
		case '"':
			return lexer_string;
		case 'n':
			return lexer_null;
		case 't':
		case 'f':
			return lexer_boolean;
		case '-':
			if (isdigit(peek(lexer))) {
				return lexer_number;
			}

			__fallthrough;
		default:
			if (isspace(chr)) {
				ignore(lexer);
				continue;
			}

			if (isdigit(chr)) {
				return lexer_number;
			}

			emit(lexer, JSON_TOK_ERROR);
			return NULL;
		}
	}
}

static void lexer_init(struct lexer *lexer, char *data, size_t len)
{
	lexer->state = lexer_json;
	lexer->start = data;
	lexer->pos = data;
	lexer->end = data + len;
	lexer->token.type = JSON_TOK_NONE;
}

static int obj_init(struct json_obj *json, char *data, size_t len)
{
	struct token token;

	lexer_init(&json->lexer, data, len);

	if (!lexer_next(&json->lexer, &token)) {
		return -EINVAL;
	}

	if (token.type != JSON_TOK_OBJECT_START) {
		return -EINVAL;
	}

	return 0;
}

static int arr_init(struct json_obj *json, char *data, size_t len)
{
	struct token token;

	lexer_init(&json->lexer, data, len);

	if (!lexer_next(&json->lexer, &token)) {
		return -EINVAL;
	}

	if (token.type != JSON_TOK_ARRAY_START) {
		return -EINVAL;
	}

	return 0;
}

static int element_token(enum json_tokens token)
{
	switch (token) {
	case JSON_TOK_OBJECT_START:
	case JSON_TOK_ARRAY_START:
	case JSON_TOK_STRING:
	case JSON_TOK_NUMBER:
	case JSON_TOK_TRUE:
	case JSON_TOK_FALSE:
		return 0;
	default:
		return -EINVAL;
	}
}

static int obj_next(struct json_obj *json,
		    struct json_obj_key_value *kv)
{
	struct token token;

	if (!lexer_next(&json->lexer, &token)) {
		return -EINVAL;
	}

	/* Match end of object or next key */
	switch (token.type) {
	case JSON_TOK_OBJECT_END:
		kv->key = NULL;
		kv->key_len = 0;
		kv->value = token;

		return 0;
	case JSON_TOK_COMMA:
		if (!lexer_next(&json->lexer, &token)) {
			return -EINVAL;
		}

		if (token.type != JSON_TOK_STRING) {
			return -EINVAL;
		}

		__fallthrough;
	case JSON_TOK_STRING:
		kv->key = token.start;
		kv->key_len = (size_t)(token.end - token.start);
		break;
	default:
		return -EINVAL;
	}

	/* Match : after key */
	if (!lexer_next(&json->lexer, &token)) {
		return -EINVAL;
	}

	if (token.type != JSON_TOK_COLON) {
		return -EINVAL;
	}

	/* Match value */
	if (!lexer_next(&json->lexer, &kv->value)) {
		return -EINVAL;
	}

	return element_token(kv->value.type);
}

static int arr_next(struct json_obj *json, struct token *value)
{
	if (!lexer_next(&json->lexer, value)) {
		return -EINVAL;
	}

	if (value->type == JSON_TOK_ARRAY_END) {
		return 0;
	}

	if (value->type == JSON_TOK_COMMA) {
		if (!lexer_next(&json->lexer, value)) {
			return -EINVAL;
		}
	}

	return element_token(value->type);
}

static int decode_num(const struct token *token, int32_t *num)
{
	/* FIXME: strtod() is not available in newlib/minimal libc,
	 * so using strtol() here.
	 */
	char *endptr;
	char prev_end;<commit_msg>json: Fix rule 5.7 violations (Tag name should be unique)<commit_after>/*
 * Copyright (c) 2017 Intel Corporation
 *
 * SPDX-License-Identifier: Apache-2.0
 */

#include <sys/__assert.h>
#include <ctype.h>
#include <errno.h>
#include <limits.h>
#include <sys/printk.h>
#include <sys/util.h>
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
#include <zephyr/types.h>

#include <data/json.h>

struct token {
	enum json_tokens type;
	char *start;
	char *end;
};

struct lexer {
	void *(*state)(struct lexer *lex);
	char *start;
	char *pos;
	char *end;
	struct token tok;
};

struct json_obj {
	struct lexer lex;
};

struct json_obj_key_value {
	const char *key;
	size_t key_len;
	struct token value;
};

static bool lexer_consume(struct lexer *lex, struct token *tok,
			  enum json_tokens empty_token)
{
	if (lex->tok.type == empty_token) {
		return false;
	}

	*tok = lex->tok;
	lex->tok.type = empty_token;

	return true;
}

static bool lexer_next(struct lexer *lex, struct token *tok)
{
	while (lex->state) {
		if (lexer_consume(lex, tok, JSON_TOK_NONE)) {
			return true;
		}

		lex->state = lex->state(lex);
	}

	return lexer_consume(lex, tok, JSON_TOK_EOF);
}

static void *lexer_json(struct lexer *lex);

static void emit(struct lexer *lex, enum json_tokens token)
{
	lex->tok.type = token;
	lex->tok.start = lex->start;
	lex->tok.end = lex->pos;
	lex->start = lex->pos;
}

static int next(struct lexer *lex)
{
	if (lex->pos >= lex->end) {
		lex->pos = lex->end + 1;

		return '\0';
	}

	return *lex->pos++;
}

static void ignore(struct lexer *lex)
{
	lex->start = lex->pos;
}

static void backup(struct lexer *lex)
{
	lex->pos--;
}

static int peek(struct lexer *lex)
{
	int chr = next(lex);

	backup(lex);

	return chr;
}

static void *lexer_string(struct lexer *lex)
{
	ignore(lex);

	while (true) {
		int chr = next(lex);

		if (chr == '\0') {
			emit(lex, JSON_TOK_ERROR);
			return NULL;
		}

		if (chr == '\\') {
			switch (next(lex)) {
			case '"':
			case '\\':
			case '/':
			case 'b':
			case 'f':
			case 'n':
			case 'r':
			case 't':
				continue;
			case 'u':
				if (!isxdigit(next(lex))) {
					goto error;
				}

				if (!isxdigit(next(lex))) {
					goto error;
				}

				if (!isxdigit(next(lex))) {
					goto error;
				}

				if (!isxdigit(next(lex))) {
					goto error;
				}

				break;
			default:
				goto error;
			}
		}

		if (chr == '"') {
			backup(lex);
			emit(lex, JSON_TOK_STRING);

			next(lex);
			ignore(lex);

			return lexer_json;
		}
	}

error:
	emit(lex, JSON_TOK_ERROR);
	return NULL;
}

static int accept_run(struct lexer *lex, const char *run)
{
	for (; *run; run++) {
		if (next(lex) != *run) {
			return -EINVAL;
		}
	}

	return 0;
}

static void *lexer_boolean(struct lexer *lex)
{
	backup(lex);

	switch (next(lex)) {
	case 't':
		if (!accept_run(lex, "rue")) {
			emit(lex, JSON_TOK_TRUE);
			return lexer_json;
		}
		break;
	case 'f':
		if (!accept_run(lex, "alse")) {
			emit(lex, JSON_TOK_FALSE);
			return lexer_json;
		}
		break;
	}

	emit(lex, JSON_TOK_ERROR);
	return NULL;
}

static void *lexer_null(struct lexer *lex)
{
	if (accept_run(lex, "ull") < 0) {
		emit(lex, JSON_TOK_ERROR);
		return NULL;
	}

	emit(lex, JSON_TOK_NULL);
	return lexer_json;
}

static void *lexer_number(struct lexer *lex)
{
	while (true) {
		int chr = next(lex);

		if (isdigit(chr) || chr == '.') {
			continue;
		}

		backup(lex);
		emit(lex, JSON_TOK_NUMBER);

		return lexer_json;
	}
}

static void *lexer_json(struct lexer *lex)
{
	while (true) {
		int chr = next(lex);

		switch (chr) {
		case '\0':
			emit(lex, JSON_TOK_EOF);
			return NULL;
		case '}':
		case '{':
		case '[':
		case ']':
		case ',':
		case ':':
			emit(lex, (enum json_tokens)chr);
			return lexer_json;
		case '"':
			return lexer_string;
		case 'n':
			return lexer_null;
		case 't':
		case 'f':
			return lexer_boolean;
		case '-':
			if (isdigit(peek(lex))) {
				return lexer_number;
			}

			__fallthrough;
		default:
			if (isspace(chr)) {
				ignore(lex);
				continue;
			}

			if (isdigit(chr)) {
				return lexer_number;
			}

			emit(lex, JSON_TOK_ERROR);
			return NULL;
		}
	}
}

static void lexer_init(struct lexer *lex, char *data, size_t len)
{
	lex->state = lexer_json;
	lex->start = data;
	lex->pos = data;
	lex->end = data + len;
	lex->tok.type = JSON_TOK_NONE;
}

static int obj_init(struct json_obj *json, char *data, size_t len)
{
	struct token tok;

	lexer_init(&json->lex, data, len);

	if (!lexer_next(&json->lex, &tok)) {
		return -EINVAL;
	}

	if (tok.type != JSON_TOK_OBJECT_START) {
		return -EINVAL;
	}

	return 0;
}

static int arr_init(struct json_obj *json, char *data, size_t len)
{
	struct token tok;

	lexer_init(&json->lex, data, len);

	if (!lexer_next(&json->lex, &tok)) {
		return -EINVAL;
	}

	if (tok.type != JSON_TOK_ARRAY_START) {
		return -EINVAL;
	}

	return 0;
}

static int element_token(enum json_tokens token)
{
	switch (token) {
	case JSON_TOK_OBJECT_START:
	case JSON_TOK_ARRAY_START:
	case JSON_TOK_STRING:
	case JSON_TOK_NUMBER:
	case JSON_TOK_TRUE:
	case JSON_TOK_FALSE:
		return 0;
	default:
		return -EINVAL;
	}
}

static int obj_next(struct json_obj *json,
		    struct json_obj_key_value *kv)
{
	struct token tok;

	if (!lexer_next(&json->lex, &tok)) {
		return -EINVAL;
	}

	/* Match end of object or next key */
	switch (tok.type) {
	case JSON_TOK_OBJECT_END:
		kv->key = NULL;
		kv->key_len = 0;
		kv->value = tok;

		return 0;
	case JSON_TOK_COMMA:
		if (!lexer_next(&json->lex, &tok)) {
			return -EINVAL;
		}

		if (tok.type != JSON_TOK_STRING) {
			return -EINVAL;
		}

		__fallthrough;
	case JSON_TOK_STRING:
		kv->key = tok.start;
		kv->key_len = (size_t)(tok.end - tok.start);
		break;
	default:
		return -EINVAL;
	}

	/* Match : after key */
	if (!lexer_next(&json->lex, &tok)) {
		return -EINVAL;
	}

	if (tok.type != JSON_TOK_COLON) {
		return -EINVAL;
	}

	/* Match value */
	if (!lexer_next(&json->lex, &kv->value)) {
		return -EINVAL;
	}

	return element_token(kv->value.type);
}

static int arr_next(struct json_obj *json, struct token *value)
{
	if (!lexer_next(&json->lex, value)) {
		return -EINVAL;
	}

	if (value->type == JSON_TOK_ARRAY_END) {
		return 0;
	}

	if (value->type == JSON_TOK_COMMA) {
		if (!lexer_next(&json->lex, value)) {
			return -EINVAL;
		}
	}

	return element_token(value->type);
}

static int decode_num(const struct token *token, int32_t *num)
{
	/* FIXME: strtod() is not available in newlib/minimal libc,
	 * so using strtol() here.
	 */
	char *endptr;
	char prev_end;
{'fraction_non_alphanumeric': 0.15646773704937897, 'fraction_numerical': 0.0031808542865798244, 'mean_word_length': 3.3537751401252884, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 119, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1720790', 'n_tokens_mistral': 6038, 'n_tokens_neox': 5343, 'n_words': 1254}