<commit_before>#include <ctype.h> // isspace isdigit
#include <stdarg.h> // va_list va_start
#include <stdbool.h> // bool true false
#include <stddef.h> // NULL
#include <stdlib.h> // atoi atof
#include <string.h> // strdup

#include "encoding.h"
#include "memory.h"
#include "parser.h"

typedef struct {
	const char *key;
	Store *value;
} Entry;

static Store *parseValue(const char *input, StoreParseState *state);
static Store *parseString(const char *input, StoreParseState *state);
static Store *parseInt(const char *input, StoreParseState *state);
static Store *parseFloat(const char *input, StoreParseState *state);
static Store *parseList(const char *input, StoreParseState *state);
static Store *parseElements(const char *input, StoreParseState *state);
static Store *parseMap(const char *input, StoreParseState *state);
static Store *parseEntries(const char *input, StoreParseState *state);
static Entry *parseEntry(const char *input, StoreParseState *state);
static StoreDynamicString parseDigits(const char *input, StoreParseState *state);
static StoreDynamicString parseFloating(const char *input, StoreParseState *state);
static StoreDynamicString parseExponential(const char *input, StoreParseState *state);
static StoreDynamicString parseShortString(const char *input, StoreParseState *state);
static StoreDynamicString parseLongString(const char *input, StoreParseState *state);
static StoreDynamicString parseLongStringChunk(const char *input, StoreParseState *state);
static char parseTerminal(const char *input, StoreParseState *state);
static char parseHex(const char *input, StoreParseState *state);
static char parseDigit(const char *input, StoreParseState *state);
static char parseDelimiter(const char *input, StoreParseState *state);
static void report(bool success, StoreParseState *parentState, StoreParseState *currentState, const char *type, const char *message, ...);
static void freeParseReport(StoreParseReport *lastReport);
static bool isHex(char c);
static bool isSeparator(char c);

Store *StoreParse(const char *input, StoreParseState *state)
{
	StoreParseState storeState;
	storeState.position = state->position;
	storeState.lastReport = NULL;

	// Try to parse the store as a value
	Store *valueStore = parseValue(input, &storeState);
	if(valueStore != NULL) {
		report(true, state, &storeState, "store", "parsed value store of type %s", StoreGetTypeName(valueStore));
		return valueStore;
	}

	report(false, state, &storeState, "store", "expected value");
	return NULL;
}

/**
 * value	: nonterminal* int separator
 * 			| nonterminal* float separator
 * 			| nonterminal* string separator
 * 			| nonterminal* list separator
 * 			| nonterminal* struct separator
 */
static Store *parseValue(const char *input, StoreParseState *state)
{<commit_msg>changed StoreParse to parse both values and entries<commit_after>#include <ctype.h> // isspace isdigit
#include <stdarg.h> // va_list va_start
#include <stdbool.h> // bool true false
#include <stddef.h> // NULL
#include <stdlib.h> // atoi atof
#include <string.h> // strdup

#include "encoding.h"
#include "memory.h"
#include "parser.h"

typedef struct {
	const char *key;
	Store *value;
} Entry;

static Store *parseValue(const char *input, StoreParseState *state);
static Store *parseString(const char *input, StoreParseState *state);
static Store *parseInt(const char *input, StoreParseState *state);
static Store *parseFloat(const char *input, StoreParseState *state);
static Store *parseList(const char *input, StoreParseState *state);
static Store *parseElements(const char *input, StoreParseState *state);
static Store *parseMap(const char *input, StoreParseState *state);
static Store *parseEntries(const char *input, StoreParseState *state);
static Entry *parseEntry(const char *input, StoreParseState *state);
static StoreDynamicString parseDigits(const char *input, StoreParseState *state);
static StoreDynamicString parseFloating(const char *input, StoreParseState *state);
static StoreDynamicString parseExponential(const char *input, StoreParseState *state);
static StoreDynamicString parseShortString(const char *input, StoreParseState *state);
static StoreDynamicString parseLongString(const char *input, StoreParseState *state);
static StoreDynamicString parseLongStringChunk(const char *input, StoreParseState *state);
static char parseTerminal(const char *input, StoreParseState *state);
static char parseHex(const char *input, StoreParseState *state);
static char parseDigit(const char *input, StoreParseState *state);
static char parseDelimiter(const char *input, StoreParseState *state);
static void report(bool success, StoreParseState *parentState, StoreParseState *currentState, const char *type, const char *message, ...);
static void freeParseReport(StoreParseReport *lastReport);
static bool isHex(char c);
static bool isSeparator(char c);

/**
 * store	: value '\0'
 * 			| entries '\0'
 */
Store *StoreParse(const char *input, StoreParseState *state)
{
	StoreParseState storeState;
	storeState.position = state->position;
	storeState.lastReport = NULL;

	Store *valueStore = parseValue(input, &storeState);
	if(valueStore != NULL) {
		char c = parseTerminal(input, &storeState);
		if(c == '\0') {
			// make sure it's and actual EOF, not just a parseTerminal failure
			if(input[storeState.position.index] == '\0') {
				report(true, state, &storeState, "store", "parsed value store of type %s", StoreGetTypeName(valueStore));
				state->position = storeState.position;
				return valueStore;
			}
		}

		StoreParseState valueState;
		valueState.position = storeState.position;
		valueState.lastReport = NULL;
		report(false, &storeState, &valueState, "value", "expected termination by end of input, but got '%c'", c);
		StoreFree(valueStore);
	}

	storeState.position = state->position;
	Store *entriesStore = parseEntries(input, &storeState);
	if(entriesStore != NULL) {
		char c = parseTerminal(input, &storeState);
		if(c == '\0') {
			// make sure it's and actual EOF, not just a parseTerminal failure
			if(input[storeState.position.index] == '\0') {
				report(true, state, &storeState, "entries", "parsed entries store");
				state->position = storeState.position;
				return valueStore;
			}
		}

		StoreParseState entriesState;
		entriesState.position = storeState.position;
		entriesState.lastReport = NULL;
		report(false, &storeState, &entriesState, "entries", "expected termination by end of input, but got '%c'", c);
		StoreFree(valueStore);
	}

	report(false, state, &storeState, "store", "expected value or entries");
	return NULL;
}

/**
 * value	: nonterminal* int separator
 * 			| nonterminal* float separator
 * 			| nonterminal* string separator
 * 			| nonterminal* list separator
 * 			| nonterminal* struct separator
 */
static Store *parseValue(const char *input, StoreParseState *state)
{
{'fraction_non_alphanumeric': 0.11207022677395757, 'fraction_numerical': 0.0008778346744696416, 'mean_word_length': 5.961303462321792, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6226687', 'n_tokens_mistral': 2055, 'n_tokens_neox': 1967, 'n_words': 639}