<commit_before>#include "json.h"

#define nil ((void*)0)
#define must(b) do { if (!(b)) return 0; } while (0)

typedef struct Parser Parser;

struct Parser {
	char *s;
	int  n;
	JSON *j;
	int  nj;
};

static int parsevalue(Parser*, JSON*, JSON**);


static int
consume(Parser *p, char *s)
{
	while (*s) {
		must(*p->s == *s);
		p->s++;
		s++;
	}
	return 1;
}


static void
skipws(Parser *p)
{
	while (*p->s == ' ' || *p->s == '\t' || *p->s == '\n' || *p->s == '\r') {
		p->s++;
	}
}


static int
scandigits(Parser *p)
{
	int d = 0;
	while ('0' <= *p->s && *p->s <= '9') {
		p->s++;
		d = 1;
	}
	return d;
}


static int
scanhex4(Parser *p)
{
	int i;
	for (i = 0; i < 4; i++) {
		char c = *p->s++;
		must(('0'<=c && c<='9') || ('a'<=c && c<='f') || ('A'<=c && c<='F'));
	}
	return 1;
}


static JSON *
inititem(Parser *p, JSON *parent, JSON **prev, char type)
{
	p->n++;
	if (p->nj > 0) {
		JSON *v = p->j;
		p->j++;
		p->nj--;
		v->type = type;
		v->src = p->s;
		v->parent = parent;
		v->next = nil;
		v->prev = *prev;
		if (*prev) {
			(*prev)->next = v;
		}
		*prev = v;
		return v;
	}
	return nil;
}


static int
parseword(Parser *p, JSON *parent, JSON **prev, char *lit)
{
	JSON *v = inititem(p, parent, prev, lit[0]);
	must(consume(p, lit));
	if (v) {
		v->end = p->s;
	}
	return 1;
}


static int
parsestring(Parser *p, JSON *parent, JSON **prev)
{
	JSON *v = inititem(p, parent, prev, '"');
	p->s++; /* consume " */
	while (*p->s != '"') {
		char c = *p->s;
		must(c >= ' '); /* no control chars */
		p->s++;
		if (c == '\\') {
			switch (c = *p->s++) {
			case 'b': case 'f': case 'n': case 'r':
			case 't': case '"': case '\\': case '/':
				continue;
			case 'u':
				must(scanhex4(p));
				continue;
			}
			return 0;
		}
	}
	p->s++; /* consume " */
	if (v) {
		v->end = p->s;
	}
	return 1;
}


static int
parsenumber(Parser *p, JSON *parent, JSON **prev)
{
	char c;
	JSON *v = inititem(p, parent, prev, '0');
	if (*p->s == '-') {
		p->s++;
	}
	c = *p->s;
	if (c == '0') { /* special case, single 0 */
		p->s++;
	} else {
		must(scandigits(p));
	}
	if (*p->s == '.') {
		p->s++;
		must(scandigits(p));
	}
	if (*p->s == 'e' || *p->s == 'E') {
		p->s++;
		if (*p->s == '+' || *p->s == '-') {
			p->s++;
		}
		must(scandigits(p));
	}
	if (v) {
		v->end = p->s;
	}
	return 1;
}


static int
parsepair(Parser *p, JSON *parent, JSON **kprev, JSON **vprev)
{
	must(*p->s == '"');
	must(parsestring(p, parent, kprev));
	skipws(p);
	must(consume(p, ":"));
	skipws(p);
	must(parsevalue(p, parent, vprev));
	return 1;
}


static int
parseobject(Parser *p, JSON *parent, JSON **prev)
{
	JSON *v = inititem(p, parent, prev, '{');
	must(consume(p, "{"));
	skipws(p);
	if (*p->s != '}') {
		JSON *kprev = nil, *vprev = nil;
		must(parsepair(p, v, &kprev, &vprev));
		for (skipws(p); *p->s == ','; skipws(p)) {
			p->s++; /* consume , */
			skipws(p);
			must(parsepair(p, v, &kprev, &vprev));
		}
	}
	must(consume(p, "}"));
	if (v) {
		v->end = p->s;
	}
	return 1;
}


static int
parsearray(Parser *p, JSON *parent, JSON **prev)
{
	JSON *v = inititem(p, parent, prev, '[');
	must(consume(p, "["));
	skipws(p);
	if (*p->s != ']') {
		JSON *aprev = nil;
		must(parsevalue(p, v, &aprev));
		for (skipws(p); *p->s == ','; skipws(p)) {
			p->s++; /* consume , */
			skipws(p);
			must(parsevalue(p, v, &aprev));
		}
	}
	must(consume(p, "]"));
	if (v) {
		v->end = p->s;
	}
	return 1;
}


static int
parsevalue(Parser *p, JSON *parent, JSON **prev)<commit_msg>factor out some common logic<commit_after>#include "json.h"

#define nil ((void*)0)
#define must(b) do { if (!(b)) return 0; } while (0)

typedef struct Parser Parser;

struct Parser {
	char *s;
	int  n;
	JSON *j;
	int  nj;
};

static int parsevalue(Parser*, JSON*, JSON**);


static int
consume(Parser *p, char *s)
{
	while (*s) {
		must(*p->s == *s);
		p->s++;
		s++;
	}
	return 1;
}


static void
skipws(Parser *p)
{
	while (*p->s == ' ' || *p->s == '\t' || *p->s == '\n' || *p->s == '\r') {
		p->s++;
	}
}


static int
scandigits(Parser *p)
{
	int d = 0;
	while ('0' <= *p->s && *p->s <= '9') {
		p->s++;
		d = 1;
	}
	return d;
}


static int
scanhex4(Parser *p)
{
	int i;
	for (i = 0; i < 4; i++) {
		char c = *p->s++;
		must(('0'<=c && c<='9') || ('a'<=c && c<='f') || ('A'<=c && c<='F'));
	}
	return 1;
}


static JSON *
inititem(Parser *p, JSON *parent, JSON **prev, char type)
{
	p->n++;
	if (p->nj > 0) {
		JSON *v = p->j;
		p->j++;
		p->nj--;
		v->type = type;
		v->src = p->s;
		v->parent = parent;
		v->next = nil;
		v->prev = *prev;
		if (*prev) {
			(*prev)->next = v;
		}
		*prev = v;
		return v;
	}
	return nil;
}


static void
setend(Parser *p, JSON *v)
{
	if (v) {
		v->end = p->s;
	}
}


static int
parseword(Parser *p, JSON *parent, JSON **prev, char *lit)
{
	JSON *v = inititem(p, parent, prev, lit[0]);
	must(consume(p, lit));
	setend(p, v);
	return 1;
}


static int
parsestring(Parser *p, JSON *parent, JSON **prev)
{
	JSON *v = inititem(p, parent, prev, '"');
	p->s++; /* consume " */
	while (*p->s != '"') {
		char c = *p->s;
		must(c >= ' '); /* no control chars */
		p->s++;
		if (c == '\\') {
			switch (c = *p->s++) {
			case 'b': case 'f': case 'n': case 'r':
			case 't': case '"': case '\\': case '/':
				continue;
			case 'u':
				must(scanhex4(p));
				continue;
			}
			return 0;
		}
	}
	p->s++; /* consume " */
	setend(p, v);
	return 1;
}


static int
parsenumber(Parser *p, JSON *parent, JSON **prev)
{
	char c;
	JSON *v = inititem(p, parent, prev, '0');
	if (*p->s == '-') {
		p->s++;
	}
	c = *p->s;
	if (c == '0') { /* special case, single 0 */
		p->s++;
	} else {
		must(scandigits(p));
	}
	if (*p->s == '.') {
		p->s++;
		must(scandigits(p));
	}
	if (*p->s == 'e' || *p->s == 'E') {
		p->s++;
		if (*p->s == '+' || *p->s == '-') {
			p->s++;
		}
		must(scandigits(p));
	}
	setend(p, v);
	return 1;
}


static int
parsepair(Parser *p, JSON *parent, JSON **kprev, JSON **vprev)
{
	must(*p->s == '"');
	must(parsestring(p, parent, kprev));
	skipws(p);
	must(consume(p, ":"));
	skipws(p);
	must(parsevalue(p, parent, vprev));
	return 1;
}


static int
parseobject(Parser *p, JSON *parent, JSON **prev)
{
	JSON *v = inititem(p, parent, prev, '{');
	must(consume(p, "{"));
	skipws(p);
	if (*p->s != '}') {
		JSON *kprev = nil, *vprev = nil;
		must(parsepair(p, v, &kprev, &vprev));
		for (skipws(p); *p->s == ','; skipws(p)) {
			p->s++; /* consume , */
			skipws(p);
			must(parsepair(p, v, &kprev, &vprev));
		}
	}
	must(consume(p, "}"));
	setend(p, v);
	return 1;
}


static int
parsearray(Parser *p, JSON *parent, JSON **prev)
{
	JSON *v = inititem(p, parent, prev, '[');
	must(consume(p, "["));
	skipws(p);
	if (*p->s != ']') {
		JSON *aprev = nil;
		must(parsevalue(p, v, &aprev));
		for (skipws(p); *p->s == ','; skipws(p)) {
			p->s++; /* consume , */
			skipws(p);
			must(parsevalue(p, v, &aprev));
		}
	}
	must(consume(p, "]"));
	setend(p, v);
	return 1;
}


static int
parsevalue(Parser *p, JSON *parent, JSON **prev)
{'fraction_non_alphanumeric': 0.2718600953895072, 'fraction_numerical': 0.007804596039890158, 'mean_word_length': 2.88327721661055, 'pattern_counts': {'":': 2, '<': 21, '<?xml version=': 0, '>': 115, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3323312', 'n_tokens_mistral': 3417, 'n_tokens_neox': 3186, 'n_words': 777}