<commit_before><!DOCTYPE html>
<html>
<head>
	<meta charset="utf-8">
	<title>JSON Streams Tokenizer Test</title>
	<style>body { background: #333; color: #ccc;  }</style>
</head>
<body>
<h1>JSON Streams Tokenizer Test</h1>
<script type="module">
import { JSONStreamParser } from "../dist/json-stream-parser.esm.js";

const delegate = {
	key(label) {
		console.info("key", label);
	},

	null() {
		console.info("null");
	},
	true() {
		console.info("true");
	},
	false() {
		console.info("false");
	},

	number(value) {
		console.info(value);
	},
	string(value) {
		console.info(value);
	},

	objectStart() {
		console.info("{");
	},
	objectEnd() {
		console.info("}");
	},

	arrayStart() {
		console.info("[");
	},
	arrayEnd() {
		console.info("]");
	},

	error(message) {
		console.warn(message);
	},
};

async function runTest(collection, index, expectedError) {
	const parser = new JSONStreamParser(delegate);
	const text = await fetch(`./jsonchecker/${collection}${index}.json`).then(r => r.text());
	parser.append(text);
	console.info(index, "complete", parser.completed);
	console.info("--------");
}

for (let x = 1; x < 34; ++x) {<commit_msg>test(json-stream): feed json data in per character<commit_after><!DOCTYPE html>
<html>
<head>
	<meta charset="utf-8">
	<title>JSON Streams Tokenizer Test</title>
	<style>body { background: #333; color: #ccc;  }</style>
</head>
<body>
<h1>JSON Streams Tokenizer Test</h1>
<script type="module">
import { JSONStreamParser } from "../dist/json-stream-parser.esm.js";

const delegate = {
	key(label) {
		console.info("key", label);
	},

	null() {
		console.info("null");
	},
	true() {
		console.info("true");
	},
	false() {
		console.info("false");
	},

	number(value) {
		console.info(value);
	},
	string(value) {
		console.info(value);
	},

	objectStart() {
		console.info("{");
	},
	objectEnd() {
		console.info("}");
	},

	arrayStart() {
		console.info("[");
	},
	arrayEnd() {
		console.info("]");
	},

	error(message) {
		console.warn(message);
	},
};

async function runTest(collection, index, expectedError) {
	const parser = new JSONStreamParser(delegate);
	const text = await fetch(`./jsonchecker/${collection}${index}.json`).then(r => r.text());
	const chars = text.split("");
	for (const c of chars) {
		parser.append(c);
	}
	console.info(index, "complete", parser.completed);
	console.info("--------");
}

for (let x = 1; x < 34; ++x) {
{'fraction_non_alphanumeric': 0.22022565816966153, 'fraction_numerical': 0.007521938988717091, 'mean_word_length': 4.22707423580786, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3618160', 'n_tokens_mistral': 931, 'n_tokens_neox': 878, 'n_words': 172}