<commit_before><commit_msg>extract JSON with a little help from JSON.stringify<commit_after>// http://www.whatwg.org/specs/web-apps/current-work/multipage/converting-html-to-other-formats.html#json
function getJSON(nodes) {
    function getObject(item) {
	var result = {};
	if (item.itemType)
	    result.type = item.itemType;
	if (item.itemId)
	    result.id = item.itemId;
	result.properties = {};
	for (var propIndex=0; propIndex<item.properties.length; propIndex++) {
	    var elem = item.properties[propIndex];
	    var value;
	    if (elem.itemScope)
		value = getObject(elem);
	    else
		value = elem.itemValue;
	    for (var nameIndex=0; nameIndex<elem.itemProp.length; nameIndex++) {
		var name = elem.itemProp[nameIndex];
		if (!result.properties[name])
		    result.properties[name] = [];
		result.properties[name].push(value);
	    }
	}
	return result;
    }
    var result = {};
    result.items = [];
    for (var nodeIndex=0; nodeIndex<nodes.length; nodeIndex++) {
	if (nodes[nodeIndex].itemScope && !nodes[nodeIndex].hasAttribute('itemprop'))
	    result.items.push(getObject(nodes[nodeIndex]));
    }
    return JSON.stringify(result, undefined, 2);
}
{'fraction_non_alphanumeric': 0.14884516680923868, 'fraction_numerical': 0.003421727972626176, 'mean_word_length': 3.875, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '990675', 'n_tokens_mistral': 398, 'n_tokens_neox': 379, 'n_words': 71}