<commit_before>/*globals SAJJ, ObjectArrayDelegator, DOMParser, exports */
/*jslint todo: true */
/**
* JHTML is a format which can represent arbitrary JSON structures in a faithful, human-readable, and portable manner.
* It is also round-trippable except in the case when converting *from* object-containing JSON to JHTML when the
* ECMAScript/JSON interpreter does not iterate the properties in definition order (as it is not required to do)
* @namespace Contains methods for conversions between JSON and JHTML (as strings or objects)
* @requires SAJJ
* @requires ObjectArrayDelegator
* @requires polyfill: Array.prototype.map
* @requires polyfill: Array.prototype.reduce
* @requires polyfill: Element.prototype.textContent
* @requires polyfill: Element.prototype.itemProp
* @requires polyfill: HTMLDocument.prototype.getItems
* @todo Add polyfills, e.g., https://github.com/termi/Microdata-JS/
* @todo Add option for stringification (JSON or JHTML) to provide indentation, etc.
*/
var exports;
(function () {'use strict';
    function ignoreHarmlessNonelementNodes (node, item) {
        if (
            (node.nodeType === 3 || node.nodeType === 4) && // Text or CDATA node
            !node.nodeValue.match(/^\s*$/)
        ) {
            throw 'Non-whitespace text or CDATA nodes are not allowed directly within <' +
                            item.nodeName.toLowerCase() + '>';
        }
        // Todo: also ignore nodes like comments or processing instructions? (A mistake of JSON?); we might even convert comments into JavaScript comments if this is used in a non-JSON-restricted JavaScript environment
        return node.nodeType !== 1; // Not an element (ignore comments, whitespace text nodes, etc.)
    }
    function item2JSONObject (item, throwOnSpan) {
        function getHarmlessChildNodes (node) {
            return function (prev, childNode) {
                if (ignoreHarmlessNonelementNodes(childNode, node)) {
                    return prev;
                }
                // As per check above, will be exactly one element node in this batch
                return prev + item2JSONObject(childNode, true);
            };
        }
        var ret, state, textContent = item.textContent, topLevelJSONElement = item.nodeName.toLowerCase();
        switch (topLevelJSONElement) {
            case 'span':
                if (throwOnSpan) {
                    throw 'A <span> element is not allowed in this context';
                }
                return textContent;
            case 'i': // null, boolean, number (or undefined, function, non-finite number, Date or RegExp object)
                switch (textContent) {
                    case 'null':
                        return null;
                    case 'true':
                        return true;
                    case 'false':
                        return false;
// Todo: check option on whether allowing non-JSON
                    // Non-JSON
                    case 'undefined':
                        return undefined;
                    case 'Infinity':
                        return Infinity;
                    case '-Infinity':
                        return -Infinity;
                    case 'NaN':
                        return NaN;
                    default:
                        // number
                        if ((/^\-?(?:0|[1-9]\d*)(?:\.\d+)?(?:e(?:\+|\-)?\d+)?$/i).test(textContent)) {
                            return parseFloat(textContent);
                        }
                        // function
                        var funcMatch = textContent.match(/^function \w*([\w, ]*) {([\s\S]*)}$/);
                        if (funcMatch) {
                            return Function.apply(null, funcMatch[1].split(/, /).concat(funcMatch[2]));
                        }
                        // Date
                        if ((/^(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun) /).test(textContent)) {
                            return new Date(Date.parse(textContent));
                        }
                        // RegExp
                        var regexMatch = textContent.match(/^\/([\s\S]*)\/(g?)(i?)(m?)$/);
                        if (regexMatch) {
                            var flags = (regexMatch[2] ? 'g' : '') + (regexMatch[3] ? 'i' : '') + (regexMatch[4] ? 'm' : '');
                            return new RegExp(regexMatch[1], flags);
                        }
                        break;
                }
                return;
            case 'dl': // object
                // JSON allows empty objects (and HTML allows empty <dl>'s) so we do also
                state = 'dt';
                ret = [].reduce.call(item.childNodes, function (prev, node) {
                    if (ignoreHarmlessNonelementNodes(node, item)) {
                        return prev;
                    }
                    var nodeName = node.nodeName.toLowerCase();
                    if (state !== nodeName) {
                        throw 'Unexpected state encountered: ' + state;
                    }
                    if (nodeName === 'dt') {
                        if (node.children.length) {
                            throw '<dt> should not have any children';
                        }
                        state = 'dd';
                        return prev + JSON.stringify(node.textContent) + ':';
                    }
                    // Can now only be a <dd>
                    state = 'dt';
                    if (node.children.length > 1) {
                        throw '<dd> should not have more than one child';
                    }
                    if (!node.children.length) {
                        return prev + JSON.stringify(node.textContent) + ',';
                    }
                    return prev + [].reduce.call(node.childNodes, getHarmlessChildNodes(node), '') + ',';
                }, '{').replace(/,$/, '') + '}';
                if (state !== 'dt') {
                    throw 'Ended a definition list without a final <dd> to match the previous <dt>.';
                }
                return ret;
            case 'ol': // array
                if (item.getAttribute('start') !== '0') {
                    throw 'For the sake of readability, <ol> must include a start="0" attribute within JHTML.';
                }
                // JSON allows empty arrays (and HTML allows empty <ol>'s) so we do also
                return [].reduce.call(item.childNodes, function (prev, node) {
                    if (ignoreHarmlessNonelementNodes(node, item)) {
                        return prev;
                    }
                    var nodeName = node.nodeName.toLowerCase();
                    if (nodeName !== 'li') {
                        throw 'Unexpected child of <ol> element: ' + nodeName;
                    }
                    if (node.children.length > 1) {
                        throw '<li> should not have more than a single <ol> or <dl> child';
                    }
                    if (!node.children.length) {
                        return prev + JSON.stringify(node.textContent) + ',';
                    }
                    return prev + node.childNodes.reduce(getHarmlessChildNodes(node), '') + ',';
                }, '[').replace(/,$/, '') + ']';
        }
    }
    function escapeHTMLText (str) {
        return str.replace(/&/g, '&amp;').replace(/</g, '&lt;');
    }
    var exp,
        jhtmlNs = 'http://brett-zamir.me/ns/microdata/json-as-html/2',
        JHTMLStringifier = SAJJ.createAndReturn({inherits: ObjectArrayDelegator, methods: 
{

// JSON terminal handler methods

// These four methods can be overridden without affecting the logic of the objectHandler and arrayHandler to utilize
//   reporting of the object as a whole
beginObjectHandler: function beginObjectHandler (value, parentObject, parentKey, parentObjectArrayBool) {
    return '<dl' + (parentObject ? '' : ' itemscope="" itemtype="' + jhtmlNs + '"') + '>';
},
endObjectHandler: function endObjectHandler (value, parentObject, parentKey, parentObjectArrayBool) {
    return '</dl>';
},
beginArrayHandler: function beginArrayHandler (value, parentObject, parentKey, parentObjectArrayBool) {
    return '<ol start="0"' + (parentObject ? '' : ' itemscope="" itemtype="' + jhtmlNs + '"') + '>';
},
endArrayHandler: function endArrayHandler (value, parentObject, parentKey, parentObjectArrayBool) {<commit_msg>further work on converting to JSON object<commit_after>/*globals SAJJ, ObjectArrayDelegator, DOMParser, exports */
/*jslint todo: true */
/**
* JHTML is a format which can represent arbitrary JSON structures in a faithful, human-readable, and portable manner.
* It is also round-trippable except in the case when converting *from* object-containing JSON to JHTML when the
* ECMAScript/JSON interpreter does not iterate the properties in definition order (as it is not required to do)
* @namespace Contains methods for conversions between JSON and JHTML (as strings or objects)
* @requires SAJJ
* @requires ObjectArrayDelegator
* @requires polyfill: Array.prototype.map
* @requires polyfill: Array.prototype.reduce
* @requires polyfill: Element.prototype.textContent
* @requires polyfill: Element.prototype.itemProp
* @requires polyfill: HTMLDocument.prototype.getItems
* @todo Add polyfills, e.g., https://github.com/termi/Microdata-JS/
* @todo Add option for stringification (JSON or JHTML) to provide indentation, etc.
*/
var exports;
(function () {'use strict';
    function ignoreHarmlessNonelementNodes (node, item) {
        if (
            (node.nodeType === 3 || node.nodeType === 4) && // Text or CDATA node
            !node.nodeValue.match(/^\s*$/)
        ) {
            throw 'Non-whitespace text or CDATA nodes are not allowed directly within <' +
                            item.nodeName.toLowerCase() + '>';
        }
        // Todo: also ignore nodes like comments or processing instructions? (A mistake of JSON?); we might even convert comments into JavaScript comments if this is used in a non-JSON-restricted JavaScript environment
        return node.nodeType !== 1; // Not an element (ignore comments, whitespace text nodes, etc.)
    }
    function item2JSONObject (item, throwOnSpan) {
        function getHarmlessChildNodes (node) {
            return function (prev, childNode) {
                if (ignoreHarmlessNonelementNodes(childNode, node)) {
                    return prev;
                }
                // As per check above, will be exactly one element node in this batch
                return prev + item2JSONObject(childNode, true);
            };
        }
        var ret, state, textContent = item.textContent, topLevelJSONElement = item.nodeName.toLowerCase();
        switch (topLevelJSONElement) {
            case 'span':
                if (throwOnSpan) {
                    throw 'A <span> element is not allowed in this context';
                }
                return textContent;
            case 'i': // null, boolean, number (or undefined, function, non-finite number, Date or RegExp object)
                switch (textContent) {
                    case 'null':
                        return null;
                    case 'true':
                        return true;
                    case 'false':
                        return false;
// Todo: check option on whether allowing non-JSON
                    // Non-JSON
                    case 'undefined':
                        return undefined;
                    case 'Infinity':
                        return Infinity;
                    case '-Infinity':
                        return -Infinity;
                    case 'NaN':
                        return NaN;
                    default:
                        // number
                        if ((/^\-?(?:0|[1-9]\d*)(?:\.\d+)?(?:e(?:\+|\-)?\d+)?$/i).test(textContent)) {
                            return parseFloat(textContent);
                        }
                        // function
                        var funcMatch = textContent.match(/^function \w*([\w, ]*) {([\s\S]*)}$/);
                        if (funcMatch) {
                            return Function.apply(null, funcMatch[1].split(/, /).concat(funcMatch[2]));
                        }
                        // Date
                        if ((/^(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun) /).test(textContent)) {
                            return new Date(Date.parse(textContent));
                        }
                        // RegExp
                        var regexMatch = textContent.match(/^\/([\s\S]*)\/(g?)(i?)(m?)$/);
                        if (regexMatch) {
                            var flags = (regexMatch[2] ? 'g' : '') + (regexMatch[3] ? 'i' : '') + (regexMatch[4] ? 'm' : '');
                            return new RegExp(regexMatch[1], flags);
                        }
                        break;
                }
                return;
            case 'dl': // object
                // JSON allows empty objects (and HTML allows empty <dl>'s) so we do also
                state = 'dt';
                var key, obj = {};
                ret = [].forEach.call(item.childNodes, function (prev, node) {
                    if (ignoreHarmlessNonelementNodes(node, item)) {
                        return;
                    }
                    var nodeName = node.nodeName.toLowerCase();
                    if (state !== nodeName) {
                        throw 'Unexpected state encountered: ' + state;
                    }
                    if (nodeName === 'dt') {
                        if (node.children.length) {
                            throw '<dt> should not have any children';
                        }
                        state = 'dd';
                        key = node.textContent;
                        return;
                    }
                    // Can now only be a <dd>
                    state = 'dt';
                    if (node.children.length > 1) {
                        throw '<dd> should not have more than one element child';
                    }
                    if (!node.children.length) {
                        obj[key] = node.textContent;
                        return;
                    }
                    if (node.textContent) {
                        throw 'There should be no text content inside of <dd> when there is an element child.';
                    }
                    obj[key] = [].reduce.call(node.childNodes, getHarmlessChildNodes(node), '');
                    return;
                });
                if (state !== 'dt') {
                    throw 'Ended a definition list without a final <dd> to match the previous <dt>.';
                }
                return ret;
            case 'ol': // array
                if (item.getAttribute('start') !== '0') {
                    throw 'For the sake of readability, <ol> must include a start="0" attribute within JHTML.';
                }
                // JSON allows empty arrays (and HTML allows empty <ol>'s) so we do also
                return [].reduce.call(item.childNodes, function (prev, node) {
                    if (ignoreHarmlessNonelementNodes(node, item)) {
                        return prev;
                    }
                    var nodeName = node.nodeName.toLowerCase();
                    if (nodeName !== 'li') {
                        throw 'Unexpected child of <ol> element: ' + nodeName;
                    }
                    if (node.children.length > 1) {
                        throw '<li> should not have more than a single <ol> or <dl> child';
                    }
                    if (!node.children.length) {
                        return prev + JSON.stringify(node.textContent) + ',';
                    }
                    if (node.textContent) {
                        throw 'There should be no text content inside of <li> when there is an element child.';
                    }
                    return prev + node.childNodes.reduce(getHarmlessChildNodes(node), '') + ',';
                }, '[').replace(/,$/, '') + ']';
        }
    }
    function escapeHTMLText (str) {
        return str.replace(/&/g, '&amp;').replace(/</g, '&lt;');
    }
    var exp,
        jhtmlNs = 'http://brett-zamir.me/ns/microdata/json-as-html/2',
        JHTMLStringifier = SAJJ.createAndReturn({inherits: ObjectArrayDelegator, methods: 
{

// JSON terminal handler methods

// These four methods can be overridden without affecting the logic of the objectHandler and arrayHandler to utilize
//   reporting of the object as a whole
beginObjectHandler: function beginObjectHandler (value, parentObject, parentKey, parentObjectArrayBool) {
    return '<dl' + (parentObject ? '' : ' itemscope="" itemtype="' + jhtmlNs + '"') + '>';
},
endObjectHandler: function endObjectHandler (value, parentObject, parentKey, parentObjectArrayBool) {
    return '</dl>';
},
beginArrayHandler: function beginArrayHandler (value, parentObject, parentKey, parentObjectArrayBool) {
    return '<ol start="0"' + (parentObject ? '' : ' itemscope="" itemtype="' + jhtmlNs + '"') + '>';
},
endArrayHandler: function endArrayHandler (value, parentObject, parentKey, parentObjectArrayBool) {
{'fraction_non_alphanumeric': 0.11323376172282042, 'fraction_numerical': 0.002315618849137432, 'mean_word_length': 1.4859692042020434, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 43, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4053105', 'n_tokens_mistral': 4390, 'n_tokens_neox': 4134, 'n_words': 1260}