<commit_before>"use strict"; "use restrict";

var load, require = require || load;
var Shaper = Shaper || require("./shaper.js") || Shaper;

Shaper("annotater", function(root) {
    var annotations = [];
    function pushNodeFragment(node, frag) {
        // remove to-end-of-line comments (// comment)
        frag = frag.replace(/\/\/.*/g, "");

        // skip if frag doesn't contain anything but whitespace
        if (frag.search(/\S/) === -1) {
            return;
        }
        // apply annotations captured on previous node fragment to this node
        if (annotations.length > 0) {
            applyAnnotations(node);
            return;
        }

        var isTerminalNode = (node.srcs.length === 1);
            captureAnnotation(node, frag, isTerminalNode);

        // special-case for terminal node, for example where fragment is
        // (/*loose*/ x), (/*loose*/ (x)) or {/*loose*/}
        // in this case the annotations should be applied to the same node
        // for which it was captured, not next node
        if (annotations.length > 0 && isTerminalNode) {
            applyAnnotations(node);
        }
    }
    function captureAnnotation(node, frag, allowTrailing) {
        for (var i = 0; i < annotate.matchers.length; i++) {
            var match = frag.match(annotate.matchers[i].re);
            if (match === null) {
                continue;
            }
            if (!allowTrailing) {
                var tail = frag.slice(match.index + match[0].length);<commit_msg>indentation<commit_after>"use strict"; "use restrict";

var load, require = require || load;
var Shaper = Shaper || require("./shaper.js") || Shaper;

Shaper("annotater", function(root) {
    var annotations = [];
    function pushNodeFragment(node, frag) {
        // remove to-end-of-line comments (// comment)
        frag = frag.replace(/\/\/.*/g, "");

        // skip if frag doesn't contain anything but whitespace
        if (frag.search(/\S/) === -1) {
            return;
        }
        // apply annotations captured on previous node fragment to this node
        if (annotations.length > 0) {
            applyAnnotations(node);
            return;
        }

        var isTerminalNode = (node.srcs.length === 1);
        captureAnnotation(node, frag, isTerminalNode);

        // special-case for terminal node, for example where fragment is
        // (/*loose*/ x), (/*loose*/ (x)) or {/*loose*/}
        // in this case the annotations should be applied to the same node
        // for which it was captured, not next node
        if (annotations.length > 0 && isTerminalNode) {
            applyAnnotations(node);
        }
    }
    function captureAnnotation(node, frag, allowTrailing) {
        for (var i = 0; i < annotate.matchers.length; i++) {
            var match = frag.match(annotate.matchers[i].re);
            if (match === null) {
                continue;
            }
            if (!allowTrailing) {
                var tail = frag.slice(match.index + match[0].length);
{'fraction_non_alphanumeric': 0.1335978835978836, 'fraction_numerical': 0.003968253968253968, 'mean_word_length': 1.9744346116027531, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3989064', 'n_tokens_mistral': 854, 'n_tokens_neox': 829, 'n_words': 249}