<commit_before>/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

var magicType = {
    'optional'  :   {
                        'hasBottomNotch'    :   false,
                        'hasSeparateKids'   :   false,
                        'hasLoopRisk'       :   false,
                        'prettyIndicator'   :   '?'
                    },
    'choice'  :   {
                        'hasBottomNotch'    :   false,
                        'hasSeparateKids'   :   true,
                        'hasLoopRisk'       :   false,
                        'prettyIndicator'   :   '|'
                    },
    'interleave'  :   {
                        'hasBottomNotch'    :   true,
                        'hasSeparateKids'   :   true,
                        'hasLoopRisk'       :   true,
                        'prettyIndicator'   :   '&'
                    },
    'zeroOrMore'  :   {
                        'hasBottomNotch'    :   true,
                        'hasSeparateKids'   :   false,
                        'hasLoopRisk'       :   false,
                        'prettyIndicator'   :   '*'
                    },
    'oneOrMore'  :   {
                        'hasBottomNotch'    :   true,
                        'hasSeparateKids'   :   false,
                        'hasLoopRisk'       :   true,
                        'prettyIndicator'   :   '+'
                    }
};


var numberTypes=[ 'int' , 'integer' , 'double' , 'float' , 'decimal' , 'number' ];

var blockStructureDict;
var validatorDict;

function RNG2Blockly(rngDoc) {
    this.rngDoc = rngDoc;

    var rootElement = rngDoc.documentElement;
    var startContent = (rootElement.nodeName == "grammar")
        ? rngDoc.getElementsByTagName("start")[0].childNodes
        : [ rootElement ];

    removeRedundantText(rootElement);
    removeXMLComments(rootElement);

    var codeDict            = new CodeDict(this);   // Structure to store non-redundant pieces of Bockly code
    this.blockRequestQueue  = [];   // a queue that holds requests to create new blocks

    this._nextQueueIndex = 0;
    this.pushToQueue("start", this.substitutedNodeList(startContent, "{}", "START"), "[]", "[]"); // initialize the queue
    this.uni = new UnicodeIndenter();

    while(this.blockRequestQueue.length>0) {     // keep consuming from the head and pushing to the tail
        var blockRequest        = this.blockRequestQueue.shift();

        var children            = blockRequest.children;
        var blockDisplayName    = blockRequest.blockDisplayName;
        this.currentQueueIndex  = blockRequest.queueIndex;
        var blockCode           = "";   // Contains data sent by all the children merged together one after the other.

        this.localSlotNumber    = 1;    // base_1 is more convenient for eyeballing
        this.blockValidationDict = {}

        this.uni.reset();

        this.successfulOptiField            = false;    //true or false depending on whether optiField can be created or not
        this.currentlyCreatingOptiField     = false;

        var xmlStructureForBlock = [];
        for(var i=0;i<children.length;i++){
            blockCode += this.goDeeper(children[i], "{}", i, xmlStructureForBlock);
        }
        //console.log(xmlStructureForBlock);

            // We want to always have a start block and here we force its blockCode to be unique
        if( this.currentQueueIndex == 0 ) {
            blockCode += " ";
        }

        var candidateDictEntry = {
            "blockDisplayName"  : blockDisplayName,             // it is only a "suggested display name", we use numbers internally
            "blockCode"         : blockCode,
            "topList"           : blockRequest.topList,
            "bottomList"        : blockRequest.bottomList,
            "queueIndices"      : [ this.currentQueueIndex ],    // at least one value, but more may be added in case of synonyms
            "blockStructure"    : xmlStructureForBlock,
            "blockValidationDict"   : JSON.stringify(this.blockValidationDict)  // We can substitute queue-index macros
        };

        // blockCode may contain SUBSTITUTE_QUEUE_INDEX_ references to blocks that
        // have already been merged (because slotSignature is cached in the node)
        var mergedBlocks = codeDict.getAllEntries().filter( function(a) { return a.queueIndices.length > 1; } );
        if (mergedBlocks.length > 0) {
            var subst = {};
            for (var i=0; i<mergedBlocks.length; i++) {
                var b = mergedBlocks[i].queueIndices;
                for (var j=1; j<b.length; j++) {
                    subst[b[j]] = b[0];
                }
            }
            candidateDictEntry.blockCode = candidateDictEntry.blockCode.replace(/SUBSTITUTE_QUEUE_INDEX_(\d+)_/g, function replacer(match, $1) {
                return subst.hasOwnProperty($1) ? makeSubstituteMacro(subst[$1]) : match;
            } );
        }

        codeDict.mergeIfPossibleOtherwiseAdd(candidateDictEntry);
    }

    this.toolboxXML      = "";
    this.allCode         = [];
    this.hue             = new HueGenerator();

    var queueIndex_2_blockType          = {};   // Caution: this.queueIndex_2_blockType did not work in the replacer() callback
    var blockTypeToDisplayNameMapper    = {};

        // blockOrder contains entries of codeDict sorted by the youngest queueIndex
    var blockOrder = codeDict.getAllEntries().sort( function(a,b) { return string_cmp(a.queueIndices[0],b.queueIndices[0]); } );

    blockStructureDict = {};
    validatorDict = {};

    for (var blockOrderIndex=0; blockOrderIndex<blockOrder.length; blockOrderIndex++){
        var dictEntry       = blockOrder[blockOrderIndex];
        var queueIndices    = dictEntry.queueIndices;
        var blockDisplayName = dictEntry.blockDisplayName;
        var blockType       = "block_" + blockOrderIndex;
        var blockStructure  = dictEntry.blockStructure;

        dictEntry.blockType = blockType;

        for (var i=0; i<queueIndices.length; i++){
            queueIndex_2_blockType[ queueIndices[i] ] = blockType;
        }
        blockTypeToDisplayNameMapper[blockType] = dictEntry.blockDisplayName;
        blockStructureDict[blockType]           = blockStructure;
    }
    console.log(JSON.stringify(queueIndex_2_blockType));
    console.log(JSON.stringify(blockTypeToDisplayNameMapper));
    console.log(blockStructureDict);

    for (var blockOrderIndex=0; blockOrderIndex<blockOrder.length; blockOrderIndex++){
        var dictEntry       = blockOrder[blockOrderIndex];

        var blockDisplayName = dictEntry.blockDisplayName;
        var blockType       = dictEntry.blockType;
        var topText         = dictEntry.topList.length      ? "true, ["+dictEntry.topList.map(      function(x) {return "'"+x+"'";}, this).join()+"]" : "false";
        var bottomText      = dictEntry.bottomList.length   ? "true, ["+dictEntry.bottomList.map(   function(x) {return "'"+x+"'";}, this).join()+"]" : "false";

        this.toolboxXML += "<block type='" + blockType + "'></block>";

        var blockCode = "Blockly.Blocks['" + blockType + "']={ init:function() {"
                    + this.makeBlocklyCode_UnindentedLabel( "====[ " + blockType + ": " + blockDisplayName + " ]====" ) + "\n"
                    + dictEntry.blockCode
                    + "this.setPreviousStatement(" + topText + ");"
                    + "this.setNextStatement(" + bottomText + ");"
                    + "this.setColour(" + this.hue.generate() + ");"
                    + "},"
                    + "onchange:function() { validateBlock(this) },"
                    + "};";

        blockCode = blockCode.replace(/SUBSTITUTE_QUEUE_INDEX_(\d+)_/g, function replacer(match, $1) {
            var blockType   = queueIndex_2_blockType[$1];
            var displayName = blockTypeToDisplayNameMapper[blockType];
            return (displayName && !displayName.match(/SUBSTITUTE_QUEUE_INDEX_/)) ? displayName : blockType;
        } );

        var blockValidationDict = JSON.parse(dictEntry.blockValidationDict.replace(/SUBSTITUTE_QUEUE_INDEX_(\d+)_/g, function replacer(match, $1) {
            return queueIndex_2_blockType[$1];
        } ));
        validatorDict[blockType] = {};
        var keys = Object.keys(blockValidationDict);
        for(var i=0; i<keys.length; i++) {
            var k = keys[i];
            validatorDict[blockType][k] = new Validator(blockValidationDict[k]);
        }<commit_msg>Only need to process "move" events<commit_after>/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

var magicType = {
    'optional'  :   {
                        'hasBottomNotch'    :   false,
                        'hasSeparateKids'   :   false,
                        'hasLoopRisk'       :   false,
                        'prettyIndicator'   :   '?'
                    },
    'choice'  :   {
                        'hasBottomNotch'    :   false,
                        'hasSeparateKids'   :   true,
                        'hasLoopRisk'       :   false,
                        'prettyIndicator'   :   '|'
                    },
    'interleave'  :   {
                        'hasBottomNotch'    :   true,
                        'hasSeparateKids'   :   true,
                        'hasLoopRisk'       :   true,
                        'prettyIndicator'   :   '&'
                    },
    'zeroOrMore'  :   {
                        'hasBottomNotch'    :   true,
                        'hasSeparateKids'   :   false,
                        'hasLoopRisk'       :   false,
                        'prettyIndicator'   :   '*'
                    },
    'oneOrMore'  :   {
                        'hasBottomNotch'    :   true,
                        'hasSeparateKids'   :   false,
                        'hasLoopRisk'       :   true,
                        'prettyIndicator'   :   '+'
                    }
};


var numberTypes=[ 'int' , 'integer' , 'double' , 'float' , 'decimal' , 'number' ];

var blockStructureDict;
var validatorDict;

function RNG2Blockly(rngDoc) {
    this.rngDoc = rngDoc;

    var rootElement = rngDoc.documentElement;
    var startContent = (rootElement.nodeName == "grammar")
        ? rngDoc.getElementsByTagName("start")[0].childNodes
        : [ rootElement ];

    removeRedundantText(rootElement);
    removeXMLComments(rootElement);

    var codeDict            = new CodeDict(this);   // Structure to store non-redundant pieces of Bockly code
    this.blockRequestQueue  = [];   // a queue that holds requests to create new blocks

    this._nextQueueIndex = 0;
    this.pushToQueue("start", this.substitutedNodeList(startContent, "{}", "START"), "[]", "[]"); // initialize the queue
    this.uni = new UnicodeIndenter();

    while(this.blockRequestQueue.length>0) {     // keep consuming from the head and pushing to the tail
        var blockRequest        = this.blockRequestQueue.shift();

        var children            = blockRequest.children;
        var blockDisplayName    = blockRequest.blockDisplayName;
        this.currentQueueIndex  = blockRequest.queueIndex;
        var blockCode           = "";   // Contains data sent by all the children merged together one after the other.

        this.localSlotNumber    = 1;    // base_1 is more convenient for eyeballing
        this.blockValidationDict = {}

        this.uni.reset();

        this.successfulOptiField            = false;    //true or false depending on whether optiField can be created or not
        this.currentlyCreatingOptiField     = false;

        var xmlStructureForBlock = [];
        for(var i=0;i<children.length;i++){
            blockCode += this.goDeeper(children[i], "{}", i, xmlStructureForBlock);
        }
        //console.log(xmlStructureForBlock);

            // We want to always have a start block and here we force its blockCode to be unique
        if( this.currentQueueIndex == 0 ) {
            blockCode += " ";
        }

        var candidateDictEntry = {
            "blockDisplayName"  : blockDisplayName,             // it is only a "suggested display name", we use numbers internally
            "blockCode"         : blockCode,
            "topList"           : blockRequest.topList,
            "bottomList"        : blockRequest.bottomList,
            "queueIndices"      : [ this.currentQueueIndex ],    // at least one value, but more may be added in case of synonyms
            "blockStructure"    : xmlStructureForBlock,
            "blockValidationDict"   : JSON.stringify(this.blockValidationDict)  // We can substitute queue-index macros
        };

        // blockCode may contain SUBSTITUTE_QUEUE_INDEX_ references to blocks that
        // have already been merged (because slotSignature is cached in the node)
        var mergedBlocks = codeDict.getAllEntries().filter( function(a) { return a.queueIndices.length > 1; } );
        if (mergedBlocks.length > 0) {
            var subst = {};
            for (var i=0; i<mergedBlocks.length; i++) {
                var b = mergedBlocks[i].queueIndices;
                for (var j=1; j<b.length; j++) {
                    subst[b[j]] = b[0];
                }
            }
            candidateDictEntry.blockCode = candidateDictEntry.blockCode.replace(/SUBSTITUTE_QUEUE_INDEX_(\d+)_/g, function replacer(match, $1) {
                return subst.hasOwnProperty($1) ? makeSubstituteMacro(subst[$1]) : match;
            } );
        }

        codeDict.mergeIfPossibleOtherwiseAdd(candidateDictEntry);
    }

    this.toolboxXML      = "";
    this.allCode         = [];
    this.hue             = new HueGenerator();

    var queueIndex_2_blockType          = {};   // Caution: this.queueIndex_2_blockType did not work in the replacer() callback
    var blockTypeToDisplayNameMapper    = {};

        // blockOrder contains entries of codeDict sorted by the youngest queueIndex
    var blockOrder = codeDict.getAllEntries().sort( function(a,b) { return string_cmp(a.queueIndices[0],b.queueIndices[0]); } );

    blockStructureDict = {};
    validatorDict = {};

    for (var blockOrderIndex=0; blockOrderIndex<blockOrder.length; blockOrderIndex++){
        var dictEntry       = blockOrder[blockOrderIndex];
        var queueIndices    = dictEntry.queueIndices;
        var blockDisplayName = dictEntry.blockDisplayName;
        var blockType       = "block_" + blockOrderIndex;
        var blockStructure  = dictEntry.blockStructure;

        dictEntry.blockType = blockType;

        for (var i=0; i<queueIndices.length; i++){
            queueIndex_2_blockType[ queueIndices[i] ] = blockType;
        }
        blockTypeToDisplayNameMapper[blockType] = dictEntry.blockDisplayName;
        blockStructureDict[blockType]           = blockStructure;
    }
    console.log(JSON.stringify(queueIndex_2_blockType));
    console.log(JSON.stringify(blockTypeToDisplayNameMapper));
    console.log(blockStructureDict);

    for (var blockOrderIndex=0; blockOrderIndex<blockOrder.length; blockOrderIndex++){
        var dictEntry       = blockOrder[blockOrderIndex];

        var blockDisplayName = dictEntry.blockDisplayName;
        var blockType       = dictEntry.blockType;
        var topText         = dictEntry.topList.length      ? "true, ["+dictEntry.topList.map(      function(x) {return "'"+x+"'";}, this).join()+"]" : "false";
        var bottomText      = dictEntry.bottomList.length   ? "true, ["+dictEntry.bottomList.map(   function(x) {return "'"+x+"'";}, this).join()+"]" : "false";

        this.toolboxXML += "<block type='" + blockType + "'></block>";

        var blockCode = "Blockly.Blocks['" + blockType + "']={ init:function() {"
                    + this.makeBlocklyCode_UnindentedLabel( "====[ " + blockType + ": " + blockDisplayName + " ]====" ) + "\n"
                    + dictEntry.blockCode
                    + "this.setPreviousStatement(" + topText + ");"
                    + "this.setNextStatement(" + bottomText + ");"
                    + "this.setColour(" + this.hue.generate() + ");"
                    + "},"
                    + "onchange:function(event) { if (event.type == Blockly.Events.MOVE) {validateBlock(this)} },"
                    + "};";

        blockCode = blockCode.replace(/SUBSTITUTE_QUEUE_INDEX_(\d+)_/g, function replacer(match, $1) {
            var blockType   = queueIndex_2_blockType[$1];
            var displayName = blockTypeToDisplayNameMapper[blockType];
            return (displayName && !displayName.match(/SUBSTITUTE_QUEUE_INDEX_/)) ? displayName : blockType;
        } );

        var blockValidationDict = JSON.parse(dictEntry.blockValidationDict.replace(/SUBSTITUTE_QUEUE_INDEX_(\d+)_/g, function replacer(match, $1) {
            return queueIndex_2_blockType[$1];
        } ));
        validatorDict[blockType] = {};
        var keys = Object.keys(blockValidationDict);
        for(var i=0; i<keys.length; i++) {
            var k = keys[i];
            validatorDict[blockType][k] = new Validator(blockValidationDict[k]);
        }
{'fraction_non_alphanumeric': 0.1110618242306626, 'fraction_numerical': 0.0039922373163293595, 'mean_word_length': 2.0155492392576493, 'pattern_counts': {'":': 2, '<': 21, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7078383', 'n_tokens_mistral': 5035, 'n_tokens_neox': 4839, 'n_words': 1096}