<commit_before>/*
Copyright (c) 2011, Yahoo! Inc. All rights reserved.
Code licensed under the BSD License:
http://yuilibrary.com/license/
*/
YUI.add('docparser', function (Y) {

    var Lang = Y.Lang,
        trim = Lang.trim,
        fixType = Y.Lang.fixType,
        /**
         * Parses the JSON data and formats it into a nice log string for
         * filename and line number: `/file/name.js:123`
         * @method stringlog
         * @private
         * @param {Object} data The data block from the parser
         * @return {String} The formatted string.
         * @for DocParser
         */
        stringlog = function (data) {
            var line, file;

            if (data.file && data.line) {
                file = data.file;
                line = data.line;
            } else {
                data.forEach(function (d) {
                    if (d.tag === 'file') {
                        file = d.value;
                    }
                    if (d.tag === 'line') {
                        line = d.value;
                    }
                });
            }
            return ' ' + file + ':' + line;
        },
        /**
         * Flatten a string, remove all line breaks and replace them with a token
         * @method implodeString
         * @private
         * @param {String} str The string to operate on
         * @return {String} The modified string
         */
        implodeString = function (str) {
            return str.replace(REGEX_GLOBAL_LINES, '!~YUIDOC_LINE~!');
        },
        /**
         * Un-flatten a string, replace tokens injected with `implodeString`
         * @method implodeString
         * @private
         * @param {String} str The string to operate on
         * @return {String} The modified string
         */
        explodeString = function (str) {
            return str.replace(/!~YUIDOC_LINE~!/g, '\n');
        },
        CURRENT_NAMESPACE = 'currentnamespace',
        CURRENT_MODULE = 'currentmodule',
        MAIN_MODULE = 'mainmodule',
        CURRENT_SUBMODULE = 'currentsubmodule',
        CURRENT_FILE = 'currentfile',
        CURRENT_CLASS = 'currentclass',

        REGEX_TYPE = /(.*?)\{(.*?)\}(.*)/,
        REGEX_FIRSTWORD = /^\s*?(\[.+\]\*?|[^\s]+)(.*)/,
        REGEX_OPTIONAL = /^\[(.*)\]$/,
        REGEX_START_COMMENT = {
            js: /^\s*\/\*\*/,
            coffee: /^\s*###\*/
        },
        REGEX_END_COMMENT = {
            js: /\*\/\s*$/,
            coffee: /###\s*$/
        },
        REGEX_LINE_HEAD_CHAR = {
            js: /^\s*\*/,
            coffee: /^\s*[#\*]/
        },
        REGEX_LINES = /\r\n|\n/,
        REGEX_GLOBAL_LINES = /\r\n|\n/g,

        SHORT_TAGS = {
            'async': 1,
            'beta': 1,
            'chainable': 1,
            'extends': 1,
            'final': 1,
            'static': 1,
            'optional': 1,
            'required': 1
        },

        /**
         * A list of known tags.  This populates a member variable
         * during initialization, and will be updated if additional
         * digesters are added.
         * @property TAGLIST
         * @type Array
         * @final
         * @for DocParser
         */
        TAGLIST = [
            "async",        // bool, custom events can fire the listeners in a setTimeout
            "author",       // author best for projects and modules, but can be used anywhere // multi
            "attribute",    // YUI attributes -- get/set with change notification, etc
            "beta",         // module maturity identifier
            "broadcast",    // bool, events
            "bubbles",      // custom events that bubble
            "category",     // modules can be in multiple categories
            "chainable",    // methods that return the host object
            "class",        // pseudo class
            "conditional",  // conditional module
            "config",       // a config param (not an attribute, so no change events)
            "const",        // not standardized yet, converts to final property
            "constructs",   // factory methods (not yet used)
            "constructor",  // this is a constructor
            "contributor",  // like author
            "default",      // property/attribute default value
            "deprecated",   // please specify what to use instead
            "description",  // can also be free text at the beginning of a comment is
            "emitfacade",   // bool, YUI custom event can have a dom-like event facade
            "event",        // YUI custom event
            "evil",         // uses eval
            "extension",    // this is an extension for [entity]
            "extensionfor", // this is an extension for [entity]
            "extension_for",// this is an extension for [entity]
            "example",      // 0..n code snippets.  snippets can also be embedded in the desc
            "experimental", // module maturity identifier
            "extends",      // pseudo inheritance
            "file",         // file name (used by the parser)
            "final",        // not meant to be changed
            "fireonce",     // bool, YUI custom event config allows
            "for",          // used to change class context
            "global",       // declare your globals
            "icon",         // project icon(s)
            "in",           // indicates module this lives in (obsolete now)
            "initonly",     // attribute writeonce value
            "injects",      // injects {HTML|script|CSS}
            "knownissue",   // 0..n known issues for your consumption
            "line",         // line number for the comment block (used by the parser)
            "method",       // a method
            "module",       // YUI module name
            "main",         // Description for the module
            "namespace",    // Y.namespace, used to fully qualify class names
            "optional",     // For optional attributes
            "required",     // For required attributes
            "param",        // member param
            "plugin",       // this is a plugin for [entityl]
            "preventable",  // YUI custom events can be preventable ala DOM events
            "private",      // > access
            "project",      // project definition, one per source tree allowed
            "property",     // a regular-ole property
            "protected",    // > access
            "public",       // > access
            "queuable",     // bool, events
            "readonly",     // YUI attribute config
            "requires",     // YUI module requirements
            "return",       // {type} return desc -- returns is converted to this
            "see",          // 0..n things to look at
            "since",        // when it was introduced
            "static",       // static
            "submodule",    // YUI submodule
            "throws",       // {execption type} description
            "title",        // this should be something for the project description
            "todo",         // 0..n things to revisit eventually (hopefully)
            "type",         // the var type
            "url",          // project url(s)
            "uses",         // 0..n compents mixed (usually, via augment) into the prototype
            "value",        // the value of a constant
            "writeonce"     // YUI attribute config
        ],

        /**
         * Common errors will get scrubbed instead of being ignored.
         * @property CORRECTIONS
         * @type Object
         * @final
         * @for DocParser
         */
        CORRECTIONS = {
            'augments': 'uses', // YUI convention for prototype mixins
            'depreciated': 'deprecated', // subtle difference
            'desciption': 'description', // shouldn't need the @description tag at all
            'exception': 'throws', // we may want standalone inner functions at some point
            'extend': 'extends', // typo
            'function': 'method', // we may want standalone inner functions at some point
            'member': 'method', // probably meant method
            'parm': 'param', // typo
            'params': 'param', // typo
            'pamra': 'param', // typo
            'parma': 'param', // typo
            'propery': 'property', // typo
            'prop': 'property', // probably meant property
            'returns': 'return', // need to standardize on one or the other
            'throw': 'throws' // need to standardize on one or the other
        },

        /**
         * A map of the default tag processors, keyed by the
         * tag name.  Multiple tags can use the same digester
         * by supplying the string name that points to the
         * implementation rather than a function.
         * @property DIGESTERS
         * @type Object
         * @final
         * @for DocParser
         */
        DIGESTERS = {
            // "params": [<commit_msg>Fix comment for exception as correction<commit_after>/*
Copyright (c) 2011, Yahoo! Inc. All rights reserved.
Code licensed under the BSD License:
http://yuilibrary.com/license/
*/
YUI.add('docparser', function (Y) {

    var Lang = Y.Lang,
        trim = Lang.trim,
        fixType = Y.Lang.fixType,
        /**
         * Parses the JSON data and formats it into a nice log string for
         * filename and line number: `/file/name.js:123`
         * @method stringlog
         * @private
         * @param {Object} data The data block from the parser
         * @return {String} The formatted string.
         * @for DocParser
         */
        stringlog = function (data) {
            var line, file;

            if (data.file && data.line) {
                file = data.file;
                line = data.line;
            } else {
                data.forEach(function (d) {
                    if (d.tag === 'file') {
                        file = d.value;
                    }
                    if (d.tag === 'line') {
                        line = d.value;
                    }
                });
            }
            return ' ' + file + ':' + line;
        },
        /**
         * Flatten a string, remove all line breaks and replace them with a token
         * @method implodeString
         * @private
         * @param {String} str The string to operate on
         * @return {String} The modified string
         */
        implodeString = function (str) {
            return str.replace(REGEX_GLOBAL_LINES, '!~YUIDOC_LINE~!');
        },
        /**
         * Un-flatten a string, replace tokens injected with `implodeString`
         * @method implodeString
         * @private
         * @param {String} str The string to operate on
         * @return {String} The modified string
         */
        explodeString = function (str) {
            return str.replace(/!~YUIDOC_LINE~!/g, '\n');
        },
        CURRENT_NAMESPACE = 'currentnamespace',
        CURRENT_MODULE = 'currentmodule',
        MAIN_MODULE = 'mainmodule',
        CURRENT_SUBMODULE = 'currentsubmodule',
        CURRENT_FILE = 'currentfile',
        CURRENT_CLASS = 'currentclass',

        REGEX_TYPE = /(.*?)\{(.*?)\}(.*)/,
        REGEX_FIRSTWORD = /^\s*?(\[.+\]\*?|[^\s]+)(.*)/,
        REGEX_OPTIONAL = /^\[(.*)\]$/,
        REGEX_START_COMMENT = {
            js: /^\s*\/\*\*/,
            coffee: /^\s*###\*/
        },
        REGEX_END_COMMENT = {
            js: /\*\/\s*$/,
            coffee: /###\s*$/
        },
        REGEX_LINE_HEAD_CHAR = {
            js: /^\s*\*/,
            coffee: /^\s*[#\*]/
        },
        REGEX_LINES = /\r\n|\n/,
        REGEX_GLOBAL_LINES = /\r\n|\n/g,

        SHORT_TAGS = {
            'async': 1,
            'beta': 1,
            'chainable': 1,
            'extends': 1,
            'final': 1,
            'static': 1,
            'optional': 1,
            'required': 1
        },

        /**
         * A list of known tags.  This populates a member variable
         * during initialization, and will be updated if additional
         * digesters are added.
         * @property TAGLIST
         * @type Array
         * @final
         * @for DocParser
         */
        TAGLIST = [
            "async",        // bool, custom events can fire the listeners in a setTimeout
            "author",       // author best for projects and modules, but can be used anywhere // multi
            "attribute",    // YUI attributes -- get/set with change notification, etc
            "beta",         // module maturity identifier
            "broadcast",    // bool, events
            "bubbles",      // custom events that bubble
            "category",     // modules can be in multiple categories
            "chainable",    // methods that return the host object
            "class",        // pseudo class
            "conditional",  // conditional module
            "config",       // a config param (not an attribute, so no change events)
            "const",        // not standardized yet, converts to final property
            "constructs",   // factory methods (not yet used)
            "constructor",  // this is a constructor
            "contributor",  // like author
            "default",      // property/attribute default value
            "deprecated",   // please specify what to use instead
            "description",  // can also be free text at the beginning of a comment is
            "emitfacade",   // bool, YUI custom event can have a dom-like event facade
            "event",        // YUI custom event
            "evil",         // uses eval
            "extension",    // this is an extension for [entity]
            "extensionfor", // this is an extension for [entity]
            "extension_for",// this is an extension for [entity]
            "example",      // 0..n code snippets.  snippets can also be embedded in the desc
            "experimental", // module maturity identifier
            "extends",      // pseudo inheritance
            "file",         // file name (used by the parser)
            "final",        // not meant to be changed
            "fireonce",     // bool, YUI custom event config allows
            "for",          // used to change class context
            "global",       // declare your globals
            "icon",         // project icon(s)
            "in",           // indicates module this lives in (obsolete now)
            "initonly",     // attribute writeonce value
            "injects",      // injects {HTML|script|CSS}
            "knownissue",   // 0..n known issues for your consumption
            "line",         // line number for the comment block (used by the parser)
            "method",       // a method
            "module",       // YUI module name
            "main",         // Description for the module
            "namespace",    // Y.namespace, used to fully qualify class names
            "optional",     // For optional attributes
            "required",     // For required attributes
            "param",        // member param
            "plugin",       // this is a plugin for [entityl]
            "preventable",  // YUI custom events can be preventable ala DOM events
            "private",      // > access
            "project",      // project definition, one per source tree allowed
            "property",     // a regular-ole property
            "protected",    // > access
            "public",       // > access
            "queuable",     // bool, events
            "readonly",     // YUI attribute config
            "requires",     // YUI module requirements
            "return",       // {type} return desc -- returns is converted to this
            "see",          // 0..n things to look at
            "since",        // when it was introduced
            "static",       // static
            "submodule",    // YUI submodule
            "throws",       // {execption type} description
            "title",        // this should be something for the project description
            "todo",         // 0..n things to revisit eventually (hopefully)
            "type",         // the var type
            "url",          // project url(s)
            "uses",         // 0..n compents mixed (usually, via augment) into the prototype
            "value",        // the value of a constant
            "writeonce"     // YUI attribute config
        ],

        /**
         * Common errors will get scrubbed instead of being ignored.
         * @property CORRECTIONS
         * @type Object
         * @final
         * @for DocParser
         */
        CORRECTIONS = {
            'augments': 'uses', // YUI convention for prototype mixins
            'depreciated': 'deprecated', // subtle difference
            'desciption': 'description', // shouldn't need the @description tag at all
            'exception': 'throws', // need to standardize on one or the other
            'extend': 'extends', // typo
            'function': 'method', // we may want standalone inner functions at some point
            'member': 'method', // probably meant method
            'parm': 'param', // typo
            'params': 'param', // typo
            'pamra': 'param', // typo
            'parma': 'param', // typo
            'propery': 'property', // typo
            'prop': 'property', // probably meant property
            'returns': 'return', // need to standardize on one or the other
            'throw': 'throws' // need to standardize on one or the other
        },

        /**
         * A map of the default tag processors, keyed by the
         * tag name.  Multiple tags can use the same digester
         * by supplying the string name that points to the
         * implementation rather than a function.
         * @property DIGESTERS
         * @type Object
         * @final
         * @for DocParser
         */
        DIGESTERS = {
            // "params": [
{'fraction_non_alphanumeric': 0.11473490550259241, 'fraction_numerical': 0.0022300273178346435, 'mean_word_length': 1.3674277418503364, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3256782', 'n_tokens_mistral': 4615, 'n_tokens_neox': 4454, 'n_words': 1566}