<commit_before><!--
@license
Copyright (c) 2015 The ExpandJS authors. All rights reserved.
This code may only be used under the BSD style license found at https://expandjs.github.io/LICENSE.txt
The complete set of authors may be found at https://expandjs.github.io/AUTHORS.txt
The complete set of contributors may be found at https://expandjs.github.io/CONTRIBUTORS.txt
-->

<!--
This element is used to parse an html or javascript file, extracting it's documentation.

@element xp-doc-parser
@description A custom element able to parse an html or javascript file, extracting it's documentation
@keywords JSDoc, web app, html5, expandjs, web-components
@group functionality
@homepage http://expandjs.com/elements/xp-doc-parser
@repository https://github.com/ExpandJS/xp-doc-parser

@dependency polymer Polymer/polymer#^1.0.0
@dependency expandjs ExpandJS/expandjs#0.9.6
@dependency xp-request ExpandJS/xp-request#0.9.6
-->

<link rel="import" href="../polymer/polymer.html">
<link rel="import" href="../expandjs/expandjs.html">
<link rel="import" href="../xp-request/xp-request.html">

<dom-module id="xp-doc-parser">

    <style>
        :host {
            display: none !important;
        }
    </style>

    <template>
        <xp-request url="[[url]]" on-data-changed="_handleData" auto></xp-request>
    </template>

    <script>
        Polymer({

            // ELEMENT
            is: 'xp-doc-parser',

            /*********************************************************************/

            /**
             * Fired when data has been parsed.
             *
             * @event xp-doc-parse
             * @param {Element} firer
             * @param {Object} data
             * @bubbles
             */

            /*********************************************************************/

            /**
             * Merges the passed data with parsers data.
             *
             * @method merge
             * @param {Object} data
             * @param {Array} parsers
             * @returns {Object}
             */
            merge: function (data, parsers) {

                // Asserting
                XP.assertArgument(XP.isObject(data), 1, 'Object');
                XP.assertArgument(XP.isArray(parsers), 2, 'Array');

                // Vars
                var self = this;

                // Merging
                XP.forEachRight(parsers, function (parser) {

                    // Inherited
                    XP.forEach(self.inheritance, function (feature) {
                        XP.forEach(parser.data[feature] || [], function (feat) {
                            if (XP.find(data[feature] || [], {name: feat.name})) { return; }
                            data[feature] = data[feature] || [];
                            data[feature].push(feat);
                        });
                    });
                });

                return data;
            },

            /**
             * Parses the passed text.
             *
             * @method parse
             * @param {string} [text = ""]
             * @returns {Object}
             */
            parse: function (text) {

                // Asserting
                XP.assertArgument(XP.isVoid(text) || XP.isString(text), 1, 'string');

                // Vars
                var self         = this,
                    regExpScript = '\\/\\*\\*([\\s\\S]*?)\\*\\/',
                    regExpHtml   = '<!--([\\s\\S]*?)-->',
                    regExp       = text ? new RegExp(regExpScript + '|' + regExpHtml, 'g') : null,
                    blocks       = text ? XP.map(XP.match(text, regExp), function (block) { return self.parseBlock(block); }) : [],
                    entity       = text ? XP.find(blocks, function (block) { return XP.includes(self.entities, block.type); }) : null;

                // Interpreting
                if (entity) { self._interpretEntity(entity, XP.pull(blocks, entity)); }

                return entity || {type: 'Unknown', name: 'Unknown', summary: '**Undocumented**'};
            },

            /**
             * Parses a comment block.
             *
             * @method parseBlock
             * @param {string} [text = ""]
             * @returns {Object}
             */
            parseBlock: function (text) {

                // Asserting
                XP.assertArgument(XP.isVoid(text) || XP.isString(text), 1, 'string');

                // Preparing
                text = text || '';
                text = text.replace(/\r\n/g, '\n'); // Unifies line ends
                text = text.replace(/^\s*\/\*\*|^\s*\*\/|^\s*\* ?|^\s*<!-\-|^s*\-\->/gm, ''); // Removes all comment characters

                // Vars
                var lines  = text.split('\n'),
                    result = {};

                // Filtering
                lines = XP.filter(lines, function (line) {

                    // Vars
                    var parts = XP.match(line, /^@(\w+)(.*)|^\s*\*\s+@(\w+)(.*)/);

                    // Checking
                    if (!parts.length) { return true; }

                    // Setting
                    if (result.pragma) {
                        result.pragma.push({type: parts[1], value: XP.clean(parts[2]) || true});
                    } else {
                        result.type   = parts[1];
                        result.name   = XP.clean(parts[2]);
                        result.pragma = [];
                    }
                });

                // Setting
                result.type    = result.type || '';
                result.name    = result.name || '';
                result.summary = XP.trim(lines.join('\n'));

                return result;
            },

            /**
             * Sorts the passed data.
             *
             * @method sort
             * @param {Object} data
             * @param {boolean} [useful = false]
             * @returns {Object}
             */
            sort: function (data, useful) {<commit_msg>- Added mat-breadcrumb - Added xp-finder-behavior - Added xp-master-behavior - Added xp-slave-behavior - Major documentation improvements - Minor bug fixes - Moved item finding from xp-selected-behavior to xp-finder-behavior - Replaced xp-master-state with xp-master-behavior - Replaced xp-slave-state with xp-slave-behavior<commit_after><!--
@license
Copyright (c) 2015 The ExpandJS authors. All rights reserved.
This code may only be used under the BSD style license found at https://expandjs.github.io/LICENSE.txt
The complete set of authors may be found at https://expandjs.github.io/AUTHORS.txt
The complete set of contributors may be found at https://expandjs.github.io/CONTRIBUTORS.txt
-->

<!--
This element is used to parse an html or javascript file, extracting it's documentation.

@element xp-doc-parser
@description A custom element able to parse an html or javascript file, extracting it's documentation
@keywords JSDoc, web app, html5, expandjs, web-components
@group functionality
@homepage http://expandjs.com/elements/xp-doc-parser
@repository https://github.com/ExpandJS/xp-doc-parser

@dependency polymer Polymer/polymer#^1.0.0
@dependency expandjs ExpandJS/expandjs#0.9.6
@dependency xp-request ExpandJS/xp-request#0.9.6
-->

<link rel="import" href="../polymer/polymer.html">
<link rel="import" href="../expandjs/expandjs.html">
<link rel="import" href="../xp-request/xp-request.html">

<dom-module id="xp-doc-parser">

    <style>
        :host {
            display: none !important;
        }
    </style>

    <template>
        <xp-request url="[[url]]" on-data-changed="_handleData" auto></xp-request>
    </template>

    <script>
        Polymer({

            // ELEMENT
            is: 'xp-doc-parser',

            /*********************************************************************/

            /**
             * Fired when data has been parsed.
             *
             * @event xp-doc-parse
             * @param {Element} firer
             * @param {Object} data
             * @bubbles
             */

            /*********************************************************************/

            /**
             * Merges the passed data with parsers data.
             *
             * @method merge
             * @param {Object} data
             * @param {Array} parsers
             * @returns {Object}
             */
            merge: function (data, parsers) {

                // Asserting
                XP.assertArgument(XP.isObject(data), 1, 'Object');
                XP.assertArgument(XP.isArray(parsers), 2, 'Array');

                // Vars
                var self = this;

                // Merging
                XP.forEachRight(parsers, function (parser) {

                    // Inherited
                    XP.forEach(self.inheritance, function (feature) {
                        XP.forEach(parser.data[feature] || [], function (feat) {
                            if (XP.find(data[feature] || [], {name: feat.name})) { return; }
                            data[feature] = data[feature] || [];
                            data[feature].push(feat);
                        });
                    });
                });

                return data;
            },

            /**
             * Parses the passed text.
             *
             * @method parse
             * @param {string} [text = ""]
             * @returns {Object}
             */
            parse: function (text) {

                // Asserting
                XP.assertArgument(XP.isVoid(text) || XP.isString(text), 1, 'string');

                // Vars
                var self         = this,
                    regExpScript = '\\/\\*\\*([\\s\\S]*?)\\*\\/',
                    regExpHtml   = '<!--([\\s\\S]*?)-->',
                    regExp       = text ? new RegExp(regExpScript + '|' + regExpHtml, 'g') : null,
                    blocks       = text ? XP.map(XP.match(text, regExp), function (block) { return self.parseBlock(block); }) : [],
                    entity       = text ? XP.find(blocks, function (block) { return XP.includes(self.entities, block.type); }) : null;

                // Interpreting
                if (entity) { self._interpretEntity(entity, XP.pull(blocks, entity)); }

                return entity || {type: 'Unknown', name: 'Unknown', summary: '**Undocumented**'};
            },

            /**
             * Parses a comment block.
             *
             * @method parseBlock
             * @param {string} [text = ""]
             * @returns {Object}
             */
            parseBlock: function (text) {

                // Asserting
                XP.assertArgument(XP.isVoid(text) || XP.isString(text), 1, 'string');

                // Preparing
                text = text || '';
                text = text.replace(/\r\n/g, '\n'); // Unifies line ends
                text = text.replace(/^\s*\/\*\*|^\s*\*\/|^\s*\* ?|^\s*<!-\-|^s*\-\->/gm, ''); // Removes all comment characters

                // Vars
                var lines  = text.split('\n'),
                    result = {};

                // Filtering
                lines = XP.filter(lines, function (line) {

                    // Vars
                    var parts = XP.match(line, /^@(\w+)(.*)|^\s+\s+@(\w+)(.*)/);

                    // Checking
                    if (!parts.length) { return true; }

                    // Setting
                    if (result.pragma) {
                        result.pragma.push({type: parts[1] || parts[3], value: XP.clean(parts[2] || parts[4]) || true});
                    } else {
                        result.type   = parts[1] || parts[3];
                        result.name   = XP.clean(parts[2] || parts[4]);
                        result.pragma = [];
                    }
                });

                // Setting
                result.type    = result.type || '';
                result.name    = result.name || '';
                result.summary = XP.trim(lines.join('\n'));

                return result;
            },

            /**
             * Sorts the passed data.
             *
             * @method sort
             * @param {Object} data
             * @param {boolean} [useful = false]
             * @returns {Object}
             */
            sort: function (data, useful) {
{'fraction_non_alphanumeric': 0.1620921382883977, 'fraction_numerical': 0.0038863249939276173, 'mean_word_length': 1.3496290660072285, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1025627', 'n_tokens_mistral': 3449, 'n_tokens_neox': 3149, 'n_words': 745}