<commit_before>
/*========== DEPENDENCIES ==========*/

var nodeExtend = require('node.extend');
var colors = require('colors');

var DOM = require('./dom.js').DOM;
var htmlEscape = require('./dom.js').htmlEscape;

var doc_utils = require('./doc_utils.js');


/*========== PRIVATE VARIABLES AND METHODS ==========*/

var globalID = 0;
var NEW_LINE = /\n\r?/;
var ISDOC = /^(doc|doc-\S+)/i;
var HASGROUPSECTION = /^[^\.\:]{1,}\/(.*){1,}/i;

var COMPARE_ORDER = function (a, b) {
    var val = 'order';
    if (a[val] < b[val]) {
        return -1;
    } else if (a[val] > b[val]) {
        return 1;
    } else {
        return 0;
    }
};

/*========== Doc CLASS DEFINITION ==========*/

/**
 * [Doc description]
 * @param {[type]} configs_in [description]
 */
var Doc = function (configs_in) {

    var defaults = {

        //file information
        text: '',
        file: '',
        line: '',

        //category information
        group: 'main',
        section: 'section',

        //types
        requires: [],
        param: [],
        properties: [],
        methods: [],
        events: [],
        links: [],

        //what doc_api are we using?
        doc_api_extensions: {}
    };

    //merge in the defaults and configs
    nodeExtend(this, defaults, configs_in);

    if(!this.doc_api_extensions) {
        console.log("No doc_api extensions provided".red);
    }
};


Doc.METADATA_IGNORE = (function() {
  var words = require('fs').readFileSync(__dirname + '/ignore.words', 'utf8');
  return words.toString().split(/[,\s\n\r]+/gm);
})();


Doc.prototype = {

    /**
     * THIS PROTOTYPE METHOD EXTACTS RELEVANT KEYWORDS FOR SEARCHING IN THE TEXT AND DESCRIPTION FIELDS
     * @return {[type]} [description]
     */
    keywords: function keywords() {

        var kwords = {};
        var words = [];
        Doc.METADATA_IGNORE.forEach(function(ignore){ kwords[ignore] = true; });

        function extractWords(text) {
            var tokens = text.toLowerCase().split(/[\.\s,`'"#]+/mg);
            tokens.forEach(function(key){
                var match = key.match(/^((ng:|[\$_a-z])[\w\-_]+)/);
                if (match){
                    key = match[1];
                    if (!kwords[key]) {
                        kwords[key] = true;
                        words.push(key);
                    }
                }
            });
        }

        extractWords(this.text);
        this.properties.forEach(function(prop) {
          extractWords(prop.text || prop.description || '');
        });
        this.methods.forEach(function(method) {
          extractWords(method.text || method.description || '');
        });
        words.sort();
        return words.join(' ');
    },

    getModuleInfo: function () {
        var self = this;
        var str = self.id;
        var specialNames = [
            'provider',
            'method',
            'factory',
            'service',
            'controller',
            'directive',
            'function',
            'filter'
        ]
        str = str.replace('#', '_');
        console.log(str);
        var match;
        var moduleInfo = {module: null, section: null, item: null, subItem: null};
        
        if(this.docType == 'input') {
            this.docType = 'directive';
        }
        
        if(match = str.match(/^(.*)[\\._]+/)) {
            moduleInfo.module = match[1];
            var modulePieces = moduleInfo.module.split('.');
            
            //Module pieces length may change - do not store reference
            for(var mi = 0; mi < modulePieces.length; mi++) {
                for(var i = 0, l = specialNames.length; i < l; i++) {
                    if(modulePieces[mi] == specialNames[i]) {
                        modulePieces.splice(mi);
                        mi--;
                    }
                }
            }
            moduleInfo.module = modulePieces.join('.');
        } else {
            /**
             * If this is something that cannot have child methods, it's an angular thing.
             */
            if(this.docType == 'directive' || this.docType == 'function' || this.docType == 'filter') {
                moduleInfo.module = 'angular';
            } else {
                moduleInfo.module = str;
            }
        }
        
        console.log("Module Name: " + moduleInfo.module)
        
        moduleInfo.section = this.docType;
        if(!moduleInfo.section) {
            if(match = str.match(/^.*\.(.*)[_]/)) {
                moduleInfo.section = match[1];
            }
            if(!moduleInfo.section && (match = str.match(/^.*\.(.*)/))) {
                moduleInfo.section = match[1];
            }
        }
        
        console.log("Module Section: " + moduleInfo.section)
        
        if(match = str.match(/^.*[_]([^_]+)/)) {
            moduleInfo.item = match[1];
        }
        if(!moduleInfo.item && (match = str.match(/^.*\.([^\._]+)$/))) {
            moduleInfo.item = match[1];
            if(this.docType != 'function') {
                moduleInfo.module = moduleInfo.module + '.' + match[1];
            }
        }
        if(!moduleInfo.item && moduleInfo.section === this.docType) {
            if(moduleInfo.module.indexOf('.') == -1) {
                moduleInfo.item = str;
            }
        }
        <commit_msg>Cleanup on more angular doc samples<commit_after>
/*========== DEPENDENCIES ==========*/

var nodeExtend = require('node.extend');
var colors = require('colors');

var DOM = require('./dom.js').DOM;
var htmlEscape = require('./dom.js').htmlEscape;

var doc_utils = require('./doc_utils.js');


/*========== PRIVATE VARIABLES AND METHODS ==========*/

var globalID = 0;
var NEW_LINE = /\n\r?/;
var ISDOC = /^(doc|doc-\S+)/i;
var HASGROUPSECTION = /^[^\.\:]{1,}\/(.*){1,}/i;

var COMPARE_ORDER = function (a, b) {
    var val = 'order';
    if (a[val] < b[val]) {
        return -1;
    } else if (a[val] > b[val]) {
        return 1;
    } else {
        return 0;
    }
};

/*========== Doc CLASS DEFINITION ==========*/

/**
 * [Doc description]
 * @param {[type]} configs_in [description]
 */
var Doc = function (configs_in) {

    var defaults = {

        //file information
        text: '',
        file: '',
        line: '',

        //category information
        group: 'main',
        section: 'section',

        //types
        requires: [],
        param: [],
        properties: [],
        methods: [],
        events: [],
        links: [],

        //what doc_api are we using?
        doc_api_extensions: {}
    };

    //merge in the defaults and configs
    nodeExtend(this, defaults, configs_in);

    if(!this.doc_api_extensions) {
        console.log("No doc_api extensions provided".red);
    }
};


Doc.METADATA_IGNORE = (function() {
  var words = require('fs').readFileSync(__dirname + '/ignore.words', 'utf8');
  return words.toString().split(/[,\s\n\r]+/gm);
})();


Doc.prototype = {

    /**
     * THIS PROTOTYPE METHOD EXTACTS RELEVANT KEYWORDS FOR SEARCHING IN THE TEXT AND DESCRIPTION FIELDS
     * @return {[type]} [description]
     */
    keywords: function keywords() {

        var kwords = {};
        var words = [];
        Doc.METADATA_IGNORE.forEach(function(ignore){ kwords[ignore] = true; });

        function extractWords(text) {
            var tokens = text.toLowerCase().split(/[\.\s,`'"#]+/mg);
            tokens.forEach(function(key){
                var match = key.match(/^((ng:|[\$_a-z])[\w\-_]+)/);
                if (match){
                    key = match[1];
                    if (!kwords[key]) {
                        kwords[key] = true;
                        words.push(key);
                    }
                }
            });
        }

        extractWords(this.text);
        this.properties.forEach(function(prop) {
          extractWords(prop.text || prop.description || '');
        });
        this.methods.forEach(function(method) {
          extractWords(method.text || method.description || '');
        });
        words.sort();
        return words.join(' ');
    },

    getModuleInfo: function () {
        var self = this;
        var str = self.id;
        var specialNames = [
            'provider',
            'method',
            'factory',
            'service',
            'controller',
            'directive',
            'function',
            'filter'
        ]
        str = str.replace('#', '_');
        console.log(str);
        var match;
        var moduleInfo = {module: null, section: null, item: null, subItem: null};
        
        if(this.docType == 'input') {
            this.docType = 'directive';
        }
        
        if(match = str.match(/^(.*)[\\._]+/)) {
            moduleInfo.module = match[1];
            var modulePieces = moduleInfo.module.split('.');
            
            //Module pieces length may change - do not store reference
            for(var mi = 0; mi < modulePieces.length; mi++) {
                for(var i = 0, l = specialNames.length; i < l; i++) {
                    if(modulePieces[mi] == specialNames[i]) {
                        modulePieces.splice(mi);
                        mi--;
                    }
                }
            }
            moduleInfo.module = modulePieces.join('.');
        } else {
            /**
             * If this is something that cannot have child methods, it's an angular thing.
             */
            if(this.docType == 'directive' || this.docType == 'function' || this.docType == 'filter') {
                moduleInfo.module = 'angular';
            } else {
                moduleInfo.module = str;
            }
        }
        
        if(moduleInfo.module.indexOf('Provider')) {
            moduleInfo.module = moduleInfo.module.replace('Provider', '');
        }
        
        
        console.log("Module Name: " + moduleInfo.module)
        
        moduleInfo.section = this.docType;
        if(!moduleInfo.section) {
            if(match = str.match(/^.*\.(.*)[_]/)) {
                moduleInfo.section = match[1];
            }
            if(!moduleInfo.section && (match = str.match(/^.*\.(.*)/))) {
                moduleInfo.section = match[1];
            }
        }
        
        console.log("Module Section: " + moduleInfo.section)
        
        if(match = str.match(/^.*[_]([^_]+)/)) {
            moduleInfo.item = match[1];
        }
        if(!moduleInfo.item && (match = str.match(/^.*\.([^\._]+)$/))) {
            moduleInfo.item = match[1];
            if(this.docType != 'function') {
                moduleInfo.module = moduleInfo.module + '.' + match[1];
            }
        }
        if(!moduleInfo.item && moduleInfo.section === this.docType) {
            if(moduleInfo.module.indexOf('.') == -1) {
                moduleInfo.item = str;
            }
        }
        
{'fraction_non_alphanumeric': 0.17152311793363362, 'fraction_numerical': 0.0031252872506664217, 'mean_word_length': 1.551594746716698, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5130045', 'n_tokens_mistral': 3268, 'n_tokens_neox': 3133, 'n_words': 630}