<commit_before>var fs = require('fs');

var functions = {
    parents_of_nr: {
        map: function (doc) {
            //var property = function (key) {
            //    return function (obj) {
            //        return obj == null ? void 0 : obj[key];
            //    };
            //};
            //var MAX_ARRAY_INDEX = Math.pow(2, 53) - 1;
            //var getLength = property('length');
            //var isArrayLike = function (collection) {
            //    var length = getLength(collection);
            //    return typeof length == 'number' && length >= 0 && length <= MAX_ARRAY_INDEX;
            //};


            var xml = null;
            try {
                xml = require('views/lib/xml');
            } catch (err) {
                xml = require('../xml_util.js');
            }
            function emitNrParents(node) {
                xml.forAllChildren(node, function (child) {
                    if (xml.getTagName(child) == 'nr') {
                        emit([xml.getTagName(node), doc._id], 1);
                    }

                    emitNrParents(child);
                });
            }

            if (doc.xml) {
                emitNrParents(doc.xml);
            }
        },
        reduce: "_sum"
    },
    words_in_title: {
        map: function (doc) {
            var xml = null;
            try {
                xml = require('views/lib/xml');
            } catch (err) {
                xml = require('../xml_util.js');
            }
            if (doc.xml && xml.hasTag(doc.xml, "title")) {
                var nat = null;
                try {
                    nat = require('views/lib/natural');
                } catch (err) {
                    nat = require('natural');
                }
                var tokenizer = new nat.WordPunctTokenizer();

                var emitRecursive = function (node) {
                    if (typeof node == 'string') {
                        var tokens = tokenizer.tokenize(node);
                        for (var i = 0; i < tokens.length; i++) {
                            var token = tokens[i].toLowerCase().trim();
                            emit(token, 1);
                        }
                    } else {
                        xml.forAllChildren(node, function (child) {
                            emitRecursive(child);
                        });
                    }
                };

                var emitTitleTokens = function (node) {
                    xml.forAllChildren(node, function (child) {
                        if (xml.getTagName(child) == 'title') {
                            emitRecursive(child);
                        } else {
                            emitTitleTokens(child);
                        }
                    });
                };

                emitTitleTokens(doc.xml);
            }
        },
        reduce: "_sum",
        dbcopy: "word_count_title"
    },
    section_nrs: {
        map: function (doc) {
            var xml = null;
            try {
                xml = require('views/lib/xml');
            } catch (err) {
                xml = require('../xml_util.js');
            }

            var elementToEmitFrom = 'nr';

            if (xml.hasTag(doc.xml, elementToEmitFrom)) {
                var nat = null;
                try {
                    nat = require('views/lib/natural');
                } catch (err) {
                    nat = require('natural');
                }
                var tokenizer = new nat.WordPunctTokenizer();

                var emitRecursive = function (node) {
                    if (typeof node == 'string') {
                        var tokens = tokenizer.tokenize(node);
                        for (var i = 0; i < tokens.length; i++) {
                            var token = tokens[i].toLowerCase().trim();
                            emit(token, 1);
                        }
                    } else {
                        xml.forAllChildren(node, function (child) {
                            emitRecursive(child);
                        });
                    }
                };

                var emitNrTokens = function (node) {
                    xml.forAllChildren(node, function (child) {
                        if (xml.getTagName(node) == elementToEmitFrom) {
                            emitRecursive(node);
                        } else {
                            emitNrTokens(child);
                        }
                    });
                };

                emitNrTokens(doc.xml);
            }
        },<commit_msg>Don't emit whitespace when counting words in title/nr tags<commit_after>var fs = require('fs');

var functions = {
    parents_of_nr: {
        map: function (doc) {
            //var property = function (key) {
            //    return function (obj) {
            //        return obj == null ? void 0 : obj[key];
            //    };
            //};
            //var MAX_ARRAY_INDEX = Math.pow(2, 53) - 1;
            //var getLength = property('length');
            //var isArrayLike = function (collection) {
            //    var length = getLength(collection);
            //    return typeof length == 'number' && length >= 0 && length <= MAX_ARRAY_INDEX;
            //};


            var xml = null;
            try {
                xml = require('views/lib/xml');
            } catch (err) {
                xml = require('../xml_util.js');
            }
            function emitNrParents(node) {
                xml.forAllChildren(node, function (child) {
                    if (xml.getTagName(child) == 'nr') {
                        emit([xml.getTagName(node), doc._id], 1);
                    }

                    emitNrParents(child);
                });
            }

            if (doc.xml) {
                emitNrParents(doc.xml);
            }
        },
        reduce: "_sum"
    },
    words_in_title: {
        map: function (doc) {
            var xml = null;
            try {
                xml = require('views/lib/xml');
            } catch (err) {
                xml = require('../xml_util.js');
            }
            if (doc.xml && xml.hasTag(doc.xml, "title")) {
                var nat = null;
                try {
                    nat = require('views/lib/natural');
                } catch (err) {
                    nat = require('natural');
                }
                var tokenizer = new nat.WordPunctTokenizer();

                var emitRecursive = function (node) {
                    if (typeof node == 'string') {
                        var tokens = tokenizer.tokenize(node);
                        for (var i = 0; i < tokens.length; i++) {
                            var token = tokens[i].toLowerCase().replace(/\s/g,'');
                            if (token.length > 0) {
                                emit(token, 1);//Don't emit whitespace
                            }
                        }
                    } else {
                        xml.forAllChildren(node, function (child) {
                            emitRecursive(child);
                        });
                    }
                };

                var emitTitleTokens = function (node) {
                    xml.forAllChildren(node, function (child) {
                        if (xml.getTagName(child) == 'title') {
                            emitRecursive(child);
                        } else {
                            emitTitleTokens(child);
                        }
                    });
                };

                emitTitleTokens(doc.xml);
            }
        },
        reduce: "_sum",
        dbcopy: "word_count_title"
    },
    section_nrs: {
        map: function (doc) {
            var xml = null;
            try {
                xml = require('views/lib/xml');
            } catch (err) {
                xml = require('../xml_util.js');
            }

            var elementToEmitFrom = 'nr';

            if (xml.hasTag(doc.xml, elementToEmitFrom)) {
                var nat = null;
                try {
                    nat = require('views/lib/natural');
                } catch (err) {
                    nat = require('natural');
                }
                var tokenizer = new nat.WordPunctTokenizer();

                var emitRecursive = function (node) {
                    if (typeof node == 'string') {
                        var tokens = tokenizer.tokenize(node);
                        for (var i = 0; i < tokens.length; i++) {
                            var token = tokens[i].toLowerCase().replace(/\s/g,'');
                            if (token.length > 0) {
                                emit(token, 1);//Don't emit whitespace
                            }
                        }
                    } else {
                        xml.forAllChildren(node, function (child) {
                            emitRecursive(child);
                        });
                    }
                };

                var emitNrTokens = function (node) {
                    xml.forAllChildren(node, function (child) {
                        if (xml.getTagName(node) == elementToEmitFrom) {
                            emitRecursive(node);
                        } else {
                            emitNrTokens(child);
                        }
                    });
                };

                emitNrTokens(doc.xml);
            }
        },
{'fraction_non_alphanumeric': 0.11308834368748026, 'fraction_numerical': 0.002527113825418553, 'mean_word_length': 0.8830293417922284, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 66}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3738895', 'n_tokens_mistral': 2410, 'n_tokens_neox': 2258, 'n_words': 470}