<commit_before>'use strict';


module.exports = {
    type: function(obj) {
        return /\[object (\w+)\]/.exec(Object.prototype.toString.call(obj))[1].toLowerCase()
    },
    copyArray: function(arr) {
        var len = arr.length
        var nArr = new Array(len)
        while (len--) {
            nArr[len] = arr[len]
        }
        return nArr
    },
    objEach: function(obj, fn) {
        if (!obj) return
        for (var key in obj) {
            if (obj.hasOwnProperty(key)) {
                fn(key, obj[key])
            }
        }
    },
    /**
     *  two level diff
     */
    diff: function(next, pre) {
        var method
        if (this.type(next) == 'array' && this.type(pre) == 'array')
            method = this.arrayDiff
        else if (this.type(next) == 'object' && this.type(pre) == 'object')
            method = this.objDiff
        else method = this.valueDiff

        return method.call(this, next, pre)
    },
    objDiff: function(next, pre) {
        var nkeys = Object.keys(next)
        var pkeys = Object.keys(pre)
        if (nkeys.length != pkeys.length) return true

        var that = this
        return nkeys.some(function(k) {
            return (!~pkeys.indexOf(k)) || that.valueDiff(next[k], pre[k])
        })
    },
    arrayDiff: function(next, pre) {
        if (next.length !== pre.length) return true
        var that = this
        next.some(function(item, index) {
            return that.valueDiff(item, pre[index])
        })
    },
    valueDiff: function(next, pre) {
        return next !== pre || next instanceof Object
    }
}<commit_msg>add walk util<commit_after>'use strict';


module.exports = {
    type: function(obj) {
        return /\[object (\w+)\]/.exec(Object.prototype.toString.call(obj))[1].toLowerCase()
    },
    copyArray: function(arr) {
        var len = arr.length
        var nArr = new Array(len)
        while (len--) {
            nArr[len] = arr[len]
        }
        return nArr
    },
    objEach: function(obj, fn) {
        if (!obj) return
        for (var key in obj) {
            if (obj.hasOwnProperty(key)) {
                fn(key, obj[key])
            }
        }
    },
    /**
     *  two level diff
     */
    diff: function(next, pre) {
        var method
        if (this.type(next) == 'array' && this.type(pre) == 'array')
            method = this.arrayDiff
        else if (this.type(next) == 'object' && this.type(pre) == 'object')
            method = this.objDiff
        else method = this.valueDiff

        return method.call(this, next, pre)
    },
    objDiff: function(next, pre) {
        var nkeys = Object.keys(next)
        var pkeys = Object.keys(pre)
        if (nkeys.length != pkeys.length) return true

        var that = this
        return nkeys.some(function(k) {
            return (!~pkeys.indexOf(k)) || that.valueDiff(next[k], pre[k])
        })
    },
    arrayDiff: function(next, pre) {
        if (next.length !== pre.length) return true
        var that = this
        next.some(function(item, index) {
            return that.valueDiff(item, pre[index])
        })
    },
    valueDiff: function(next, pre) {
        return next !== pre || next instanceof Object
    },
    walk: function(node, fn) {
        node = fn(node) === false ? node.nextSibling : node.firstChild

        while (node) {
            this.walk(node, fn)
            node = node.nextSibling
        }
    }
}
{'fraction_non_alphanumeric': 0.14352256186317322, 'fraction_numerical': 0.0005822416302765648, 'mean_word_length': 1.7687348912167606, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4570448', 'n_tokens_mistral': 1062, 'n_tokens_neox': 1059, 'n_words': 248}