<commit_before>﻿(function () {
    var _;

    if (typeof (window) !== 'undefined')
        _ = window._;
    else
        _ = require("underscore");


    var Scrubber = function () {
        this.processors = [];
    };

    Scrubber.prototype.when = function (condition, fn) {
        if (typeof (condition) === 'function' && typeof (fn) === 'function') {
            this.processors.push({
                condition: condition,
                fn: fn
            });
        }
    };

    Scrubber.prototype.scrub = function (obj) {
        var self = this;

        var result = obj;

        if (_.isArray(obj)) {
            _.each(obj, function (x, i) {
                var scrubbed = self.scrub(x);
                obj[i] = scrubbed || x;
            });
        }
        else if (_.isObject(obj)) {
            for (var objKey in obj) {

                var key = objKey;

                var flagCtx = {
                    parent: obj,
                    key: key,
                    value: obj[key]
                };

                var processCtx = {
                    parent: obj,
                    key: key,
                    value: obj[key],
                    scrub: function (value) {
                        self.scrub(value);
                    }
                }

                _.each(self.processors, function (processor) {
                    if (processor.condition(flagCtx)) {
                        var res = processor.fn(processCtx);<commit_msg>not scrubbing null values<commit_after>﻿(function () {
    var _;

    if (typeof (window) !== 'undefined')
        _ = window._;
    else
        _ = require("underscore");


    var Scrubber = function () {
        this.processors = [];
    };

    Scrubber.prototype.when = function (condition, fn) {
        if (typeof (condition) === 'function' && typeof (fn) === 'function') {
            this.processors.push({
                condition: condition,
                fn: fn
            });
        }
    };

    Scrubber.prototype.scrub = function (obj) {
        var self = this;

        var result = obj;

        if (_.isArray(obj)) {
            _.each(obj, function (x, i) {
                if (x) {
                    var scrubbed = self.scrub(x);
                    obj[i] = scrubbed || x;
                }
            });
        }
        else if (_.isObject(obj)) {
            for (var objKey in obj) {

                var key = objKey;

                var flagCtx = {
                    parent: obj,
                    key: key,
                    value: obj[key]
                };

                var processCtx = {
                    parent: obj,
                    key: key,
                    value: obj[key],
                    scrub: function (value) {
                        self.scrub(value);
                    }
                }

                _.each(self.processors, function (processor) {
                    if (processor.condition(flagCtx)) {
                        var res = processor.fn(processCtx);
{'fraction_non_alphanumeric': 0.11633256654617154, 'fraction_numerical': 0.0, 'mean_word_length': 1.03475935828877, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2712589', 'n_tokens_mistral': 831, 'n_tokens_neox': 774, 'n_words': 182}