<commit_before>/*!
 * react-backbone v0.23.0
 * https://github.com/jhudson8/react-backbone
 *
 * Copyright (c) 2014 <NAME><<EMAIL>>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */
(function(main) {
    if (typeof define === 'function' && define.amd) {
        define([], function() {
            // with AMD
            //  require(
            //    ['react', 'backbone', 'underscore', 'jquery', react-backbone'], function(React, Backbone, _, $, reactBackbone) {
            //    reactBackbone(React, Backbone, _, $); 
            //  });
            return main;
        });
    } else if (typeof exports !== 'undefined' && typeof require !== 'undefined') {
        // with CommonJS
        // require('react-backbone')(require('react'), require('backbone'), require('underscore')), require('jquery'));
        module.exports = main;
    } else {
        main(React, Backbone, _, $);
    }
})(function(React, Backbone, _, $) {

    // main body start
    // create local references to existing vars
    var namespace = 'react-backbone.',
        xhrEventName = Backbone.xhrEventName,
        xhrModelLoadingAttribute = Backbone.xhrModelLoadingAttribute,
        getState = React.mixins.getState,
        setState = React.mixins.setState,
        logDebugWarnings = React.reactBackboneDebugWarnings;
    if (_.isUndefined(logDebugWarnings)) {
        logDebugWarnings = true;
    }

    // use Backbone.Events as the events impl if none is already defined
    React.events.mixin = React.events.mixin || Backbone.Events;

    function addMixin() {
        var args = _.toArray(arguments);
        if (_.isString(args)) {
            args[0] = namespace + args[0];
        } else {
            args.name = namespace + args.name;
        }
        React.mixins.add.apply(React.mixins, args);
    }

    function firstModel(component) {
        if (component.getModel) {
            return component.getModel();
        }
    }

    function getModelOrCollections(type, context, callback, props) {
        if (type === 'collection') {
            return context.getCollection(callback, props);
        } else {
            return context.getModel(callback, props);
        }
    }

    /**
     * Return either an array of elements (if src provided is not an array)
     * or undefined if the src is undefined
     */
    function ensureArray(src) {
        if (!src) {
            return;
        }
        if (_.isArray(src)) {
            return src;
        }
        return [src];
    }

    /**
     * Return a callback function that will provide the model or collection
     */
    function targetModelOrCollections(type, context, modelOrCollectionToUse) {
        return function() {
            if (modelOrCollectionToUse) {
                return modelOrCollectionToUse;
            }
            return getModelOrCollections(type, context);
        };
    }

    /**
     * Return a model attribute key used for attribute specific operations
     */
    function getKey(context) {
        if (context.getModelKey) {
            return context.getModelKey();
        }
        return context.props.name || context.props.key || context.props.ref;
    }
    React.mixins.getModelKey = getKey;

    /**
     * Returns model validation errors in a standard format.
     * expected return is { field1Key: errorMessage, field2Key: errorMessage, ... }
     *
     * This implementation will look for [{field1Key: message}, {field2Key: message}, ...]
     */
    function modelIndexErrors(errors, context) {
        if (context && context.modelIndexErrors) {
            return context.modelIndexErrors(errors);
        }
        if (Array.isArray(errors)) {
            var rtn = {};
            _.each(errors, function(data) {
                for (var name in data) {
                    if (data.hasOwnProperty(name)) {
                        rtn[name] = data[name];
                    }
                }
            });
            return rtn;
        } else {
            return errors;
        }
    }
    React.mixins.modelIndexErrors = modelIndexErrors;

    /**
     * Return the callback function (key, model) if both the model exists
     * and the model key is available
     */
    function ifModelAndKey(component, callback) {
        var model = firstModel(component);
        if (model) {
            var key = getKey(component);
            return callback(key, model);
        }
    }

    /**
     * Utility method used to handle model/collection events
     */
    function modelOrCollectionEventHandler(type, identifier, context, eventFormat, callback) {
        var keys = Array.isArray(identifier) ? identifier : ensureArray(context.props[identifier]),
            key, eventName;
        if (keys) {
            // register the event handlers to watch for these events
            for (var i = 0; i < keys.length; i++) {
                key = keys[i];
                eventName = eventFormat.replace('{key}', key);
                context[type + 'On'](eventName, _.bind(callback, context), this);
            }
            return keys;
        }
    }

    /**
     * Provide modelOn and modelOnce argument with proxied arguments
     * arguments are event, callback, context
     */
    function modelOrCollectionOnOrOnce(modelType, type, args, _this, _modelOrCollection) {
        var modelEvents = getModelAndCollectionEvents(modelType, _this);
        var ev = args[0];
        var cb = args[1];
        var ctx = args[2];
        modelEvents[ev] = {
            type: type,
            ev: ev,
            cb: cb,
            ctx: ctx
        };

        function _on(modelOrCollection) {
            _this[type === 'on' ? 'listenTo' : 'listenToOnce'](modelOrCollection, ev, cb, ctx);
        }
        if (_this.isMounted()) {
            if (_modelOrCollection) {
                _on(_modelOrCollection);
            } else {
                getModelOrCollections(modelType, _this, _on);
            }
        }
    }

    function unbindAndRebind(type, unbindModel, bindModel, context, force) {
        if (unbindModel === bindModel && !force) {
            // nothing to do
            return;
        }
        var events = getModelAndCollectionEvents(type, context);
        if (unbindModel) {
            context.trigger(type + ':unbind', bindModel);
            // turn off models that will be replaced
            _.each(events, function(eventData) {
                this.stopListening(unbindModel, eventData.ev, eventData.cb, eventData.ctx);
            }, context);
        }
        if (bindModel) {
            context.trigger(type + ':bind', bindModel);
            _.each(events, function(eventData) {
                modelOrCollectionOnOrOnce(type, eventData.type,
                        [eventData.ev, eventData.cb, eventData.ctx], this, bindModel, force);
            }, context);
        }
    }

    /**
     * Return all bound model events
     */
    function getModelAndCollectionEvents(type, context) {
        var key = '__' + type + 'Events',
            modelEvents = getState(key, context);
        if (!modelEvents) {
            modelEvents = {};
            var stateVar = {};
            stateVar[key] = modelEvents;
            setState(stateVar, context);
        }
        return modelEvents;
    }

    // loading state helpers
    function pushLoadingState(xhrEvent, modelOrCollection, context) {
        var currentLoads = getState('loading', context),
            currentlyLoading = currentLoads && currentLoads.length;
        if (!currentLoads) {
            currentLoads = [];
        }
        if (_.isArray(currentLoads)) {
            currentLoads.push(xhrEvent);
            if (!currentlyLoading) {
                setState({
                    loading: currentLoads
                }, context);
            }

            xhrEvent.on('complete', function() {
                popLoadingState(xhrEvent, modelOrCollection, context);
            });
        }
    }

    // remove the xhrEvent from the loading state
    function popLoadingState(xhrEvent, modelOrCollection, context) {
        var currentLoads = getState('loading', context);
        if (_.isArray(currentLoads)) {
            var i = currentLoads.indexOf(xhrEvent);
            while (i >= 0) {
                currentLoads.splice(i, 1);
                i = currentLoads.indexOf(xhrEvent);
            }
            if (!currentLoads.length) {
                setState({
                    loading: false
                }, context);
            }
        }
    }

    // if there is any current xhrEvent that match the method, add a reference to it with this context
    function joinCurrentModelActivity(method, modelOrCollection, context) {
        var xhrActivity = modelOrCollection[xhrModelLoadingAttribute];
        if (xhrActivity) {
            _.each(xhrActivity, function(xhrEvent) {
                if (!method || xhrEvent.method === method) {
                    // this is one that is applicable
                    pushLoadingState(xhrEvent, modelOrCollection, context);
                }
            });
        }
    }

    // helpers to get and set a model value when only the component is known
    Backbone.input = Backbone.input || {};
    var getModelValue = Backbone.input.getModelValue = function(component) {
        return ifModelAndKey(component, function(key, model) {
            return model.get(key);
        });
    };
    Backbone.input.setModelValue = function(component, value, options) {
        return ifModelAndKey(component, function(key, model) {
            return model.set(key, value, options);
        });
    };

    // create mixins that are duplicated for both models and collections
    _.each([{
        type: 'model',
        defaultParams: [['model']],
        capType: 'Model',
        changeEvents: ['change']
    }, {
        type: 'collection',
        defaultParams: [['collection']],
        capType: 'Collection',
        changeEvents: ['add', 'remove', 'reset', 'sort']
    }], function(typeData) {
        var getThings = 'get' + typeData.capType;

        /**
         * Simple overrideable mixin to get/set models or collections.  Model/Collection can
         * be set on props or by calling setModel or setCollection.
         * This will return the *first* model (if there are multiple models) but a callback method
         * can be provided as the first parameter function(model, propName) which will be called
         * for every available model.
         */
        var typeAware = function(referenceArgs) {
            // use initiatedOnce format so model prop names can be changed (or multiple can be used)
            // for example, to get all model change monitoring with models set as "foo" and "bar" props, simply use
            // this will not render as a result of 2 way binding (unless the bind attribute is not === true)
            // mixins: ['modelAware("foo", "bar")', 'modelChangeAware']
            var rtn = {};
            rtn[getThings] = function(callback, props) {
                var _referenceArgs = referenceArgs,
                    propsProvided = !!props;
                props = props || this.props;
                if (!_referenceArgs || _referenceArgs.length === 0) {
                    _referenceArgs = typeData.defaultParams;
                }

                var firstModel, singleReferenceArgs;
                for (var i=0; i<_referenceArgs.length; i++) {
                    singleReferenceArgs = _referenceArgs[i];
                    for (var j=0; j<singleReferenceArgs.length; j++) {
                        var propName = singleReferenceArgs[j],
                            obj = getState(propName, this) || props[propName];
                        if (obj) {
                            firstModel = firstModel || obj;
                            if (callback) {
                                callback.call(this, obj, propName);
                            } else {
                                return obj;
                            }
                        } else if (propsProvided && callback && propName) {
                            // make callback anyway to let callers know of the prop transition
                            callback.call(this, undefined, propName);
                        }
                    }
                }
                return firstModel;
            };
            rtn['set' + typeData.capType] = function(modelOrCollection, key) {
                key = key || typeData.type;
                var stateData = {};
                var prevModelOrCollection;
                this['get' + typeData.capType](function(modelOrCollection, _key) {
                    if (_key === key) {
                        prevModelOrCollection = modelOrCollection;
                    }
                });
                // unbind previous model
                unbindAndRebind(typeData.type, prevModelOrCollection, prevModelOrCollection, this);
                stateData[key] = modelOrCollection;
                this.trigger(typeData.type + ':set', modelOrCollection, key, prevModelOrCollection);
            };
            return rtn;
        };
        addMixin({ name: typeData.type + 'Aware', initiatedOnce: true }, typeAware, 'state');

        /**
         * Exposes model binding registration functions that will
         * be cleaned up when the component is unmounted and not actually registered
         * until the component is mounted.  The context will be "this" if not provided.
         *
         * This is similar to the "listenTo" mixin but model event bindings here will
         * be transferred to another model if a new one is set on the props.
         */
        var typeEvents = {
            getInitialState: function() {
                // model sanity check
                getModelOrCollections(typeData.type, this, function(obj, propName) {
                    if (logDebugWarnings && !obj.off || !obj.on) {
                        console.error('props.' + propName + ' does not implement on/off functions - you will see event binding problems (object logged to console below)');
                        console.log(obj);
                    }
                });
                return {};
            },<commit_msg>remove setModel/setCollection<commit_after>/*!
 * react-backbone v0.23.0
 * https://github.com/jhudson8/react-backbone
 *
 * Copyright (c) 2014 <NAME><joehud_AT_gmail.com>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */
(function(main) {
    if (typeof define === 'function' && define.amd) {
        define([], function() {
            // with AMD
            //  require(
            //    ['react', 'backbone', 'underscore', 'jquery', react-backbone'], function(React, Backbone, _, $, reactBackbone) {
            //    reactBackbone(React, Backbone, _, $); 
            //  });
            return main;
        });
    } else if (typeof exports !== 'undefined' && typeof require !== 'undefined') {
        // with CommonJS
        // require('react-backbone')(require('react'), require('backbone'), require('underscore')), require('jquery'));
        module.exports = main;
    } else {
        main(React, Backbone, _, $);
    }
})(function(React, Backbone, _, $) {

    // main body start
    // create local references to existing vars
    var namespace = 'react-backbone.',
        xhrEventName = Backbone.xhrEventName,
        xhrModelLoadingAttribute = Backbone.xhrModelLoadingAttribute,
        getState = React.mixins.getState,
        setState = React.mixins.setState,
        logDebugWarnings = React.reactBackboneDebugWarnings;
    if (_.isUndefined(logDebugWarnings)) {
        logDebugWarnings = true;
    }

    // use Backbone.Events as the events impl if none is already defined
    React.events.mixin = React.events.mixin || Backbone.Events;

    function addMixin() {
        var args = _.toArray(arguments);
        if (_.isString(args)) {
            args[0] = namespace + args[0];
        } else {
            args.name = namespace + args.name;
        }
        React.mixins.add.apply(React.mixins, args);
    }

    function firstModel(component) {
        if (component.getModel) {
            return component.getModel();
        }
    }

    function getModelOrCollections(type, context, callback, props) {
        if (type === 'collection') {
            return context.getCollection(callback, props);
        } else {
            return context.getModel(callback, props);
        }
    }

    /**
     * Return either an array of elements (if src provided is not an array)
     * or undefined if the src is undefined
     */
    function ensureArray(src) {
        if (!src) {
            return;
        }
        if (_.isArray(src)) {
            return src;
        }
        return [src];
    }

    /**
     * Return a callback function that will provide the model or collection
     */
    function targetModelOrCollections(type, context, modelOrCollectionToUse) {
        return function() {
            if (modelOrCollectionToUse) {
                return modelOrCollectionToUse;
            }
            return getModelOrCollections(type, context);
        };
    }

    /**
     * Return a model attribute key used for attribute specific operations
     */
    function getKey(context) {
        if (context.getModelKey) {
            return context.getModelKey();
        }
        return context.props.name || context.props.key || context.props.ref;
    }
    React.mixins.getModelKey = getKey;

    /**
     * Returns model validation errors in a standard format.
     * expected return is { field1Key: errorMessage, field2Key: errorMessage, ... }
     *
     * This implementation will look for [{field1Key: message}, {field2Key: message}, ...]
     */
    function modelIndexErrors(errors, context) {
        if (context && context.modelIndexErrors) {
            return context.modelIndexErrors(errors);
        }
        if (Array.isArray(errors)) {
            var rtn = {};
            _.each(errors, function(data) {
                for (var name in data) {
                    if (data.hasOwnProperty(name)) {
                        rtn[name] = data[name];
                    }
                }
            });
            return rtn;
        } else {
            return errors;
        }
    }
    React.mixins.modelIndexErrors = modelIndexErrors;

    /**
     * Return the callback function (key, model) if both the model exists
     * and the model key is available
     */
    function ifModelAndKey(component, callback) {
        var model = firstModel(component);
        if (model) {
            var key = getKey(component);
            return callback(key, model);
        }
    }

    /**
     * Utility method used to handle model/collection events
     */
    function modelOrCollectionEventHandler(type, identifier, context, eventFormat, callback) {
        var keys = Array.isArray(identifier) ? identifier : ensureArray(context.props[identifier]),
            key, eventName;
        if (keys) {
            // register the event handlers to watch for these events
            for (var i = 0; i < keys.length; i++) {
                key = keys[i];
                eventName = eventFormat.replace('{key}', key);
                context[type + 'On'](eventName, _.bind(callback, context), this);
            }
            return keys;
        }
    }

    /**
     * Provide modelOn and modelOnce argument with proxied arguments
     * arguments are event, callback, context
     */
    function modelOrCollectionOnOrOnce(modelType, type, args, _this, _modelOrCollection) {
        var modelEvents = getModelAndCollectionEvents(modelType, _this);
        var ev = args[0];
        var cb = args[1];
        var ctx = args[2];
        modelEvents[ev] = {
            type: type,
            ev: ev,
            cb: cb,
            ctx: ctx
        };

        function _on(modelOrCollection) {
            _this[type === 'on' ? 'listenTo' : 'listenToOnce'](modelOrCollection, ev, cb, ctx);
        }
        if (_this.isMounted()) {
            if (_modelOrCollection) {
                _on(_modelOrCollection);
            } else {
                getModelOrCollections(modelType, _this, _on);
            }
        }
    }

    function unbindAndRebind(type, unbindModel, bindModel, context, force) {
        if (unbindModel === bindModel && !force) {
            // nothing to do
            return;
        }
        var events = getModelAndCollectionEvents(type, context);
        if (unbindModel) {
            context.trigger(type + ':unbind', bindModel);
            // turn off models that will be replaced
            _.each(events, function(eventData) {
                this.stopListening(unbindModel, eventData.ev, eventData.cb, eventData.ctx);
            }, context);
        }
        if (bindModel) {
            context.trigger(type + ':bind', bindModel);
            _.each(events, function(eventData) {
                modelOrCollectionOnOrOnce(type, eventData.type,
                        [eventData.ev, eventData.cb, eventData.ctx], this, bindModel, force);
            }, context);
        }
    }

    /**
     * Return all bound model events
     */
    function getModelAndCollectionEvents(type, context) {
        var key = '__' + type + 'Events',
            modelEvents = getState(key, context);
        if (!modelEvents) {
            modelEvents = {};
            var stateVar = {};
            stateVar[key] = modelEvents;
            setState(stateVar, context);
        }
        return modelEvents;
    }

    // loading state helpers
    function pushLoadingState(xhrEvent, modelOrCollection, context) {
        var currentLoads = getState('loading', context),
            currentlyLoading = currentLoads && currentLoads.length;
        if (!currentLoads) {
            currentLoads = [];
        }
        if (_.isArray(currentLoads)) {
            currentLoads.push(xhrEvent);
            if (!currentlyLoading) {
                setState({
                    loading: currentLoads
                }, context);
            }

            xhrEvent.on('complete', function() {
                popLoadingState(xhrEvent, modelOrCollection, context);
            });
        }
    }

    // remove the xhrEvent from the loading state
    function popLoadingState(xhrEvent, modelOrCollection, context) {
        var currentLoads = getState('loading', context);
        if (_.isArray(currentLoads)) {
            var i = currentLoads.indexOf(xhrEvent);
            while (i >= 0) {
                currentLoads.splice(i, 1);
                i = currentLoads.indexOf(xhrEvent);
            }
            if (!currentLoads.length) {
                setState({
                    loading: false
                }, context);
            }
        }
    }

    // if there is any current xhrEvent that match the method, add a reference to it with this context
    function joinCurrentModelActivity(method, modelOrCollection, context) {
        var xhrActivity = modelOrCollection[xhrModelLoadingAttribute];
        if (xhrActivity) {
            _.each(xhrActivity, function(xhrEvent) {
                if (!method || xhrEvent.method === method) {
                    // this is one that is applicable
                    pushLoadingState(xhrEvent, modelOrCollection, context);
                }
            });
        }
    }

    // helpers to get and set a model value when only the component is known
    Backbone.input = Backbone.input || {};
    var getModelValue = Backbone.input.getModelValue = function(component) {
        return ifModelAndKey(component, function(key, model) {
            return model.get(key);
        });
    };
    Backbone.input.setModelValue = function(component, value, options) {
        return ifModelAndKey(component, function(key, model) {
            return model.set(key, value, options);
        });
    };

    // create mixins that are duplicated for both models and collections
    _.each([{
        type: 'model',
        defaultParams: [['model']],
        capType: 'Model',
        changeEvents: ['change']
    }, {
        type: 'collection',
        defaultParams: [['collection']],
        capType: 'Collection',
        changeEvents: ['add', 'remove', 'reset', 'sort']
    }], function(typeData) {
        var getThings = 'get' + typeData.capType;

        /**
         * Simple overrideable mixin to get/set models or collections.  Model/Collection can
         * be set on props or by calling setModel or setCollection.
         * This will return the *first* model (if there are multiple models) but a callback method
         * can be provided as the first parameter function(model, propName) which will be called
         * for every available model.
         */
        var typeAware = function(referenceArgs) {
            // use initiatedOnce format so model prop names can be changed (or multiple can be used)
            // for example, to get all model change monitoring with models set as "foo" and "bar" props, simply use
            // this will not render as a result of 2 way binding (unless the bind attribute is not === true)
            // mixins: ['modelAware("foo", "bar")', 'modelChangeAware']
            var rtn = {};
            rtn[getThings] = function(callback, props) {
                var _referenceArgs = referenceArgs,
                    propsProvided = !!props;
                props = props || this.props;
                if (!_referenceArgs || _referenceArgs.length === 0) {
                    _referenceArgs = typeData.defaultParams;
                }

                var firstModel, singleReferenceArgs;
                for (var i=0; i<_referenceArgs.length; i++) {
                    singleReferenceArgs = _referenceArgs[i];
                    for (var j=0; j<singleReferenceArgs.length; j++) {
                        var propName = singleReferenceArgs[j],
                            obj = getState(propName, this) || props[propName];
                        if (obj) {
                            firstModel = firstModel || obj;
                            if (callback) {
                                callback.call(this, obj, propName);
                            } else {
                                return obj;
                            }
                        } else if (propsProvided && callback && propName) {
                            // make callback anyway to let callers know of the prop transition
                            callback.call(this, undefined, propName);
                        }
                    }
                }
                return firstModel;
            };
            return rtn;
        };
        addMixin({ name: typeData.type + 'Aware', initiatedOnce: true }, typeAware, 'state');

        /**
         * Exposes model binding registration functions that will
         * be cleaned up when the component is unmounted and not actually registered
         * until the component is mounted.  The context will be "this" if not provided.
         *
         * This is similar to the "listenTo" mixin but model event bindings here will
         * be transferred to another model if a new one is set on the props.
         */
        var typeEvents = {
            getInitialState: function() {
                // model sanity check
                getModelOrCollections(typeData.type, this, function(obj, propName) {
                    if (logDebugWarnings && !obj.off || !obj.on) {
                        console.error('props.' + propName + ' does not implement on/off functions - you will see event binding problems (object logged to console below)');
                        console.log(obj);
                    }
                });
                return {};
            },
{'fraction_non_alphanumeric': 0.09927247125088008, 'fraction_numerical': 0.0016763335233177994, 'mean_word_length': 1.8254238893625083, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 10, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2437842', 'n_tokens_mistral': 7657, 'n_tokens_neox': 7441, 'n_words': 2252}