<commit_before>define([
    'jquery',
    'd3',
    'lodash',
    'base/utils',
    'base/class',
    'base/intervals',
    'base/events',
    'base/data',
], function($, d3, _, utils, Class, Intervals, Events, DataManager) {

    var model = Class.extend({

        /**
         * Initializes the model.
         * @param {Object} values The initial values of this model
         * @param {Object} parent reference to parent
         * @param {Object} bind Initial events to bind
         */
        init: function(values, parent, bind) {

            this._type = this._type || "model"; //type of this model
            this._id = _.uniqueId("m"); //model unique id
            this._data = {}; //holds attributes of this model
            this._parent = parent; //parent model
            this._set = false; //is this model set?
            this._ready = false; //is this model ready?
            this._debugEvents = this._debugEvents || false;

            //intervals should be the same from tool
            this._intervals = this.getIntervals();
            //each model has its own event handling
            this._events = new Events();

            //will the model be hooked to data?
            this._hooks = {};
            this._loading = []; //array of processes that are loading
            this._items = []; //holds hook items for this hook

            //bind initial events
            if (bind) {
                this.on(bind);
            }

            //initial values
            if (values) {
                this.set(values);
            }
        },

        /* ==========================
         * Getters and Setters
         * ==========================
         */

        /**
         * Gets an attribute from this model or all fields.
         * @param attr Optional attribute
         * @returns attr value or all values if attr is undefined
         */
        get: function(attr) {
            if (!attr) {
                return this._data;
            }
            return this._data[attr];
        },

        /**
         * Sets an attribute or multiple for this model
         * @param attr property name
         * @param val property value (object or value)
         * @param {Boolean} silent Prevents events from being fired
         * @param {Boolean} block_validation prevents model validation
         * @returns defer defer that will be resolved when set is done
         */
        set: function(attr, val, silent, block_validation) {

            var defer = $.Deferred(),
                promises = [],
                events = [];

            //expect object as default
            if (!_.isPlainObject(attr)) {
                var obj = {};
                obj[attr] = val;
                return this.set(obj, silent, block_validation);
            }

            //correct format
            block_validation = silent;
            silent = val;
            for (var a in attr) {

                var vals = attr[a],
                    promise;
                //if it's an object, set or create submodel
                if (_.isPlainObject(vals)) {
                    if (this._data[a] && utils.isModel(this._data[a])) {
                        promise = this._data[a].set(vals, silent, block_validation);
                    }
                    //submodel doesnt exist, create it
                    else {
                        promise = this._initSubmodel(a, vals);
                    }
                }
                //otherwise, just set value :)
                else {

                    //if it's the same value, do not change anything
                    if (this._data[a] === vals) {
                        continue;
                    } else {
                        this._data[a] = vals;
                        //different events whether it's first time or not
                        var evt_name = (this._set) ? "change" : "init";
                        events.push(evt_name + ":" + a);
                        promise = true;
                    }
                }
                promises.push(promise);
            }

            //not ready at this point
            this._ready = false;

            //after all is done
            var _this = this;
            var size = promises.length;
            $.when.apply(null, promises).then(function() {

                //bind magic getters and setters
                _this._bindSettersGetters();
                //if we don't block validation, validate
                if (!block_validation) {

                    //attempt to validate
                    var val_promise = false;
                    if (_this.validate) {
                        val_promise = _this.validate();
                    }

                    //if validation is not a promise, make it a confirmed one
                    if (!val_promise || !val_promise.always) {
                        val_promise = $.when.apply(null, [this]);
                    }

                    //confirm that the model has been validated
                    val_promise.always(function() {
                        //trigger change if not silent
                        if (!_this._set) {
                            _this._set = true;
                            events.push("set");
                        }
                        if (!silent) {
                            _.defer(function() {
                                _this.triggerAll(events, _this.getObject());
                            });
                        }
                        defer.resolve();
                    });
                }
            });

            return defer;
        },

        /**
         * Gets the type of this model
         * @returns {String} type of the model
         */
        getType: function() {
            return this._type;
        },

        /**
         * Loads a submodel, when necessaary
         * @param attr Name of submodel
         * @param val Initial values
         * @returns defer defer that will be resolved when submodel is ready
         */
        _initSubmodel: function(attr, val) {
            //naming convention: underscore -> time, time_2, time_overlay
            var name = attr.split("_")[0],
                modl = 'models/' + name,
                defer = $.Deferred(),
                _this = this;

            //special model
            //hotfix: global _vzb_available_plugins has all variable modules
            if (_vzb_available_plugins.indexOf(modl) !== -1) {
                require([modl], function(model) {
                    _this._instantiateSubmodel(attr, val, model, defer);
                });
            } else {
                //always implement base model if not found
                var model = _getBaseModelClass();
                this._instantiateSubmodel(attr, val, model, defer);
            }
            return defer;
        },

        /**
         * Instantiates a new model as a submodel of this one
         * @param name Name of submodel
         * @param values Initial values
         * @param model Model class
         * @param defer defer to be resolved when model is ready
         */
        _instantiateSubmodel: function(name, values, model, defer) {

            var _this = this;

            this._data[name] = new model(values, this, {
                //the submodel has been set (only once)
                'set': function(evt, vals) {
                    //solve the defer after it's been set
                    defer.resolve();
                },
                //the submodel has initialized (only once)
                'init': function(evt, vals) {
                    evt = evt.replace('init', 'init:' + name);
                    _this.triggerAll(evt, _this.getObject());
                },
                //the submodel has changed (multiple times)
                'change': function(evt, vals) {
                    evt = evt.replace('change', 'change:' + name);
                    _this.triggerAll(evt, _this.getObject());
                },
                //loading has started in this submodel (multiple times)
                'load_start': function(evt, vals) {
                    evt = evt.replace('load_start', 'load_start:' + name);
                    _this.triggerAll(evt, _this.getObject());
                },
                //loading has failed in this submodel (multiple times)
                'load_error': function(evt, vals) {
                    evt = evt.replace('load_error', 'load_error:' + name);
                    _this.triggerAll(evt, vals);
                },
                //loading has ended in this submodel (multiple times)
                'ready': function(evt, vals) {
                    //trigger only for submodel
                    evt = evt.replace('ready', 'ready:' + name);
                    _this.trigger(evt, vals);

                    //if this model is not loading trigger for this model
                    if (!_this.isLoading()) {
                        _this.triggerOnce('ready', vals);
                    }
                }
            });
        },

        /**
         * Generate getter for a certain attribute
         * @param prop name of attribute
         * @returns {Function} getter function
         */
        _funcGetter: function(prop) {
            var _this = this;
            return function() {
                return _this.get(prop);
            };
        },

        /**
         * Generate setter for a certain attribute
         * @param prop name of attribute
         * @returns {Function} setter function
         */
        _funcSetter: function(prop) {
            var _this = this;
            return function(val) {
                return _this.set(prop, val);
            };
        },

        /**
         * Binds all attributes in _data to magic setters and getters
         */
        _bindSettersGetters: function() {
            for (var prop in this._data) {
                this.__defineGetter__(prop, this._funcGetter(prop));
                this.__defineSetter__(prop, this._funcSetter(prop));
            }
        },

        /**
         * Gets all submodels of the current model
         */
        getSubmodels: function() {
            return _.filter(this._data, function(s) {
                return !_.isUndefined(s._id);
            });
        },


        /**
         * Resets this model
         * @param values new values
         * @param {Boolean} prevent events from being fired
         * @returns defer defer that will be resolved when reset is done
         */
        reset: function(values, silent) {
            this.clear();
            return this.set(values, silent);
        },

        /**
         * Clears this model, submodels, data and events
         */
        clear: function() {
            var submodels = this.getSubmodels();
            for (var i in submodels) {
                submodels[i].clear();
            }
            this._ready = false;
            this._events.unbindAll();
            this._intervals.clearAllIntervals();
            this._data = {};
        },

        /**
         * Gets the current model and submodel values as a JS object
         * @returns {Object} All model as JS object
         */
        getObject: function() {
            var obj = {};
            for (var i in this._data) {
                //if it's a submodel
                if (this._data[i] && typeof this._data[i].getObject === 'function') {
                    obj[i] = this._data[i].getObject();
                } else {
                    obj[i] = this._data[i];
                }
            }
            return obj;
        },

        /**
         * gets intervals from this model or parent
         * @returns {Object} Intervals object
         */
        getIntervals: function() {
            if (this.intervals) {
                return this.intervals;
            } else if (this._parent && this._parent.getIntervals) {
                return this._parent.getIntervals();
            } else {
                return new Intervals();
            }
        },


        /* ==========================
         * Model loading method
         * ==========================
         */

        /**
         * checks whether this model is loading anything
         * @param {String} optional process id (to check only one)
         * @returns {Boolean} is it loading?
         */
        isLoading: function(p_id) {
            if (p_id) {
                return (this._loading.indexOf(p_id) !== -1);
            }
            //if loading something
            else if (this._loading.length > 0) {
                return true;
            }
            //if not loading anything, check submodels
            else {
                return !_.every(this.getSubmodels(), function(sm) {
                    return !sm.isLoading();
                });
            }
        },

        /**
         * specifies that the model is loading data
         * @param {String} id of the loading process
         */
        setLoading: function(p_id) {
            //if this is the first time we're loading anything
            if (!this.isLoading()) {
                this.trigger("load_start");
            }
            //add id to the list of processes that are loading
            this._loading.push(p_id);
        },

        /**
         * specifies that the model is done with loading data
         * @param {String} id of the loading process
         */
        setLoadingDone: function(p_id) {
            //remove he process from the list of things that are loading
            this._loading = _.without(this._loading, p_id);
            //set ready if it is ready
            this.setReady();
        },

        /**
         * Sets the model as ready or not depending on its loading status
         */
        setReady: function() {
            if (this._ready = !this.isLoading()) {
                this.trigger("ready");
            }
        },

        /**
         * loads data (if hook)
         * Hooks loads data, models ask children to load data
         * Basically, this method:
         * loads is theres something to be loaded:
         * does not load if there's nothing to be loaded
         * @returns defer
         */
        load: function() {
            //we dont need to load if it's a hook
            var _this = this,
                promises = [],
                submodels = this.getSubmodels(),
                data_hook = this.getHook("data"),
                language_hook = this.getHook("language"),
                defer = $.Deferred(),
                query = this.getQuery();


            //load hook
            //if its not a hook, the promise will not be created
            if (this.isHook() && data_hook && query.length) {

                //get reader omfp
                var promise = $.Deferred(),
                    reader = data_hook.getObject(),
                    lang = "en";

                //get current language
                if (language_hook) {
                    lang = language_hook.id || "en";
                }

                var evts = {
                    'load_start': function() {
                        _this.setLoading("_hook_data");
                    },
                    'load_end': function() {
                        _this.setLoadingDone("_hook_data");
                    }
                };

                console.timeStamp("Vizabi Model: Loading Data: " + _this._id);

                this._dataManager.load(query, lang, reader, evts)
                    .done(function(data) {
                        if (data === 'error') {
                            _this.trigger("load_error", query);
                            promise.resolve();
                        } else {
                            _this._items = _.flatten(data);

                            //TODO this is a temporary solution that does preprocessing of data
                            // data should have time as Dates and be sorted by time
                            // put me in the proper place please!
                            _this._items = _this._items
                                .map(function(d) {
                                    d.time = new Date(d.time);
                                    d.time.setHours(0);
                                    return d;
                                })
                                .sort(function(a, b) {
                                    return a.time - b.time
                                });

                            console.timeStamp("Vizabi Model: Data loaded: " + _this._id);

                            promise.resolve();
                        }
                    });

                promises.push(promise);

            }

            //load submodels as well
            for (var i in submodels) {
                promises.push(submodels[i].load());
            }

            //when all promises/loading have been done successfully
            //we will consider this done
            $.when.apply(null, promises).then(function() {

                //but first, we need to validate
                var val_promise = false;
                if (_this.validate) {<commit_msg>added to model.js: try to restore "geo" from "geo.name" if it's missing (ebola data has that problem)<commit_after>define([
    'jquery',
    'd3',
    'lodash',
    'base/utils',
    'base/class',
    'base/intervals',
    'base/events',
    'base/data',
], function($, d3, _, utils, Class, Intervals, Events, DataManager) {

    var model = Class.extend({

        /**
         * Initializes the model.
         * @param {Object} values The initial values of this model
         * @param {Object} parent reference to parent
         * @param {Object} bind Initial events to bind
         */
        init: function(values, parent, bind) {

            this._type = this._type || "model"; //type of this model
            this._id = _.uniqueId("m"); //model unique id
            this._data = {}; //holds attributes of this model
            this._parent = parent; //parent model
            this._set = false; //is this model set?
            this._ready = false; //is this model ready?
            this._debugEvents = this._debugEvents || false;

            //intervals should be the same from tool
            this._intervals = this.getIntervals();
            //each model has its own event handling
            this._events = new Events();

            //will the model be hooked to data?
            this._hooks = {};
            this._loading = []; //array of processes that are loading
            this._items = []; //holds hook items for this hook

            //bind initial events
            if (bind) {
                this.on(bind);
            }

            //initial values
            if (values) {
                this.set(values);
            }
        },

        /* ==========================
         * Getters and Setters
         * ==========================
         */

        /**
         * Gets an attribute from this model or all fields.
         * @param attr Optional attribute
         * @returns attr value or all values if attr is undefined
         */
        get: function(attr) {
            if (!attr) {
                return this._data;
            }
            return this._data[attr];
        },

        /**
         * Sets an attribute or multiple for this model
         * @param attr property name
         * @param val property value (object or value)
         * @param {Boolean} silent Prevents events from being fired
         * @param {Boolean} block_validation prevents model validation
         * @returns defer defer that will be resolved when set is done
         */
        set: function(attr, val, silent, block_validation) {

            var defer = $.Deferred(),
                promises = [],
                events = [];

            //expect object as default
            if (!_.isPlainObject(attr)) {
                var obj = {};
                obj[attr] = val;
                return this.set(obj, silent, block_validation);
            }

            //correct format
            block_validation = silent;
            silent = val;
            for (var a in attr) {

                var vals = attr[a],
                    promise;
                //if it's an object, set or create submodel
                if (_.isPlainObject(vals)) {
                    if (this._data[a] && utils.isModel(this._data[a])) {
                        promise = this._data[a].set(vals, silent, block_validation);
                    }
                    //submodel doesnt exist, create it
                    else {
                        promise = this._initSubmodel(a, vals);
                    }
                }
                //otherwise, just set value :)
                else {

                    //if it's the same value, do not change anything
                    if (this._data[a] === vals) {
                        continue;
                    } else {
                        this._data[a] = vals;
                        //different events whether it's first time or not
                        var evt_name = (this._set) ? "change" : "init";
                        events.push(evt_name + ":" + a);
                        promise = true;
                    }
                }
                promises.push(promise);
            }

            //not ready at this point
            this._ready = false;

            //after all is done
            var _this = this;
            var size = promises.length;
            $.when.apply(null, promises).then(function() {

                //bind magic getters and setters
                _this._bindSettersGetters();
                //if we don't block validation, validate
                if (!block_validation) {

                    //attempt to validate
                    var val_promise = false;
                    if (_this.validate) {
                        val_promise = _this.validate();
                    }

                    //if validation is not a promise, make it a confirmed one
                    if (!val_promise || !val_promise.always) {
                        val_promise = $.when.apply(null, [this]);
                    }

                    //confirm that the model has been validated
                    val_promise.always(function() {
                        //trigger change if not silent
                        if (!_this._set) {
                            _this._set = true;
                            events.push("set");
                        }
                        if (!silent) {
                            _.defer(function() {
                                _this.triggerAll(events, _this.getObject());
                            });
                        }
                        defer.resolve();
                    });
                }
            });

            return defer;
        },

        /**
         * Gets the type of this model
         * @returns {String} type of the model
         */
        getType: function() {
            return this._type;
        },

        /**
         * Loads a submodel, when necessaary
         * @param attr Name of submodel
         * @param val Initial values
         * @returns defer defer that will be resolved when submodel is ready
         */
        _initSubmodel: function(attr, val) {
            //naming convention: underscore -> time, time_2, time_overlay
            var name = attr.split("_")[0],
                modl = 'models/' + name,
                defer = $.Deferred(),
                _this = this;

            //special model
            //hotfix: global _vzb_available_plugins has all variable modules
            if (_vzb_available_plugins.indexOf(modl) !== -1) {
                require([modl], function(model) {
                    _this._instantiateSubmodel(attr, val, model, defer);
                });
            } else {
                //always implement base model if not found
                var model = _getBaseModelClass();
                this._instantiateSubmodel(attr, val, model, defer);
            }
            return defer;
        },

        /**
         * Instantiates a new model as a submodel of this one
         * @param name Name of submodel
         * @param values Initial values
         * @param model Model class
         * @param defer defer to be resolved when model is ready
         */
        _instantiateSubmodel: function(name, values, model, defer) {

            var _this = this;

            this._data[name] = new model(values, this, {
                //the submodel has been set (only once)
                'set': function(evt, vals) {
                    //solve the defer after it's been set
                    defer.resolve();
                },
                //the submodel has initialized (only once)
                'init': function(evt, vals) {
                    evt = evt.replace('init', 'init:' + name);
                    _this.triggerAll(evt, _this.getObject());
                },
                //the submodel has changed (multiple times)
                'change': function(evt, vals) {
                    evt = evt.replace('change', 'change:' + name);
                    _this.triggerAll(evt, _this.getObject());
                },
                //loading has started in this submodel (multiple times)
                'load_start': function(evt, vals) {
                    evt = evt.replace('load_start', 'load_start:' + name);
                    _this.triggerAll(evt, _this.getObject());
                },
                //loading has failed in this submodel (multiple times)
                'load_error': function(evt, vals) {
                    evt = evt.replace('load_error', 'load_error:' + name);
                    _this.triggerAll(evt, vals);
                },
                //loading has ended in this submodel (multiple times)
                'ready': function(evt, vals) {
                    //trigger only for submodel
                    evt = evt.replace('ready', 'ready:' + name);
                    _this.trigger(evt, vals);

                    //if this model is not loading trigger for this model
                    if (!_this.isLoading()) {
                        _this.triggerOnce('ready', vals);
                    }
                }
            });
        },

        /**
         * Generate getter for a certain attribute
         * @param prop name of attribute
         * @returns {Function} getter function
         */
        _funcGetter: function(prop) {
            var _this = this;
            return function() {
                return _this.get(prop);
            };
        },

        /**
         * Generate setter for a certain attribute
         * @param prop name of attribute
         * @returns {Function} setter function
         */
        _funcSetter: function(prop) {
            var _this = this;
            return function(val) {
                return _this.set(prop, val);
            };
        },

        /**
         * Binds all attributes in _data to magic setters and getters
         */
        _bindSettersGetters: function() {
            for (var prop in this._data) {
                this.__defineGetter__(prop, this._funcGetter(prop));
                this.__defineSetter__(prop, this._funcSetter(prop));
            }
        },

        /**
         * Gets all submodels of the current model
         */
        getSubmodels: function() {
            return _.filter(this._data, function(s) {
                return !_.isUndefined(s._id);
            });
        },


        /**
         * Resets this model
         * @param values new values
         * @param {Boolean} prevent events from being fired
         * @returns defer defer that will be resolved when reset is done
         */
        reset: function(values, silent) {
            this.clear();
            return this.set(values, silent);
        },

        /**
         * Clears this model, submodels, data and events
         */
        clear: function() {
            var submodels = this.getSubmodels();
            for (var i in submodels) {
                submodels[i].clear();
            }
            this._ready = false;
            this._events.unbindAll();
            this._intervals.clearAllIntervals();
            this._data = {};
        },

        /**
         * Gets the current model and submodel values as a JS object
         * @returns {Object} All model as JS object
         */
        getObject: function() {
            var obj = {};
            for (var i in this._data) {
                //if it's a submodel
                if (this._data[i] && typeof this._data[i].getObject === 'function') {
                    obj[i] = this._data[i].getObject();
                } else {
                    obj[i] = this._data[i];
                }
            }
            return obj;
        },

        /**
         * gets intervals from this model or parent
         * @returns {Object} Intervals object
         */
        getIntervals: function() {
            if (this.intervals) {
                return this.intervals;
            } else if (this._parent && this._parent.getIntervals) {
                return this._parent.getIntervals();
            } else {
                return new Intervals();
            }
        },


        /* ==========================
         * Model loading method
         * ==========================
         */

        /**
         * checks whether this model is loading anything
         * @param {String} optional process id (to check only one)
         * @returns {Boolean} is it loading?
         */
        isLoading: function(p_id) {
            if (p_id) {
                return (this._loading.indexOf(p_id) !== -1);
            }
            //if loading something
            else if (this._loading.length > 0) {
                return true;
            }
            //if not loading anything, check submodels
            else {
                return !_.every(this.getSubmodels(), function(sm) {
                    return !sm.isLoading();
                });
            }
        },

        /**
         * specifies that the model is loading data
         * @param {String} id of the loading process
         */
        setLoading: function(p_id) {
            //if this is the first time we're loading anything
            if (!this.isLoading()) {
                this.trigger("load_start");
            }
            //add id to the list of processes that are loading
            this._loading.push(p_id);
        },

        /**
         * specifies that the model is done with loading data
         * @param {String} id of the loading process
         */
        setLoadingDone: function(p_id) {
            //remove he process from the list of things that are loading
            this._loading = _.without(this._loading, p_id);
            //set ready if it is ready
            this.setReady();
        },

        /**
         * Sets the model as ready or not depending on its loading status
         */
        setReady: function() {
            if (this._ready = !this.isLoading()) {
                this.trigger("ready");
            }
        },

        /**
         * loads data (if hook)
         * Hooks loads data, models ask children to load data
         * Basically, this method:
         * loads is theres something to be loaded:
         * does not load if there's nothing to be loaded
         * @returns defer
         */
        load: function() {
            //we dont need to load if it's a hook
            var _this = this,
                promises = [],
                submodels = this.getSubmodels(),
                data_hook = this.getHook("data"),
                language_hook = this.getHook("language"),
                defer = $.Deferred(),
                query = this.getQuery();


            //load hook
            //if its not a hook, the promise will not be created
            if (this.isHook() && data_hook && query.length) {

                //get reader omfp
                var promise = $.Deferred(),
                    reader = data_hook.getObject(),
                    lang = "en";

                //get current language
                if (language_hook) {
                    lang = language_hook.id || "en";
                }

                var evts = {
                    'load_start': function() {
                        _this.setLoading("_hook_data");
                    },
                    'load_end': function() {
                        _this.setLoadingDone("_hook_data");
                    }
                };

                console.timeStamp("Vizabi Model: Loading Data: " + _this._id);

                this._dataManager.load(query, lang, reader, evts)
                    .done(function(data) {
                        if (data === 'error') {
                            _this.trigger("load_error", query);
                            promise.resolve();
                        } else {
                            _this._items = _.flatten(data);

                            //TODO this is a temporary solution that does preprocessing of data
                            // data should have time as Dates and be sorted by time
                            // put me in the proper place please!
                            _this._items = _this._items
                                // try to restore "geo" from "geo.name" if it's missing (ebola data has that problem)
                                .map(function(d){
                                    if(d["geo"] == null) d["geo"] = d["geo.name"];
                                    return d
                                })
                                // convert time to Date()
                                .map(function(d) {
                                    d.time = new Date(d.time);
                                    d.time.setHours(0);
                                    return d;
                                })
                                // sort records by time
                                .sort(function(a, b) {
                                    return a.time - b.time
                                });

                            console.timeStamp("Vizabi Model: Data loaded: " + _this._id);

                            promise.resolve();
                        }
                    });

                promises.push(promise);

            }

            //load submodels as well
            for (var i in submodels) {
                promises.push(submodels[i].load());
            }

            //when all promises/loading have been done successfully
            //we will consider this done
            $.when.apply(null, promises).then(function() {

                //but first, we need to validate
                var val_promise = false;
                if (_this.validate) {
{'fraction_non_alphanumeric': 0.10211699005613015, 'fraction_numerical': 0.0004558794198934382, 'mean_word_length': 1.1241905223022453, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7345733', 'n_tokens_mistral': 8628, 'n_tokens_neox': 8131, 'n_words': 2495}