<commit_before>define(['logger', 'voc', 'underscore', 'data/Data' ], function(Logger, Voc, _, Data){
    var m = Object.create(Data);
    m.init = function(vie) {
        this.LOG.debug("initialize EntityData");
        this.vie = vie;
        this.vie.entities.on('add', this.filter, this);
        this.setIntegrityCheck(Voc.belongsToTimeline, Voc.TIMELINE, Voc.hasEntity);
        this.setIntegrityCheck(Voc.author, Voc.USER);
    };
    m.LOG = Logger.get('EntityData');
    /** 
     * Filters entities from added entities to vie.entities
     */
    m.filter= function(model, collection, options) {
        if( model.isof(Voc.ENTITY) || model.isof(Voc.FILE) ){
            this.checkIntegrity(model, options);
            if ( model.has(Voc.author) ) {
                this.initUser(model);
            } 
            model.on('change:'+this.vie.namespaces.uri(Voc.author), this.initUser, this); 
            model.on('change:'+this.vie.namespaces.uri(Voc.hasTag), this.changedTags, this);
            model.on('change:'+this.vie.namespaces.uri(Voc.importance), this.setImportance, this);
            this.loadViewCount(model);
            this.loadRecommTagsBasedOnUserEntityTagTime(model);
        }
    };
    m.initUser = function(model, value, options) {
        var user = model.get(Voc.author);
        if( user.isEntity ) {
            user.fetch();
        }
    };
    m.changedTags = function(model, set, options) {
        options = options || {};
        // Only change in case user_initiated flag is set
        if ( options.user_initiated !== true ) return;
        set = set || [];
        if( !_.isArray(set)) set = [set];
        var previous = model.previous(Voc.hasTag) || [];
        if( !_.isArray(previous)) previous = [previous];
        this.LOG.debug('previous', previous);  
        var that = this;
        var added = _.difference(set, previous);
        this.LOG.debug('added', added);
        this.vie.onUrisReady(
            model.getSubject(),
            function(modelUri) {
                _.each(added, function(tag){
                    that.vie.save({
                        'service' : 'tagAdd',
                        'data' : {
                            'entity' : modelUri,
                            'label' : tag,
                            'space' : 'privateSpace'
                        }
                    }).to('sss').execute().success(function(s){
                        that.LOG.debug('success addTag', s);
                        that.loadRecommTagsBasedOnUserEntityTagTime(model);
                    }).fail(function(f){
                        var tags = model.get(Voc.hasTag) || [];
                        if( !_.isArray(tags)) tags = [tags];
                        model.set(Voc.hasTag, _.without(tags, tag));
                        that.LOG.debug('options', options);
                        options.error();
                    });
                });
            }
        );

        var deleted = _.difference(previous, set);
        this.LOG.debug('deleted', deleted);
        this.vie.onUrisReady(
            model.getSubject(),
            function(modelUri) {
                _.each(deleted, function(tag){
                    that.vie.remove({
                        'service' : 'tagsRemove',
                        'data' : {
                            'entity' : modelUri,
                            'label' : tag
                        }
                    }).from('sss').execute().success(function(s){
                        that.loadRecommTagsBasedOnUserEntityTagTime(model);
                        that.LOG.debug('success removeTag', s);
                    });
                });
            }
        );
    };
    m.setLabel = function(model, label, options) {
        var that = this;
        options = options || {};
        // Only change if user_initiated flag is set to true
        if ( options.user_initiated !== true ) return;
        if ( model.previous(Voc.label) === label ) return;
        model.save(Voc.label, label, {
            'success' : function(result) {
                that.LOG.debug('success setLabel', result);
                // TODO check whether tag recomms can change due to label change
                that.loadRecommTagsBasedOnUserEntityTagTime(model);
                if(options.success) {
                    options.success(result);
                }
            },
            'error' : function(result) {
                that.LOG.debug('fail setLabel', result);
                if( options.error ) {
                    options.error(result);
                }
            }
        });
    };
    m.search = function(tags, callback) {
        var that = this;
        this.vie.load({
            'service' : 'search',
            'data' : {
                'keywordsToSearchFor' : tags,
                'typesToSearchOnlyFor' : ['entity', 'file', 'evernoteResource', 'evernoteNote', 'evernoteNotebook']
            }
        }).using('sss').execute().success(function(entities){
            that.LOG.debug('search entities', entities);
            entities = that.vie.entities.addOrUpdate(entities);
            _.each(entities, function(entity) {
                // XXX This is quite a bad check, in case the search results will
                // change in future. Need a better check to determine entity
                // being fully loaded.
                if ( !entity.has(Voc.author) ) {
                    entity.fetch();
                }
            });
            callback(entities);<commit_msg>Tuned the search a bit.<commit_after>define(['logger', 'voc', 'underscore', 'data/Data' ], function(Logger, Voc, _, Data){
    var m = Object.create(Data);
    m.init = function(vie) {
        this.LOG.debug("initialize EntityData");
        this.vie = vie;
        this.vie.entities.on('add', this.filter, this);
        this.setIntegrityCheck(Voc.belongsToTimeline, Voc.TIMELINE, Voc.hasEntity);
        this.setIntegrityCheck(Voc.author, Voc.USER);
    };
    m.LOG = Logger.get('EntityData');
    /** 
     * Filters entities from added entities to vie.entities
     */
    m.filter= function(model, collection, options) {
        if( model.isof(Voc.ENTITY) || model.isof(Voc.FILE) ){
            this.checkIntegrity(model, options);
            if ( model.has(Voc.author) ) {
                this.initUser(model);
            } 
            model.on('change:'+this.vie.namespaces.uri(Voc.author), this.initUser, this); 
            model.on('change:'+this.vie.namespaces.uri(Voc.hasTag), this.changedTags, this);
            model.on('change:'+this.vie.namespaces.uri(Voc.importance), this.setImportance, this);
            this.loadViewCount(model);
            this.loadRecommTagsBasedOnUserEntityTagTime(model);
        }
    };
    m.initUser = function(model, value, options) {
        var user = model.get(Voc.author);
        if( user.isEntity ) {
            user.fetch();
        }
    };
    m.changedTags = function(model, set, options) {
        options = options || {};
        // Only change in case user_initiated flag is set
        if ( options.user_initiated !== true ) return;
        set = set || [];
        if( !_.isArray(set)) set = [set];
        var previous = model.previous(Voc.hasTag) || [];
        if( !_.isArray(previous)) previous = [previous];
        this.LOG.debug('previous', previous);  
        var that = this;
        var added = _.difference(set, previous);
        this.LOG.debug('added', added);
        this.vie.onUrisReady(
            model.getSubject(),
            function(modelUri) {
                _.each(added, function(tag){
                    that.vie.save({
                        'service' : 'tagAdd',
                        'data' : {
                            'entity' : modelUri,
                            'label' : tag,
                            'space' : 'privateSpace'
                        }
                    }).to('sss').execute().success(function(s){
                        that.LOG.debug('success addTag', s);
                        that.loadRecommTagsBasedOnUserEntityTagTime(model);
                    }).fail(function(f){
                        var tags = model.get(Voc.hasTag) || [];
                        if( !_.isArray(tags)) tags = [tags];
                        model.set(Voc.hasTag, _.without(tags, tag));
                        that.LOG.debug('options', options);
                        options.error();
                    });
                });
            }
        );

        var deleted = _.difference(previous, set);
        this.LOG.debug('deleted', deleted);
        this.vie.onUrisReady(
            model.getSubject(),
            function(modelUri) {
                _.each(deleted, function(tag){
                    that.vie.remove({
                        'service' : 'tagsRemove',
                        'data' : {
                            'entity' : modelUri,
                            'label' : tag
                        }
                    }).from('sss').execute().success(function(s){
                        that.loadRecommTagsBasedOnUserEntityTagTime(model);
                        that.LOG.debug('success removeTag', s);
                    });
                });
            }
        );
    };
    m.setLabel = function(model, label, options) {
        var that = this;
        options = options || {};
        // Only change if user_initiated flag is set to true
        if ( options.user_initiated !== true ) return;
        if ( model.previous(Voc.label) === label ) return;
        model.save(Voc.label, label, {
            'success' : function(result) {
                that.LOG.debug('success setLabel', result);
                // TODO check whether tag recomms can change due to label change
                that.loadRecommTagsBasedOnUserEntityTagTime(model);
                if(options.success) {
                    options.success(result);
                }
            },
            'error' : function(result) {
                that.LOG.debug('fail setLabel', result);
                if( options.error ) {
                    options.error(result);
                }
            }
        });
    };
    m.search = function(tags, callback) {
        var that = this;
        this.vie.load({
            'service' : 'search',
            'data' : {
                'keywordsToSearchFor' : tags,
                'includeTags' : true,
                'includeLabel' : true,
                'typesToSearchOnlyFor' : ['entity', 'file', 'evernoteResource', 'evernoteNote', 'evernoteNotebook']
            }
        }).using('sss').execute().success(function(entities){
            that.LOG.debug('search entities', entities);
            entities = that.vie.entities.addOrUpdate(entities);
            _.each(entities, function(entity) {
                // XXX This is quite a bad check, in case the search results will
                // change in future. Need a better check to determine entity
                // being fully loaded.
                // XXX Need to determine if this is even needed any more
                // as the method might return the full entity representation now
                if ( !entity.has(Voc.author) ) {
                    entity.fetch();
                }
            });
            callback(entities);
{'fraction_non_alphanumeric': 0.12860290877038344, 'fraction_numerical': 0.0, 'mean_word_length': 1.4563758389261745, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '730393', 'n_tokens_mistral': 3058, 'n_tokens_neox': 2952, 'n_words': 582}