<commit_before>'use strict';

treeherder.factory('ThResultSetModel',
                   ['$rootScope', 'thResultSets', 'thSocket',
                    'ThJobModel', 'thEvents', 'thAggregateIds', 'ThLog',
                   function($rootScope, thResultSets, thSocket,
                            ThJobModel, thEvents, thAggregateIds, ThLog) {

    var $log = new ThLog("ThResultSetModel");

   /******
    * Handle updating the resultset datamodel based on a queue of jobs
    * and resultsets.
    *
    * manages:
    *     resultset array
    *     socket messages
    *     resultset queue
    *     resultset map
    *     job queue
    *     job map
    */

    // the primary data model
    var repositories = {};

    var updateQueueInterval = 10000;

    var addRepository = function(repoName){
        //Initialize a new repository in the repositories structure

        if(_.isEmpty(repositories[repoName])){
            repositories[repoName] = {

                name:repoName,

                //This is set to the id of the last resultset loaded
                //and used as the offset in paging
                rsOffsetId:0,

                // queues of updates that have come over socket.io.
                // Processed at intervals
                jobUpdateQueue:[],
                rsUpdateQueue:[],

                lastJobElSelected:{},
                lastJobObjSelected:{},

                // maps to help finding objects to update/add
                rsMap:{},
                jobMap:{},
                jobMapOldestId:null,
                rsMapOldestTimestamp:null,
                resultSets:[],

                // this is "watchable" by the controller now to update its scope.
                loadingStatus: {
                    appending: false,
                    prepending: false
                }
            };

            // Add a connect listener
            thSocket.on('connect',function() {
                // subscribe to all the events for this repo
                thSocket.emit('subscribe', repoName);
                });

            //Set up job update queue
            setInterval(
                _.bind(processUpdateQueues, $rootScope, repoName),
                updateQueueInterval
                );

            //Set up the socket listener
            thSocket.on(
                "job",
                _.bind(processSocketData, $rootScope, repoName)
                );
        }
    };

    var processSocketData = function(repoName, data){
        /******
         * Process a new ``job`` event notification.
         * Check the job's ``result_set_id``.  If the id belongs to a resulset
         * we already have in memory, add it to the ``jobUpdateQueue``.  If
         * not, then check if the ``resultset_id`` is newer or older than the
         * oldest rs_id we have in memory.  If it's newer, then add the
         * ``resultset_id`` to ``rsUpdateQueue``.
         *
         * So basically, if we see a job belonging to a newer resultset we
         * don't yet have loaded, then add it to the list of resultsets to
         * fetch.  Fetching a resultset also gets all its jobs, so we don't
         * need to add it to the ``jobUpdateQueue``.
         */
        if (data.branch === repoName) {
            if (data.resultset.push_timestamp >= repositories[repoName].rsMapOldestTimestamp) {
                // we want to load this job, one way or another
                if (repositories[repoName].rsMap[data.resultset.id]) {
                    // we already have this resultset loaded, so queue the job
                    repositories[repoName].jobUpdateQueue.push(data.id);
                } else {
                    // we haven't loaded this resultset yet, so queue it
                    if (repositories[repoName].rsUpdateQueue.indexOf(data.resultset.id) < 0) {
                        repositories[repoName].rsUpdateQueue.push(data.resultset.id);
                    }
                }
            }
        }
    };

    var getJobMapKey = function(job) {
        //Build string key for jobMap entires
        return 'key' + job.id;
    };

    var getSelectedJob = function(repoName){
        return { el:repositories[repoName].lastJobElSelected,
                 job:repositories[repoName].lastJobObjSelected };
    };

    var setSelectedJob = function(
        repoName, lastJobElSelected, lastJobObjSelected){

        repositories[repoName].lastJobElSelected = lastJobElSelected;
        repositories[repoName].lastJobObjSelected = lastJobObjSelected;
    };

    var getPlatformKey = function(name, option){
        var key = name;
        if(option !== undefined){
            key += option;
        }
        return key;
    };

    /******
     * Build the Job and Resultset object mappings to make it faster and
     * easier to find and update jobs and resultsets
     *
     * @param data The array of resultsets to map.
     */
    var mapResultSets = function(repoName, data) {

        for (var rs_i = 0; rs_i < data.length; rs_i++) {
            var rs_obj = data[rs_i];
            // make a watch-able revisions array
            rs_obj.revisions = [];

            var rsMapElement = {
                rs_obj: rs_obj,
                platforms: {}
            };
            repositories[repoName].rsMap[rs_obj.id] = rsMapElement;

            // keep track of the oldest push_timestamp, so we don't auto-fetch resultsets
            // that are out of the range we care about.
            if ( !repositories[repoName].rsMapOldestTimestamp ||
                 (repositories[repoName].rsMapOldestTimestamp > rs_obj.push_timestamp)) {
                repositories[repoName].rsMapOldestTimestamp = rs_obj.push_timestamp;
            }

            //Keep track of the last resultset id for paging
            var resultsetId = parseInt(rs_obj.id, 10);
            if( (resultsetId < repositories[repoName].rsOffsetId) ||
                (repositories[repoName].rsOffsetId === 0) ){
                repositories[repoName].rsOffsetId = resultsetId;
            }

            // platforms
            for (var pl_i = 0; pl_i < rs_obj.platforms.length; pl_i++) {
                var pl_obj = rs_obj.platforms[pl_i];

                var plMapElement = {
                    pl_obj: pl_obj,
                    parent: repositories[repoName].rsMap[rs_obj.id],
                    groups: {}
                };
                var platformKey = getPlatformKey(pl_obj.name, pl_obj.option);
                repositories[repoName].rsMap[rs_obj.id].platforms[platformKey] = plMapElement;

                // groups
                for (var gp_i = 0; gp_i < pl_obj.groups.length; gp_i++) {
                    var gr_obj = pl_obj.groups[gp_i];

                    var grMapElement = {
                        grp_obj: gr_obj,
                        parent: plMapElement,
                        jobs: {}
                    };
                    plMapElement.groups[gr_obj.name] = grMapElement;

                    // jobs
                    for (var j_i = 0; j_i < gr_obj.jobs.length; j_i++) {
                        var job_obj = gr_obj.jobs[j_i];
                        var key = getJobMapKey(job_obj);

                        var jobMapElement = {
                            job_obj: job_obj,
                            parent: grMapElement
                        };
                        grMapElement.jobs[key] = jobMapElement;
                        repositories[repoName].jobMap[key] = jobMapElement;

                        // track oldest job id
                        if (!repositories[repoName].jobMapOldestId ||
                            (repositories[repoName].jobMapOldestId > job_obj.id)) {
                            repositories[repoName].jobMapOldestId = job_obj.id;
                        }
                    }
                }
            }
        }

        repositories[repoName].resultSets.sort(rsCompare);

        $log.debug("oldest job: ", repositories[repoName].jobMapOldestId);
        $log.debug("oldest result set: ", repositories[repoName].rsMapOldestTimestamp);
        $log.debug("done mapping:", repositories[repoName].rsMap);
    };

    /**
     * Sort the resultsets in place after updating the array
     */
    var rsCompare = function(a, b) {
        if (a.push_timestamp > b.push_timestamp) {
          return -1;
        }
        if (a.push_timestamp < b.push_timestamp) {
          return 1;
        }
        return 0;
    };

    /******
     * Ensure that the platform for ``newJob`` exists.  Create it if
     * necessary.  Add to the datamodel AND the map
     * @param newJob
     * @returns plMapElement
     */
    var getOrCreatePlatform = function(repoName, newJob) {
        var rsMapElement = repositories[repoName].rsMap[newJob.result_set_id];
        var platformKey = getPlatformKey(newJob.platform, newJob.option);
        var plMapElement = rsMapElement.platforms[platformKey];
        if (!plMapElement) {
            // this platform wasn't in the resultset, so add it.
            $log.debug("adding new platform");

            var pl_obj = {
                name: newJob.platform,
                option: newJob.platform_option,
                groups: []
            };

            // add the new platform to the datamodel and resort
            rsMapElement.rs_obj.platforms.push(pl_obj);

            // add the new platform to the resultset map
            rsMapElement.platforms[platformKey] = {
                pl_obj: pl_obj,
                parent: rsMapElement,
                groups: {}
            };
            plMapElement = rsMapElement.platforms[platformKey];
        }
        return plMapElement;
    };

    /******
     * Ensure that the group and platform for ``newJob`` exist.
     * Create it if necessary.  Add to the datamodel AND the map
     * @param newJob
     * @returns grpMapElement
     */
    var getOrCreateGroup = function(repoName, newJob) {
        var plMapElement = getOrCreatePlatform(repoName, newJob);
        var grMapElement = plMapElement.groups[newJob.job_group_name];
        if (!grMapElement) {
            $log.debug("adding new group");
            var grp_obj = {
                symbol: newJob.job_group_symbol,
                name: newJob.job_group_name,
                jobs: []
            };

            // add the new group to the datamodel
            plMapElement.pl_obj.groups.push(grp_obj);

            // add the new group to the platform map
            plMapElement.groups[grp_obj.name] = {
                grp_obj: grp_obj,
                parent: plMapElement,
                jobs: {}
            };

            grMapElement = plMapElement.groups[newJob.job_group_name];
        }
        return grMapElement;
    };

    /******
     * Socket.io handling for new and updated jobs and resultsets
     *
     * First we fetch new resultsets.  These may contain some of the jobs
     * in the jobUpdateQueue.  So after getting the resultsets, we check
     * if any of the jobs in the queue were fetched and remove them from
     * the job queue before fetching them.
     *
     * Then we fetch the remaining jobs in a batch and add them to their
     * appropriate resultset.
     */
    var processUpdateQueues = function(repoName) {
        $log.debug("Processing update queue.  jobs: " +
            repositories[repoName].jobUpdateQueue.length +
            ", resultsets: " +
            repositories[repoName].rsUpdateQueue.length);
        // clear the ``jobUpdateQueue`` so we won't miss items that get
        // added while in the process of fetching the current queue items.
        var rsFetchList = repositories[repoName].rsUpdateQueue;
        repositories[repoName].rsUpdateQueue = [];


        var jobFetchList = repositories[repoName].jobUpdateQueue;
        repositories[repoName].jobUpdateQueue = [];

        if (rsFetchList.length > 0) {
            // fetch these resultsets in a batch and put them into the model
            $log.debug("processing the rsFetchList");
            fetchNewResultSets(repoName, rsFetchList);
        }

        if (jobFetchList.length > 0) {
            $log.debug("processing jobFetchList", jobFetchList);

            // make an ajax call to get the job details
            fetchJobs(repoName, jobFetchList);
        }
    };
    /**
     * Fetch the job objects for the ids in ``jobFetchList`` and update them
     * in the data model.
     */
    var fetchJobs = function(repoName, jobFetchList) {
        ThJobModel.get_list({
            id__in: jobFetchList.join()
        }).then(
            _.bind(updateJobs, $rootScope, repoName),
            function(data) {
                $log.error("Error fetching jobs: " + data);
            });
    };
    var aggregateJobPlatform = function(repoName, job, platformData){

        var resultsetId, platformName, platformOption, platformAggregateId,
            platformKey, jobUpdated, resultsetAggregateId, revision,
            jobGroups;<commit_msg>fix fetchJobs to use GUID instead of ID<commit_after>'use strict';

treeherder.factory('ThResultSetModel',
                   ['$rootScope', 'thResultSets', 'thSocket',
                    'ThJobModel', 'thEvents', 'thAggregateIds', 'ThLog',
                   function($rootScope, thResultSets, thSocket,
                            ThJobModel, thEvents, thAggregateIds, ThLog) {

    var $log = new ThLog("ThResultSetModel");

   /******
    * Handle updating the resultset datamodel based on a queue of jobs
    * and resultsets.
    *
    * manages:
    *     resultset array
    *     socket messages
    *     resultset queue
    *     resultset map
    *     job queue
    *     job map
    */

    // the primary data model
    var repositories = {};

    var updateQueueInterval = 10000;

    var addRepository = function(repoName){
        //Initialize a new repository in the repositories structure

        if(_.isEmpty(repositories[repoName])){
            repositories[repoName] = {

                name:repoName,

                //This is set to the id of the last resultset loaded
                //and used as the offset in paging
                rsOffsetId:0,

                // queues of updates that have come over socket.io.
                // Processed at intervals
                jobUpdateQueue:[],
                rsUpdateQueue:[],

                lastJobElSelected:{},
                lastJobObjSelected:{},

                // maps to help finding objects to update/add
                rsMap:{},
                jobMap:{},
                jobMapOldestId:null,
                rsMapOldestTimestamp:null,
                resultSets:[],

                // this is "watchable" by the controller now to update its scope.
                loadingStatus: {
                    appending: false,
                    prepending: false
                }
            };

            // Add a connect listener
            thSocket.on('connect',function() {
                // subscribe to all the events for this repo
                thSocket.emit('subscribe', repoName);
                });

            //Set up job update queue
            setInterval(
                _.bind(processUpdateQueues, $rootScope, repoName),
                updateQueueInterval
                );

            //Set up the socket listener
            thSocket.on(
                "job",
                _.bind(processSocketData, $rootScope, repoName)
                );
        }
    };

    var processSocketData = function(repoName, data){
        /******
         * Process a new ``job`` event notification.
         * Check the job's ``result_set_id``.  If the id belongs to a resulset
         * we already have in memory, add it to the ``jobUpdateQueue``.  If
         * not, then check if the ``resultset_id`` is newer or older than the
         * oldest rs_id we have in memory.  If it's newer, then add the
         * ``resultset_id`` to ``rsUpdateQueue``.
         *
         * So basically, if we see a job belonging to a newer resultset we
         * don't yet have loaded, then add it to the list of resultsets to
         * fetch.  Fetching a resultset also gets all its jobs, so we don't
         * need to add it to the ``jobUpdateQueue``.
         */
        if (data.branch === repoName) {
            if (data.resultset.push_timestamp >= repositories[repoName].rsMapOldestTimestamp) {
                // we want to load this job, one way or another
                if (repositories[repoName].rsMap[data.resultset.id]) {
                    // we already have this resultset loaded, so queue the job
                    repositories[repoName].jobUpdateQueue.push(data.id);
                } else {
                    // we haven't loaded this resultset yet, so queue it
                    if (repositories[repoName].rsUpdateQueue.indexOf(data.resultset.id) < 0) {
                        repositories[repoName].rsUpdateQueue.push(data.resultset.id);
                    }
                }
            }
        }
    };

    var getJobMapKey = function(job) {
        //Build string key for jobMap entires
        return 'key' + job.id;
    };

    var getSelectedJob = function(repoName){
        return { el:repositories[repoName].lastJobElSelected,
                 job:repositories[repoName].lastJobObjSelected };
    };

    var setSelectedJob = function(
        repoName, lastJobElSelected, lastJobObjSelected){

        repositories[repoName].lastJobElSelected = lastJobElSelected;
        repositories[repoName].lastJobObjSelected = lastJobObjSelected;
    };

    var getPlatformKey = function(name, option){
        var key = name;
        if(option !== undefined){
            key += option;
        }
        return key;
    };

    /******
     * Build the Job and Resultset object mappings to make it faster and
     * easier to find and update jobs and resultsets
     *
     * @param data The array of resultsets to map.
     */
    var mapResultSets = function(repoName, data) {

        for (var rs_i = 0; rs_i < data.length; rs_i++) {
            var rs_obj = data[rs_i];
            // make a watch-able revisions array
            rs_obj.revisions = [];

            var rsMapElement = {
                rs_obj: rs_obj,
                platforms: {}
            };
            repositories[repoName].rsMap[rs_obj.id] = rsMapElement;

            // keep track of the oldest push_timestamp, so we don't auto-fetch resultsets
            // that are out of the range we care about.
            if ( !repositories[repoName].rsMapOldestTimestamp ||
                 (repositories[repoName].rsMapOldestTimestamp > rs_obj.push_timestamp)) {
                repositories[repoName].rsMapOldestTimestamp = rs_obj.push_timestamp;
            }

            //Keep track of the last resultset id for paging
            var resultsetId = parseInt(rs_obj.id, 10);
            if( (resultsetId < repositories[repoName].rsOffsetId) ||
                (repositories[repoName].rsOffsetId === 0) ){
                repositories[repoName].rsOffsetId = resultsetId;
            }

            // platforms
            for (var pl_i = 0; pl_i < rs_obj.platforms.length; pl_i++) {
                var pl_obj = rs_obj.platforms[pl_i];

                var plMapElement = {
                    pl_obj: pl_obj,
                    parent: repositories[repoName].rsMap[rs_obj.id],
                    groups: {}
                };
                var platformKey = getPlatformKey(pl_obj.name, pl_obj.option);
                repositories[repoName].rsMap[rs_obj.id].platforms[platformKey] = plMapElement;

                // groups
                for (var gp_i = 0; gp_i < pl_obj.groups.length; gp_i++) {
                    var gr_obj = pl_obj.groups[gp_i];

                    var grMapElement = {
                        grp_obj: gr_obj,
                        parent: plMapElement,
                        jobs: {}
                    };
                    plMapElement.groups[gr_obj.name] = grMapElement;

                    // jobs
                    for (var j_i = 0; j_i < gr_obj.jobs.length; j_i++) {
                        var job_obj = gr_obj.jobs[j_i];
                        var key = getJobMapKey(job_obj);

                        var jobMapElement = {
                            job_obj: job_obj,
                            parent: grMapElement
                        };
                        grMapElement.jobs[key] = jobMapElement;
                        repositories[repoName].jobMap[key] = jobMapElement;

                        // track oldest job id
                        if (!repositories[repoName].jobMapOldestId ||
                            (repositories[repoName].jobMapOldestId > job_obj.id)) {
                            repositories[repoName].jobMapOldestId = job_obj.id;
                        }
                    }
                }
            }
        }

        repositories[repoName].resultSets.sort(rsCompare);

        $log.debug("oldest job: ", repositories[repoName].jobMapOldestId);
        $log.debug("oldest result set: ", repositories[repoName].rsMapOldestTimestamp);
        $log.debug("done mapping:", repositories[repoName].rsMap);
    };

    /**
     * Sort the resultsets in place after updating the array
     */
    var rsCompare = function(a, b) {
        if (a.push_timestamp > b.push_timestamp) {
          return -1;
        }
        if (a.push_timestamp < b.push_timestamp) {
          return 1;
        }
        return 0;
    };

    /******
     * Ensure that the platform for ``newJob`` exists.  Create it if
     * necessary.  Add to the datamodel AND the map
     * @param newJob
     * @returns plMapElement
     */
    var getOrCreatePlatform = function(repoName, newJob) {
        var rsMapElement = repositories[repoName].rsMap[newJob.result_set_id];
        var platformKey = getPlatformKey(newJob.platform, newJob.option);
        var plMapElement = rsMapElement.platforms[platformKey];
        if (!plMapElement) {
            // this platform wasn't in the resultset, so add it.
            $log.debug("adding new platform");

            var pl_obj = {
                name: newJob.platform,
                option: newJob.platform_option,
                groups: []
            };

            // add the new platform to the datamodel and resort
            rsMapElement.rs_obj.platforms.push(pl_obj);

            // add the new platform to the resultset map
            rsMapElement.platforms[platformKey] = {
                pl_obj: pl_obj,
                parent: rsMapElement,
                groups: {}
            };
            plMapElement = rsMapElement.platforms[platformKey];
        }
        return plMapElement;
    };

    /******
     * Ensure that the group and platform for ``newJob`` exist.
     * Create it if necessary.  Add to the datamodel AND the map
     * @param newJob
     * @returns grpMapElement
     */
    var getOrCreateGroup = function(repoName, newJob) {
        var plMapElement = getOrCreatePlatform(repoName, newJob);
        var grMapElement = plMapElement.groups[newJob.job_group_name];
        if (!grMapElement) {
            $log.debug("adding new group");
            var grp_obj = {
                symbol: newJob.job_group_symbol,
                name: newJob.job_group_name,
                jobs: []
            };

            // add the new group to the datamodel
            plMapElement.pl_obj.groups.push(grp_obj);

            // add the new group to the platform map
            plMapElement.groups[grp_obj.name] = {
                grp_obj: grp_obj,
                parent: plMapElement,
                jobs: {}
            };

            grMapElement = plMapElement.groups[newJob.job_group_name];
        }
        return grMapElement;
    };

    /******
     * Socket.io handling for new and updated jobs and resultsets
     *
     * First we fetch new resultsets.  These may contain some of the jobs
     * in the jobUpdateQueue.  So after getting the resultsets, we check
     * if any of the jobs in the queue were fetched and remove them from
     * the job queue before fetching them.
     *
     * Then we fetch the remaining jobs in a batch and add them to their
     * appropriate resultset.
     */
    var processUpdateQueues = function(repoName) {
        $log.debug("Processing update queue.  jobs: " +
            repositories[repoName].jobUpdateQueue.length +
            ", resultsets: " +
            repositories[repoName].rsUpdateQueue.length);
        // clear the ``jobUpdateQueue`` so we won't miss items that get
        // added while in the process of fetching the current queue items.
        var rsFetchList = repositories[repoName].rsUpdateQueue;
        repositories[repoName].rsUpdateQueue = [];


        var jobFetchList = repositories[repoName].jobUpdateQueue;
        repositories[repoName].jobUpdateQueue = [];

        if (rsFetchList.length > 0) {
            // fetch these resultsets in a batch and put them into the model
            $log.debug("processing the rsFetchList");
            fetchNewResultSets(repoName, rsFetchList);
        }

        if (jobFetchList.length > 0) {
            $log.debug("processing jobFetchList", jobFetchList);

            // make an ajax call to get the job details
            fetchJobs(repoName, jobFetchList);
        }
    };
    /**
     * Fetch the job objects for the ids in ``jobFetchList`` and update them
     * in the data model.
     */
    var fetchJobs = function(repoName, jobFetchList) {
        ThJobModel.get_list({
            job_guid__in: jobFetchList.join()
        }).then(
            _.bind(updateJobs, $rootScope, repoName),
            function(data) {
                $log.error("Error fetching jobs: " + data);
            });
    };
    var aggregateJobPlatform = function(repoName, job, platformData){

        var resultsetId, platformName, platformOption, platformAggregateId,
            platformKey, jobUpdated, resultsetAggregateId, revision,
            jobGroups;
{'fraction_non_alphanumeric': 0.09053245358293693, 'fraction_numerical': 0.0014577259475218659, 'mean_word_length': 1.6311061768268067, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4523675', 'n_tokens_mistral': 7163, 'n_tokens_neox': 6717, 'n_words': 1842}