<commit_before>define(['d3', 'jquery', './listeners', './query/pathquery', './config', './statisticsutil', './sorting', '../pathfinder_ccle/ccle', './extradata'],
    function (d3, $, listeners, pathQuery, config, statisticsUtil, sorting, ccle, extraData) {

        var SortingStrategy = sorting.SortingStrategy;

        /**
         *
         * Data has been loaded for these nodes or data is currently loading.
         */
        var nodesWithData = {};

        var allDatasets = {};

        var stratification = {};

        var paths = [];

        var nodes = {};

        var nodeProperties = {};

        var pathStats = {};

        /**
         * Sort by stats for the whole dataset or a whole group.
         *
         * @param datasetId
         * @param stat
         * @param groupId
         * @constructor
         */
        function OverallStatsSortingStrategy(datasetId, stat, groupId) {
            SortingStrategy.call(this, SortingStrategy.prototype.STRATEGY_TYPES.ATTRIBUTE, datasetId + ": " + stat, "OVERALL_STATS");
            this.datasetId = datasetId;
            this.groupId = groupId;
            this.stat = stat;
            this.supportsScoresPerGroup = true;
        }

        OverallStatsSortingStrategy.prototype = Object.create(SortingStrategy.prototype);

        OverallStatsSortingStrategy.prototype.compare = function (a, b) {
            var scoreA = this.getInherentScoreInfo(a.path).score;
            var scoreB = this.getInherentScoreInfo(b.path).score;
            if (this.ascending) {
                return d3.ascending(scoreA, scoreB);
            }
            return d3.descending(scoreA, scoreB);
        };

        OverallStatsSortingStrategy.prototype.getInherentScoreInfo = function (path) {
            return this.getScoreInfo(path, this.datasetId, this.groupId);
        };

        OverallStatsSortingStrategy.prototype.getScoreInfo = function (path, datasetId, groupId) {
            return {score: groupId ? getPathGroupStats(path, datasetId, groupId)[this.stat] : getPathDatasetStats(path, datasetId)[this.stat]};
        };

        /**
         * Sort by stats of individual nodes.
         *
         * @param datasetId
         * @param stat
         * @param aggregateFunction
         * @param label
         * @param groupId
         * @constructor
         */
        function PerNodeStatsSortingStrategy(datasetId, stat, aggregateFunction, label, groupId) {
            SortingStrategy.call(this, SortingStrategy.prototype.STRATEGY_TYPES.ATTRIBUTE, datasetId + ": " + label, "PER_NODE_STATS");
            this.datasetId = datasetId;
            this.groupId = groupId;
            this.stat = stat;
            this.aggregateFunction = aggregateFunction;
            this.supportsScoresPerGroup = true;
        }

        PerNodeStatsSortingStrategy.prototype = Object.create(SortingStrategy.prototype);

        PerNodeStatsSortingStrategy.prototype.compare = function (a, b) {
            var scoreA = this.getInherentScoreInfo(a.path).score;
            var scoreB = this.getInherentScoreInfo(b.path).score;

            if (this.ascending) {
                return d3.ascending(scoreA, scoreB);
            }
            return d3.descending(scoreA, scoreB);
        };

        PerNodeStatsSortingStrategy.prototype.getInherentScoreInfo = function (path) {
            return this.getScoreInfo(path, this.datasetId, this.groupId);
        };

        PerNodeStatsSortingStrategy.prototype.getScoreInfo = function (path, datasetId, groupId) {
            var desc = aggregateStats(getPerNodeStatsWithinGroup(path, datasetId, groupId), this.stat, this.aggregateFunction, function (el) {
                return el.stats;
            });

            return {
                node: desc.node,
                score: desc.stats[this.stat]
            };
        };


        //-------------------------------------------------------------

        /**
         * Sort by an aggregate score of overall group stats.
         *
         * @param datasetId
         * @param stat
         * @param betweenGroupsAggregateFunction
         * @param label
         * @constructor
         */
        function OverallBetweenGroupsSortingStrategy(datasetId, stat, betweenGroupsAggregateFunction, label) {
            SortingStrategy.call(this, SortingStrategy.prototype.STRATEGY_TYPES.ATTRIBUTE, datasetId + ": " + label, "OVERALL_BETWEEN_GROUPS_STATS");
            this.datasetId = datasetId;
            this.stat = stat;
            this.betweenGroupsAggregateFunction = betweenGroupsAggregateFunction;
            this.supportsScoresPerGroup = false;
        }

        OverallBetweenGroupsSortingStrategy.prototype = Object.create(SortingStrategy.prototype);

        OverallBetweenGroupsSortingStrategy.prototype.compare = function (a, b) {

            var scoreA = this.getInherentScoreInfo(a.path).score;
            var scoreB = this.getInherentScoreInfo(b.path).score;

            if (this.ascending) {
                return d3.ascending(scoreA, scoreB);
            }
            return d3.descending(scoreA, scoreB);
        };

        OverallBetweenGroupsSortingStrategy.prototype.getInherentScoreInfo = function (path) {
            return this.getScoreInfo(path, this.datasetId);
        };

        OverallBetweenGroupsSortingStrategy.prototype.getScoreInfo = function (path, datasetId) {
            var desc = getAggregatesBetweenGroups(datasetId, path, this.stat, this.betweenGroupsAggregateFunction);
            return {
                group1: desc.el1.group,
                group2: desc.el2.group,
                score: desc.diff
            }
        };


        /**
         * Sort by an aggregate score of per node stats.
         * @param datasetId
         * @param stat
         * @param betweenGroupsAggregateFunction
         * @param nodeAggregateFunction
         * @param label
         * @constructor
         */
        function PerNodeBetweenGroupsSortingStrategy(datasetId, stat, betweenGroupsAggregateFunction, nodeAggregateFunction, label) {
            SortingStrategy.call(this, SortingStrategy.prototype.STRATEGY_TYPES.ATTRIBUTE, datasetId + ": " + label, "PER_NODE_BETWEEN_GROUPS_STATS");
            this.datasetId = datasetId;
            this.stat = stat;
            this.betweenGroupsAggregateFunction = betweenGroupsAggregateFunction;
            this.nodeAggregateFunction = nodeAggregateFunction;
            this.supportsScoresPerGroup = false;
        }

        PerNodeBetweenGroupsSortingStrategy.prototype = Object.create(SortingStrategy.prototype);

        PerNodeBetweenGroupsSortingStrategy.prototype.compare = function (a, b) {

            var scoreA = this.getInherentScoreInfo(a.path).score;
            var scoreB = this.getInherentScoreInfo(b.path).score;

            if (this.ascending) {
                return d3.ascending(scoreA, scoreB);
            }
            return d3.descending(scoreA, scoreB);
        };

        PerNodeBetweenGroupsSortingStrategy.prototype.getInherentScoreInfo = function (path) {
            return this.getScoreInfo(path, this.datasetId);
        };

        PerNodeBetweenGroupsSortingStrategy.prototype.getScoreInfo = function (path, datasetId) {
            var desc = this.nodeAggregateFunction(getPerNodeAggregatesBetweenGroups(datasetId, path, this.stat, this.betweenGroupsAggregateFunction), function (el) {
                return el.diff;
            });
            return {
                group1: desc.el1.group,
                group2: desc.el2.group,
                node: desc.el1.node,
                score: desc.diff
            };
        };


        function maxOfArray(arr, accessor) {

            var max = Number.NEGATIVE_INFINITY;
            var maxEl = null;

            arr.forEach(function (el) {
                var val = accessor ? accessor(el) : el;
                if (max < val) {
                    max = val;
                    maxEl = el;
                }
            });

            return maxEl;
        }<commit_msg>associated node highlighting for matrix dataset scores<commit_after>define(['d3', 'jquery', './listeners', './query/pathquery', './config', './statisticsutil', './sorting', '../pathfinder_ccle/ccle', './extradata'],
    function (d3, $, listeners, pathQuery, config, statisticsUtil, sorting, ccle, extraData) {

        var SortingStrategy = sorting.SortingStrategy;

        /**
         *
         * Data has been loaded for these nodes or data is currently loading.
         */
        var nodesWithData = {};

        var allDatasets = {};

        var stratification = {};

        var paths = [];

        var nodes = {};

        var nodeProperties = {};

        var pathStats = {};

        /**
         * Sort by stats for the whole dataset or a whole group.
         *
         * @param datasetId
         * @param stat
         * @param groupId
         * @constructor
         */
        function OverallStatsSortingStrategy(datasetId, stat, groupId) {
            SortingStrategy.call(this, SortingStrategy.prototype.STRATEGY_TYPES.ATTRIBUTE, datasetId + ": " + stat, "OVERALL_STATS");
            this.datasetId = datasetId;
            this.groupId = groupId;
            this.stat = stat;
            this.supportsScoresPerGroup = true;
        }

        OverallStatsSortingStrategy.prototype = Object.create(SortingStrategy.prototype);

        OverallStatsSortingStrategy.prototype.compare = function (a, b) {
            var scoreA = this.getInherentScoreInfo(a.path).score;
            var scoreB = this.getInherentScoreInfo(b.path).score;
            if (this.ascending) {
                return d3.ascending(scoreA, scoreB);
            }
            return d3.descending(scoreA, scoreB);
        };

        OverallStatsSortingStrategy.prototype.getInherentScoreInfo = function (path) {
            return this.getScoreInfo(path, this.datasetId, this.groupId);
        };

        OverallStatsSortingStrategy.prototype.getScoreInfo = function (path, datasetId, groupId) {
            return {score: groupId ? getPathGroupStats(path, datasetId, groupId)[this.stat] : getPathDatasetStats(path, datasetId)[this.stat]};
        };

        /**
         * Sort by stats of individual nodes.
         *
         * @param datasetId
         * @param stat
         * @param aggregateFunction
         * @param label
         * @param groupId
         * @constructor
         */
        function PerNodeStatsSortingStrategy(datasetId, stat, aggregateFunction, label, groupId) {
            SortingStrategy.call(this, SortingStrategy.prototype.STRATEGY_TYPES.ATTRIBUTE, datasetId + ": " + label, "PER_NODE_STATS");
            this.datasetId = datasetId;
            this.groupId = groupId;
            this.stat = stat;
            this.aggregateFunction = aggregateFunction;
            this.supportsScoresPerGroup = true;
        }

        PerNodeStatsSortingStrategy.prototype = Object.create(SortingStrategy.prototype);

        PerNodeStatsSortingStrategy.prototype.compare = function (a, b) {
            var scoreA = this.getInherentScoreInfo(a.path).score;
            var scoreB = this.getInherentScoreInfo(b.path).score;

            if (this.ascending) {
                return d3.ascending(scoreA, scoreB);
            }
            return d3.descending(scoreA, scoreB);
        };

        PerNodeStatsSortingStrategy.prototype.getInherentScoreInfo = function (path) {
            return this.getScoreInfo(path, this.datasetId, this.groupId);
        };

        PerNodeStatsSortingStrategy.prototype.getScoreInfo = function (path, datasetId, groupId) {
            var desc = aggregateStats(getPerNodeStatsWithinGroup(path, datasetId, groupId), this.stat, this.aggregateFunction, function (el) {
                return el.stats;
            });

            return {
                node: desc.node,
                score: desc.stats[this.stat],
                idType: "node",
                ids: [desc.node.id]
            };
        };


        //-------------------------------------------------------------

        /**
         * Sort by an aggregate score of overall group stats.
         *
         * @param datasetId
         * @param stat
         * @param betweenGroupsAggregateFunction
         * @param label
         * @constructor
         */
        function OverallBetweenGroupsSortingStrategy(datasetId, stat, betweenGroupsAggregateFunction, label) {
            SortingStrategy.call(this, SortingStrategy.prototype.STRATEGY_TYPES.ATTRIBUTE, datasetId + ": " + label, "OVERALL_BETWEEN_GROUPS_STATS");
            this.datasetId = datasetId;
            this.stat = stat;
            this.betweenGroupsAggregateFunction = betweenGroupsAggregateFunction;
            this.supportsScoresPerGroup = false;
        }

        OverallBetweenGroupsSortingStrategy.prototype = Object.create(SortingStrategy.prototype);

        OverallBetweenGroupsSortingStrategy.prototype.compare = function (a, b) {

            var scoreA = this.getInherentScoreInfo(a.path).score;
            var scoreB = this.getInherentScoreInfo(b.path).score;

            if (this.ascending) {
                return d3.ascending(scoreA, scoreB);
            }
            return d3.descending(scoreA, scoreB);
        };

        OverallBetweenGroupsSortingStrategy.prototype.getInherentScoreInfo = function (path) {
            return this.getScoreInfo(path, this.datasetId);
        };

        OverallBetweenGroupsSortingStrategy.prototype.getScoreInfo = function (path, datasetId) {
            var desc = getAggregatesBetweenGroups(datasetId, path, this.stat, this.betweenGroupsAggregateFunction);
            return {
                group1: desc.el1.group,
                group2: desc.el2.group,
                score: desc.diff
            }
        };


        /**
         * Sort by an aggregate score of per node stats.
         * @param datasetId
         * @param stat
         * @param betweenGroupsAggregateFunction
         * @param nodeAggregateFunction
         * @param label
         * @constructor
         */
        function PerNodeBetweenGroupsSortingStrategy(datasetId, stat, betweenGroupsAggregateFunction, nodeAggregateFunction, label) {
            SortingStrategy.call(this, SortingStrategy.prototype.STRATEGY_TYPES.ATTRIBUTE, datasetId + ": " + label, "PER_NODE_BETWEEN_GROUPS_STATS");
            this.datasetId = datasetId;
            this.stat = stat;
            this.betweenGroupsAggregateFunction = betweenGroupsAggregateFunction;
            this.nodeAggregateFunction = nodeAggregateFunction;
            this.supportsScoresPerGroup = false;
        }

        PerNodeBetweenGroupsSortingStrategy.prototype = Object.create(SortingStrategy.prototype);

        PerNodeBetweenGroupsSortingStrategy.prototype.compare = function (a, b) {

            var scoreA = this.getInherentScoreInfo(a.path).score;
            var scoreB = this.getInherentScoreInfo(b.path).score;

            if (this.ascending) {
                return d3.ascending(scoreA, scoreB);
            }
            return d3.descending(scoreA, scoreB);
        };

        PerNodeBetweenGroupsSortingStrategy.prototype.getInherentScoreInfo = function (path) {
            return this.getScoreInfo(path, this.datasetId);
        };

        PerNodeBetweenGroupsSortingStrategy.prototype.getScoreInfo = function (path, datasetId) {
            var desc = this.nodeAggregateFunction(getPerNodeAggregatesBetweenGroups(datasetId, path, this.stat, this.betweenGroupsAggregateFunction), function (el) {
                return el.diff;
            });
            return {
                group1: desc.el1.group,
                group2: desc.el2.group,
                node: desc.el1.node,
                score: desc.diff,
                idType: "node",
                ids: [desc.el1.node.id]
            };
        };


        function maxOfArray(arr, accessor) {

            var max = Number.NEGATIVE_INFINITY;
            var maxEl = null;

            arr.forEach(function (el) {
                var val = accessor ? accessor(el) : el;
                if (max < val) {
                    max = val;
                    maxEl = el;
                }
            });

            return maxEl;
        }
{'fraction_non_alphanumeric': 0.10032561282791669, 'fraction_numerical': 0.002396018922405849, 'mean_word_length': 1.9824111396115793, 'pattern_counts': {'":': 8, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2426371', 'n_tokens_mistral': 4357, 'n_tokens_neox': 4219, 'n_words': 818}