<commit_before>/**
 * Controller for statistics page
 */
(function() {
    'use strict';

    angular.module('theHiveControllers').controller('StatisticsCtrl', function($scope, $rootScope, $timeout, StatisticSrv) {
        $scope.globalFilters = StatisticSrv.getFilters() || {
            fromDate: moment().subtract(30, 'd').toDate(),
            toDate: moment().toDate(),
            tags: [],
            tagsAggregator: 'any'
        };

        $scope.tagsAggregators = {
            any: 'Any of',
            all: 'All of',
            none: 'None of'
        };

        $scope.caseByTlp = {
            title: 'Cases by TLP',
            type: 'case',
            field: 'tlp',
            dateField: 'startDate',
            tagsField: 'tags',
            colors: {
                '0': '#cccccc',
                '1': '#5cb85c',
                '2': '#f0ad4e',
                '3': '#d9534f'
            },
            names: {
                '0': 'White',
                '1': 'Green',
                '2': 'Amber',
                '3': 'Red'
            }
        };

        $scope.caseBySeverity = {
            title: 'Cases by Severity',
            type: 'case',
            field: 'severity',
            dateField: 'startDate',
            tagsField: 'tags',
            colors: {
                '1': '#5bc0de',
                '2': '#f0ad4e',
                '3': '#d9534f'
            },
            names: {
                '1': 'Low',
                '2': 'Medium',
                '3': 'High'
            }
        };

        $scope.caseByStatus = {
            title: 'Cases by status',
            type: 'case',
            field: 'status',
            dateField: 'startDate',
            tagsField: 'tags'
        };

        $scope.caseByResolution = {
            title: 'Resolved cases by resolution',
            type: 'case',
            field: 'resolutionStatus',
            dateField: 'startDate',
            tagsField: 'tags',
            filter: {status: 'Resolved'}
        };

        $scope.caseOverTime = {
            title: 'Cases over time',
            type: 'case',
            fields: ['startDate', 'endDate'],
            dateField: 'startDate',
            tagsField: 'tags',
            names: {
                startDate: 'Number of open cases',
                endDate: 'Number of resolved cases'
            },
            types: {
                startDate: 'bar'
            }
        };

        $scope.caseHandlingDurationOverTime = {
            title: 'Cases handling over time',
            dateField: 'startDate',
            tagsField: 'tags',
            names: {
                max: 'Max',
                min: 'Min',
                avg: 'Avg',
                count: 'Number of resolved cases'
            },
            types: {
                count: 'bar'
            },
            axes: {
                count: 'y2'
            },
            colors: {
                'count': '#ff7f0e',
                'max': '#d62728',
                'min': '#2ca02c',
                'avg': '#1f77b4'
            },
            filter: {status: 'Resolved'}
        };

        $scope.caseMetricsOverTime = {
            title: 'Case metrics over time',
            entity: 'case',
            type: 'line',
            field: 'startDate',
            dateField: 'startDate',
            tagsField: 'tags',
            aggregations: ['sum']
        };

        $scope.observableByDataType = {
            title: 'Observables by Type',
            type: 'case_artifact',
            field: 'dataType',
            dateField: 'startDate',
            tagsField: 'tags'
        };

        $scope.observableByIoc = {
            title: 'Observables by IOC flag',
            type: 'case_artifact',
            field: 'ioc',
            dateField: 'startDate',
            tagsField: 'tags',
            names: {
                '0': 'NOT IOC',
                '1': 'IOC'
            }
        };

        $scope.observableOverTime = {
            title: 'Observables over time',
            type: 'case/artifact',
            fields: ['startDate'],
            dateField: 'startDate',
            tagsField: 'tags',
            names: {
                startDate: 'Number of observables'
            },
            types: {
                startDate: 'bar'
            }
        };

        $scope.setTagsAggregator = function(aggregator) {
            $scope.globalFilters.tagsAggregator = aggregator;
        };


        // Prepare the global query
        $scope.prepareGlobalQuery = function() {
            // Handle date queries
            var start = $scope.globalFilters.fromDate ? $scope.globalFilters.fromDate.getTime() : '*';
            var end = $scope.globalFilters.toDate ? $scope.globalFilters.toDate.setHours(23,59,59,999) : '*';

            // Handle tags query
            var tags = _.map($scope.globalFilters.tags, function(tag) {
                return tag.text;<commit_msg>#343 Filter deleted observables from statistics page<commit_after>/**
 * Controller for statistics page
 */
(function() {
    'use strict';

    angular.module('theHiveControllers').controller('StatisticsCtrl', function($scope, $rootScope, $timeout, StatisticSrv) {
        $scope.globalFilters = StatisticSrv.getFilters() || {
            fromDate: moment().subtract(30, 'd').toDate(),
            toDate: moment().toDate(),
            tags: [],
            tagsAggregator: 'any'
        };

        $scope.tagsAggregators = {
            any: 'Any of',
            all: 'All of',
            none: 'None of'
        };

        $scope.caseByTlp = {
            title: 'Cases by TLP',
            type: 'case',
            field: 'tlp',
            dateField: 'startDate',
            tagsField: 'tags',
            colors: {
                '0': '#cccccc',
                '1': '#5cb85c',
                '2': '#f0ad4e',
                '3': '#d9534f'
            },
            names: {
                '0': 'White',
                '1': 'Green',
                '2': 'Amber',
                '3': 'Red'
            }
        };

        $scope.caseBySeverity = {
            title: 'Cases by Severity',
            type: 'case',
            field: 'severity',
            dateField: 'startDate',
            tagsField: 'tags',
            colors: {
                '1': '#5bc0de',
                '2': '#f0ad4e',
                '3': '#d9534f'
            },
            names: {
                '1': 'Low',
                '2': 'Medium',
                '3': 'High'
            }
        };

        $scope.caseByStatus = {
            title: 'Cases by status',
            type: 'case',
            field: 'status',
            dateField: 'startDate',
            tagsField: 'tags'
        };

        $scope.caseByResolution = {
            title: 'Resolved cases by resolution',
            type: 'case',
            field: 'resolutionStatus',
            dateField: 'startDate',
            tagsField: 'tags',
            filter: {status: 'Resolved'}
        };

        $scope.caseOverTime = {
            title: 'Cases over time',
            type: 'case',
            fields: ['startDate', 'endDate'],
            dateField: 'startDate',
            tagsField: 'tags',
            names: {
                startDate: 'Number of open cases',
                endDate: 'Number of resolved cases'
            },
            types: {
                startDate: 'bar'
            }
        };

        $scope.caseHandlingDurationOverTime = {
            title: 'Cases handling over time',
            dateField: 'startDate',
            tagsField: 'tags',
            names: {
                max: 'Max',
                min: 'Min',
                avg: 'Avg',
                count: 'Number of resolved cases'
            },
            types: {
                count: 'bar'
            },
            axes: {
                count: 'y2'
            },
            colors: {
                'count': '#ff7f0e',
                'max': '#d62728',
                'min': '#2ca02c',
                'avg': '#1f77b4'
            },
            filter: {status: 'Resolved'}
        };

        $scope.caseMetricsOverTime = {
            title: 'Case metrics over time',
            entity: 'case',
            type: 'line',
            field: 'startDate',
            dateField: 'startDate',
            tagsField: 'tags',
            aggregations: ['sum']
        };

        $scope.observableByDataType = {
            title: 'Observables by Type',
            type: 'case_artifact',
            field: 'dataType',
            dateField: 'startDate',
            tagsField: 'tags',
            filter: {status: 'Ok'}
        };

        $scope.observableByIoc = {
            title: 'Observables by IOC flag',
            type: 'case_artifact',
            field: 'ioc',
            dateField: 'startDate',
            tagsField: 'tags',
            names: {
                '0': 'NOT IOC',
                '1': 'IOC'
            },
            filter: {status: 'Ok'}
        };

        $scope.observableOverTime = {
            title: 'Observables over time',
            type: 'case/artifact',
            fields: ['startDate'],
            dateField: 'startDate',
            tagsField: 'tags',
            names: {
                startDate: 'Number of observables'
            },
            types: {
                startDate: 'bar'
            },
            filter: {status: 'Ok'}
        };

        $scope.setTagsAggregator = function(aggregator) {
            $scope.globalFilters.tagsAggregator = aggregator;
        };


        // Prepare the global query
        $scope.prepareGlobalQuery = function() {
            // Handle date queries
            var start = $scope.globalFilters.fromDate ? $scope.globalFilters.fromDate.getTime() : '*';
            var end = $scope.globalFilters.toDate ? $scope.globalFilters.toDate.setHours(23,59,59,999) : '*';

            // Handle tags query
            var tags = _.map($scope.globalFilters.tags, function(tag) {
                return tag.text;
{'fraction_non_alphanumeric': 0.13124193388265662, 'fraction_numerical': 0.012012310136007148, 'mean_word_length': 1.1880973066898348, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5619362', 'n_tokens_mistral': 2827, 'n_tokens_neox': 2700, 'n_words': 596}