<commit_before>export default ['$stateParams', '$scope', '$state', 'QuerySet', 'GetBasePath', 'QuerySet', 'SmartSearchService', 'i18n',
    function($stateParams, $scope, $state, QuerySet, GetBasePath, qs, SmartSearchService, i18n) {

        let path, relations,
            defaults,
            queryset,
            stateChangeSuccessListener;

        if($scope.defaultParams) {
            defaults = $scope.defaultParams;
        }
        else {
            // steps through the current tree of $state configurations, grabs default search params
            defaults = _.find($state.$current.path, (step) => {
                return step.params.hasOwnProperty(`${$scope.iterator}_search`);
            }).params[`${$scope.iterator}_search`].config.value;
        }

        if($scope.querySet) {
            queryset = _.cloneDeep($scope.querySet);
        }
        else {
            queryset = $stateParams[`${$scope.iterator}_search`];
        }

        // build $scope.tags from $stateParams.QuerySet, build fieldset key
        init();

        function init() {
            path = GetBasePath($scope.basePath) || $scope.basePath;
            relations = getRelationshipFields($scope.dataset.results);
            $scope.searchTags = stripDefaultParams($state.params[`${$scope.iterator}_search`]);
            qs.initFieldset(path, $scope.djangoModel, relations).then((data) => {
                $scope.models = data.models;
                $scope.options = data.options.data;
                $scope.$emit(`${$scope.list.iterator}_options`, data.options);
            });
            $scope.searchPlaceholder = $scope.disableSearch ? i18n._('Cannot search running job') : i18n._('Search');

            function compareParams(a, b) {
                for (let key in a) {
                    if (!(key in b) || a[key].toString() !== b[key].toString()) {
                        return false;
                    }
                }
                for (let key in b) {
                    if (!(key in a)) {
                        return false;
                    }
                }
                return true;
            }

            if(stateChangeSuccessListener) {
                stateChangeSuccessListener();
            }

            stateChangeSuccessListener = $scope.$on('$stateChangeSuccess', function(event, toState, toParams, fromState, fromParams) {
                // State has changed - check to see if this is a param change
                if(fromState.name === toState.name) {
                    if(!compareParams(fromParams[`${$scope.iterator}_search`], toParams[`${$scope.iterator}_search`])) {
                        // Params are not the same - we need to update the search.  This should only happen when the user
                        // hits the forward/back navigation buttons in their browser.
                        queryset = toParams[`${$scope.iterator}_search`];
                        qs.search(path, queryset).then((res) => {
                            $scope.dataset = res.data;
                            $scope.collection = res.data.results;
                        });

                        $scope.searchTerm = null;
                        $scope.searchTags = stripDefaultParams(queryset);
                    }
                }
            });

            $scope.$on('$destroy', stateChangeSuccessListener);

            $scope.$watch('disableSearch', function(disableSearch){
                if(disableSearch) {
                    $scope.searchPlaceholder = i18n._('Cannot search running job');
                }
                else {
                    $scope.searchPlaceholder = i18n._('Search');
                }
            });
        }

        // Removes state definition defaults and pagination terms
        function stripDefaultParams(params) {
            let stripped =_.pick(params, (value, key) => {
                // setting the default value of a term to null in a state definition is a very explicit way to ensure it will NEVER generate a search tag, even with a non-default value
                return defaults[key] !== value && key !== 'order_by' && key !== 'page' && key !== 'page_size' && defaults[key] !== null;
            });
            let strippedCopy = _.cloneDeep(stripped);
            if(_.keys(_.pick(defaults, _.keys(strippedCopy))).length > 0){
                for (var key in strippedCopy) {
                    if (strippedCopy.hasOwnProperty(key)) {
                        let value = strippedCopy[key];
                        if(_.isArray(value)){
                            let index = _.indexOf(value, defaults[key]);
                            value = value.splice(index, 1)[0];
                        }
                    }
                }
                stripped = strippedCopy;
            }
            return _(strippedCopy).map(qs.decodeParam).flatten().value();
        }

        // searchable relationships
        function getRelationshipFields(dataset) {
            let flat = _(dataset).map((value) => {
                return _.keys(value.related);
            }).flatten().uniq().value();
            return flat;
        }

        function setDefaults(term) {
            if ($scope.list.defaultSearchParams) {
                return $scope.list.defaultSearchParams(term);
            } else {
               return {
                    search: encodeURIComponent(term)
                };
            }
        }

        $scope.toggleKeyPane = function() {
            $scope.showKeyPane = !$scope.showKeyPane;
        };

        $scope.clearAll = function(){
            let cleared = _.cloneDeep(defaults);
            delete cleared.page;
            queryset = cleared;
            if(!$scope.querySet) {
                $state.go('.', {[$scope.iterator + '_search']: queryset}, {notify: false});
            }
            qs.search(path, queryset).then((res) => {
                if($scope.querySet) {
                    $scope.querySet = queryset;
                }
                $scope.dataset = res.data;
                $scope.collection = res.data.results;
            });
            $scope.searchTags = stripDefaultParams(queryset);
        };

        // remove tag, merge new queryset, $state.go
        $scope.remove = function(index) {
            let tagToRemove = $scope.searchTags.splice(index, 1)[0];
            let termParts = SmartSearchService.splitTermIntoParts(tagToRemove);
            let removed;
            if (termParts.length === 1) {
                removed = setDefaults(tagToRemove);
            }
            else {
                let root = termParts[0].split(".")[0].replace(/^-/, '');
                let encodeParams = {
                    term: tagToRemove
                };
                if(_.has($scope.options.actions.GET, root)) {
                    if($scope.options.actions.GET[root].type && $scope.options.actions.GET[root].type === 'field') {
                        encodeParams.relatedSearchTerm = true;
                    }
                    else {
                        encodeParams.searchTerm = true;
                    }
                }
                removed = qs.encodeParam(encodeParams);
            }
            _.each(removed, (value, key) => {
                if (Array.isArray(queryset[key])){
                    _.remove(queryset[key], (item) => item === value);
                    // If the array is now empty, remove that key
                    if(queryset[key].length === 0) {
                        delete queryset[key];
                    }
                }
                else {
                    delete queryset[key];
                }
            });
            if(!$scope.querySet) {
                $state.go('.', {
                    [$scope.iterator + '_search']: queryset }, {notify: false});
            }
            qs.search(path, queryset).then((res) => {
                if($scope.querySet) {
                    $scope.querySet = queryset;
                }
                $scope.dataset = res.data;
                $scope.collection = res.data.results;
            });
            $scope.searchTags = stripDefaultParams(queryset);
        };

        // add a search tag, merge new queryset, $state.go()
        $scope.add = function(terms) {
            let params = {},
                origQueryset = _.clone(queryset);

            // Remove leading/trailing whitespace if there is any
            terms = terms.trim();

            if(terms && terms !== '') {
                // Split the terms up
                let splitTerms = SmartSearchService.splitSearchIntoTerms(terms);
                _.forEach(splitTerms, (term) => {

                    let termParts = SmartSearchService.splitTermIntoParts(term);

                    function combineSameSearches(a,b){
                        if (_.isArray(a)) {
                          return a.concat(b);
                        }
                        else {
                            if(a) {
                                return [a,b];
                            }
                        }
                    }

                    // if only a value is provided, search using default keys
                    if (termParts.length === 1) {
                        params = _.merge(params, setDefaults(term), combineSameSearches);
                    } else {
                        // Figure out if this is a search term
                        let root = termParts[0].split(".")[0].replace(/^-/, '');
                        if(_.has($scope.options.actions.GET, root)) {
                            if($scope.options.actions.GET[root].type && $scope.options.actions.GET[root].type === 'field') {<commit_msg>fix for #4969, can't call trim of undefined<commit_after>export default ['$stateParams', '$scope', '$state', 'QuerySet', 'GetBasePath', 'QuerySet', 'SmartSearchService', 'i18n',
    function($stateParams, $scope, $state, QuerySet, GetBasePath, qs, SmartSearchService, i18n) {

        let path, relations,
            defaults,
            queryset,
            stateChangeSuccessListener;

        if($scope.defaultParams) {
            defaults = $scope.defaultParams;
        }
        else {
            // steps through the current tree of $state configurations, grabs default search params
            defaults = _.find($state.$current.path, (step) => {
                return step.params.hasOwnProperty(`${$scope.iterator}_search`);
            }).params[`${$scope.iterator}_search`].config.value;
        }

        if($scope.querySet) {
            queryset = _.cloneDeep($scope.querySet);
        }
        else {
            queryset = $stateParams[`${$scope.iterator}_search`];
        }

        // build $scope.tags from $stateParams.QuerySet, build fieldset key
        init();

        function init() {
            path = GetBasePath($scope.basePath) || $scope.basePath;
            relations = getRelationshipFields($scope.dataset.results);
            $scope.searchTags = stripDefaultParams($state.params[`${$scope.iterator}_search`]);
            qs.initFieldset(path, $scope.djangoModel, relations).then((data) => {
                $scope.models = data.models;
                $scope.options = data.options.data;
                $scope.$emit(`${$scope.list.iterator}_options`, data.options);
            });
            $scope.searchPlaceholder = $scope.disableSearch ? i18n._('Cannot search running job') : i18n._('Search');

            function compareParams(a, b) {
                for (let key in a) {
                    if (!(key in b) || a[key].toString() !== b[key].toString()) {
                        return false;
                    }
                }
                for (let key in b) {
                    if (!(key in a)) {
                        return false;
                    }
                }
                return true;
            }

            if(stateChangeSuccessListener) {
                stateChangeSuccessListener();
            }

            stateChangeSuccessListener = $scope.$on('$stateChangeSuccess', function(event, toState, toParams, fromState, fromParams) {
                // State has changed - check to see if this is a param change
                if(fromState.name === toState.name) {
                    if(!compareParams(fromParams[`${$scope.iterator}_search`], toParams[`${$scope.iterator}_search`])) {
                        // Params are not the same - we need to update the search.  This should only happen when the user
                        // hits the forward/back navigation buttons in their browser.
                        queryset = toParams[`${$scope.iterator}_search`];
                        qs.search(path, queryset).then((res) => {
                            $scope.dataset = res.data;
                            $scope.collection = res.data.results;
                        });

                        $scope.searchTerm = null;
                        $scope.searchTags = stripDefaultParams(queryset);
                    }
                }
            });

            $scope.$on('$destroy', stateChangeSuccessListener);

            $scope.$watch('disableSearch', function(disableSearch){
                if(disableSearch) {
                    $scope.searchPlaceholder = i18n._('Cannot search running job');
                }
                else {
                    $scope.searchPlaceholder = i18n._('Search');
                }
            });
        }

        // Removes state definition defaults and pagination terms
        function stripDefaultParams(params) {
            let stripped =_.pick(params, (value, key) => {
                // setting the default value of a term to null in a state definition is a very explicit way to ensure it will NEVER generate a search tag, even with a non-default value
                return defaults[key] !== value && key !== 'order_by' && key !== 'page' && key !== 'page_size' && defaults[key] !== null;
            });
            let strippedCopy = _.cloneDeep(stripped);
            if(_.keys(_.pick(defaults, _.keys(strippedCopy))).length > 0){
                for (var key in strippedCopy) {
                    if (strippedCopy.hasOwnProperty(key)) {
                        let value = strippedCopy[key];
                        if(_.isArray(value)){
                            let index = _.indexOf(value, defaults[key]);
                            value = value.splice(index, 1)[0];
                        }
                    }
                }
                stripped = strippedCopy;
            }
            return _(strippedCopy).map(qs.decodeParam).flatten().value();
        }

        // searchable relationships
        function getRelationshipFields(dataset) {
            let flat = _(dataset).map((value) => {
                return _.keys(value.related);
            }).flatten().uniq().value();
            return flat;
        }

        function setDefaults(term) {
            if ($scope.list.defaultSearchParams) {
                return $scope.list.defaultSearchParams(term);
            } else {
               return {
                    search: encodeURIComponent(term)
                };
            }
        }

        $scope.toggleKeyPane = function() {
            $scope.showKeyPane = !$scope.showKeyPane;
        };

        $scope.clearAll = function(){
            let cleared = _.cloneDeep(defaults);
            delete cleared.page;
            queryset = cleared;
            if(!$scope.querySet) {
                $state.go('.', {[$scope.iterator + '_search']: queryset}, {notify: false});
            }
            qs.search(path, queryset).then((res) => {
                if($scope.querySet) {
                    $scope.querySet = queryset;
                }
                $scope.dataset = res.data;
                $scope.collection = res.data.results;
            });
            $scope.searchTags = stripDefaultParams(queryset);
        };

        // remove tag, merge new queryset, $state.go
        $scope.remove = function(index) {
            let tagToRemove = $scope.searchTags.splice(index, 1)[0];
            let termParts = SmartSearchService.splitTermIntoParts(tagToRemove);
            let removed;
            if (termParts.length === 1) {
                removed = setDefaults(tagToRemove);
            }
            else {
                let root = termParts[0].split(".")[0].replace(/^-/, '');
                let encodeParams = {
                    term: tagToRemove
                };
                if(_.has($scope.options.actions.GET, root)) {
                    if($scope.options.actions.GET[root].type && $scope.options.actions.GET[root].type === 'field') {
                        encodeParams.relatedSearchTerm = true;
                    }
                    else {
                        encodeParams.searchTerm = true;
                    }
                }
                removed = qs.encodeParam(encodeParams);
            }
            _.each(removed, (value, key) => {
                if (Array.isArray(queryset[key])){
                    _.remove(queryset[key], (item) => item === value);
                    // If the array is now empty, remove that key
                    if(queryset[key].length === 0) {
                        delete queryset[key];
                    }
                }
                else {
                    delete queryset[key];
                }
            });
            if(!$scope.querySet) {
                $state.go('.', {
                    [$scope.iterator + '_search']: queryset }, {notify: false});
            }
            qs.search(path, queryset).then((res) => {
                if($scope.querySet) {
                    $scope.querySet = queryset;
                }
                $scope.dataset = res.data;
                $scope.collection = res.data.results;
            });
            $scope.searchTags = stripDefaultParams(queryset);
        };

        // add a search tag, merge new queryset, $state.go()
        $scope.add = function(terms) {
            let params = {},
                origQueryset = _.clone(queryset);

            // Remove leading/trailing whitespace if there is any
            terms = (terms) ? terms.trim() : "";

            if(terms && terms !== '') {
                // Split the terms up
                let splitTerms = SmartSearchService.splitSearchIntoTerms(terms);
                _.forEach(splitTerms, (term) => {

                    let termParts = SmartSearchService.splitTermIntoParts(term);

                    function combineSameSearches(a,b){
                        if (_.isArray(a)) {
                          return a.concat(b);
                        }
                        else {
                            if(a) {
                                return [a,b];
                            }
                        }
                    }

                    // if only a value is provided, search using default keys
                    if (termParts.length === 1) {
                        params = _.merge(params, setDefaults(term), combineSameSearches);
                    } else {
                        // Figure out if this is a search term
                        let root = termParts[0].split(".")[0].replace(/^-/, '');
                        if(_.has($scope.options.actions.GET, root)) {
                            if($scope.options.actions.GET[root].type && $scope.options.actions.GET[root].type === 'field') {
{'fraction_non_alphanumeric': 0.11924632635297731, 'fraction_numerical': 0.0026624340791562132, 'mean_word_length': 1.3313439961804727, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2403674', 'n_tokens_mistral': 4918, 'n_tokens_neox': 4629, 'n_words': 996}