<commit_before>/*
 * Copyright 2015 Red Hat, Inc. and/or its affiliates
 * and other contributors as indicated by the @author tags.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.hawkular.inventory.base;

import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toSet;

import static org.hawkular.inventory.api.Relationships.WellKnown.contains;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;

import org.hawkular.inventory.api.filters.Filter;
import org.hawkular.inventory.api.filters.Related;
import org.hawkular.inventory.api.filters.With;
import org.hawkular.inventory.api.model.CanonicalPath;
import org.hawkular.inventory.api.model.Path;

/**
 * @author <NAME>
 * @since 0.6.0
 */
final class QueryOptimizer {
    private QueryOptimizer() {

    }

    public static void appendOptimized(List<QueryFragment> fragments, QueryFragment... filters) {
        //remove duplicates
        List<QueryFragment> applicableNewFilters;

        if (fragments.isEmpty()) {
            applicableNewFilters = new ArrayList<>(Arrays.asList(filters));
        } else {
            applicableNewFilters = removeDuplicates(fragments.get(fragments.size() - 1).getFilter(),
                    new ArrayList<>(Arrays.asList(filters)));
        }

        cleanUnnecessaryNoops(applicableNewFilters);

        QueryFragment cpFilter = findLastCanonicalFilterAndPrepareNewFilters(fragments, applicableNewFilters);

        List<CanonicalPath.Extender> checkers;
        if (cpFilter == null) {
            checkers = Collections.singletonList(CanonicalPath.empty());
        } else {
            With.CanonicalPaths cps = (With.CanonicalPaths) cpFilter.getFilter();
            checkers = Arrays.asList(cps.getPaths()).stream().map(CanonicalPath::modified).collect(toList());
        }

        while (!applicableNewFilters.isEmpty()) {
            boolean isPath = applicableNewFilters.get(0) instanceof PathFragment;

            QueryFragment first, second, third;

            if ((first = removeOrReadd(applicableNewFilters, null, null)) == null) {
                break;
            }
            if ((second = removeOrReadd(applicableNewFilters, first, null)) == null) {
                break;
            }
            third = applicableNewFilters.isEmpty() ? null : applicableNewFilters.remove(0);

            //check that the query is not transitioning from a path to a filter or vice versa
            //we do this check only on the positions of the contains filter, because only that can actually influence
            //the results
            if (cpFilter != null && first.getClass() != cpFilter.getClass()) {
                //we were trying to continue a classpath filter. move the contains to the processed fragments
                //and try with the type/id as if this was a new start of the cp filter.
                reAddNonNull(false, fragments, first);
                reAddNonNull(true, applicableNewFilters, second, third);
                checkers = Collections.singletonList(CanonicalPath.empty());
                cpFilter = null;
                continue;
            }

            With.Types typeFilter;
            With.Ids idFilter;

            if (cpFilter == null) {
                //there is no prior canonical path filter, so we're starting a new one potentially
                //this means that first and second are type+id
                //additionally, the contains query fragment, if it is present, must be of the same type as the type/id
                //filters.
                //this is so that they all are either path segs or filter segs - we must not mix them together.
                typeFilter = choose(With.Types.class, first, second);
                idFilter = choose(With.Ids.class, first, second);
                if (third != null && (!isOutgoingContains(third.getFilter()) || second.getClass() != third
                        .getClass())) {
                    reAddNonNull(true, applicableNewFilters, third);
                }
            } else {
                //there is a prior canonical path filter, so we're trying to continue it.
                //this means that first must be a contains filter, and second and third are type+id
                //again we must check for the change in the query fragment type
                if (!isOutgoingContains(first.getFilter())) {
                    reAddNonNull(false, fragments, first);
                    reAddNonNull(true, applicableNewFilters, second, third);
                    checkers = Collections.singletonList(CanonicalPath.empty());
                    cpFilter = null;
                    continue;
                }

                typeFilter = choose(With.Types.class, second, third);
                idFilter = choose(With.Ids.class, second, third);
            }

            if (typeFilter == null || idFilter == null) {
                //hmm... not sure what to do with filters invalid for cp extension.. so just add them back to
                //fragments and try to continue
                reAddNonNull(false, fragments, first, second, third);
                checkers = Collections.singletonList(CanonicalPath.empty());
                cpFilter = null;
                continue;
            }
<commit_msg>Fixed a possible duplication of filter during query optimization.<commit_after>/*
 * Copyright 2015 Red Hat, Inc. and/or its affiliates
 * and other contributors as indicated by the @author tags.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.hawkular.inventory.base;

import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toSet;

import static org.hawkular.inventory.api.Relationships.WellKnown.contains;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;

import org.hawkular.inventory.api.filters.Filter;
import org.hawkular.inventory.api.filters.Related;
import org.hawkular.inventory.api.filters.With;
import org.hawkular.inventory.api.model.CanonicalPath;
import org.hawkular.inventory.api.model.Path;

/**
 * @author <NAME>
 * @since 0.6.0
 */
final class QueryOptimizer {
    private QueryOptimizer() {

    }

    public static void appendOptimized(List<QueryFragment> fragments, QueryFragment... filters) {
        //remove duplicates
        List<QueryFragment> applicableNewFilters;

        if (fragments.isEmpty()) {
            applicableNewFilters = new ArrayList<>(Arrays.asList(filters));
        } else {
            applicableNewFilters = removeDuplicates(fragments.get(fragments.size() - 1).getFilter(),
                    new ArrayList<>(Arrays.asList(filters)));
        }

        cleanUnnecessaryNoops(applicableNewFilters);

        QueryFragment cpFilter = findLastCanonicalFilterAndPrepareNewFilters(fragments, applicableNewFilters);

        List<CanonicalPath.Extender> checkers;
        if (cpFilter == null) {
            checkers = Collections.singletonList(CanonicalPath.empty());
        } else {
            With.CanonicalPaths cps = (With.CanonicalPaths) cpFilter.getFilter();
            checkers = Arrays.asList(cps.getPaths()).stream().map(CanonicalPath::modified).collect(toList());
        }

        while (!applicableNewFilters.isEmpty()) {
            boolean isPath = applicableNewFilters.get(0) instanceof PathFragment;

            QueryFragment first, second, third;

            if ((first = removeOrReadd(applicableNewFilters, null, null)) == null) {
                break;
            }
            if ((second = removeOrReadd(applicableNewFilters, first, null)) == null) {
                break;
            }
            third = applicableNewFilters.isEmpty() ? null : applicableNewFilters.remove(0);

            //check that the query is not transitioning from a path to a filter or vice versa
            //we do this check only on the positions of the contains filter, because only that can actually influence
            //the results
            if (cpFilter != null && first.getClass() != cpFilter.getClass()) {
                //we were trying to continue a classpath filter. move the contains to the processed fragments
                //and try with the type/id as if this was a new start of the cp filter.
                reAddNonNull(false, fragments, first);
                reAddNonNull(true, applicableNewFilters, second, third);
                checkers = Collections.singletonList(CanonicalPath.empty());
                cpFilter = null;
                continue;
            }

            With.Types typeFilter;
            With.Ids idFilter;

            if (cpFilter == null) {
                //there is no prior canonical path filter, so we're starting a new one potentially
                //this means that first and second are type+id
                //additionally, the contains query fragment, if it is present, must be of the same type as the type/id
                //filters.
                //this is so that they all are either path segs or filter segs - we must not mix them together.
                typeFilter = choose(With.Types.class, first, second);
                idFilter = choose(With.Ids.class, first, second);
                if (third != null && (!isOutgoingContains(third.getFilter()) || second.getClass() != third
                        .getClass())) {
                    reAddNonNull(true, applicableNewFilters, third);
                    third = null;
                }
            } else {
                //there is a prior canonical path filter, so we're trying to continue it.
                //this means that first must be a contains filter, and second and third are type+id
                //again we must check for the change in the query fragment type
                if (!isOutgoingContains(first.getFilter())) {
                    reAddNonNull(false, fragments, first);
                    reAddNonNull(true, applicableNewFilters, second, third);
                    checkers = Collections.singletonList(CanonicalPath.empty());
                    cpFilter = null;
                    continue;
                }

                typeFilter = choose(With.Types.class, second, third);
                idFilter = choose(With.Ids.class, second, third);
            }

            if (typeFilter == null || idFilter == null) {
                //hmm... not sure what to do with filters invalid for cp extension.. so just add them back to
                //fragments and try to continue
                reAddNonNull(false, fragments, first, second, third);
                checkers = Collections.singletonList(CanonicalPath.empty());
                cpFilter = null;
                continue;
            }

{'fraction_non_alphanumeric': 0.08938687104298568, 'fraction_numerical': 0.0023325558147284237, 'mean_word_length': 2.2419659735349717, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6724310', 'n_tokens_mistral': 3045, 'n_tokens_neox': 2948, 'n_words': 974}