<commit_before>/**
 *
 * Copyright (c) 2006-2017, Speedment, Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); You may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */
package com.speedment.runtime.core.internal.component.sql.optimizer;

import com.speedment.runtime.config.identifier.ColumnIdentifier;
import com.speedment.runtime.core.component.sql.Metrics;
import com.speedment.runtime.core.component.sql.SqlStreamOptimizer;
import com.speedment.runtime.core.component.sql.SqlStreamOptimizerInfo;
import com.speedment.runtime.core.db.AsynchronousQueryResult;
import com.speedment.runtime.core.db.DbmsType;
import static com.speedment.runtime.core.db.DbmsType.SkipLimitSupport.NONE;
import static com.speedment.runtime.core.db.DbmsType.SkipLimitSupport.ONLY_AFTER_SORTED;
import com.speedment.runtime.core.internal.stream.builder.action.reference.FilterAction;
import com.speedment.runtime.core.internal.stream.builder.action.reference.LimitAction;
import com.speedment.runtime.core.internal.stream.builder.action.reference.SkipAction;
import com.speedment.runtime.core.internal.stream.builder.action.reference.SortedComparatorAction;
import com.speedment.runtime.core.internal.stream.builder.streamterminator.StreamTerminatorUtil;
import com.speedment.runtime.core.internal.stream.builder.streamterminator.StreamTerminatorUtil.RenderResult;
import static com.speedment.runtime.core.internal.stream.builder.streamterminator.StreamTerminatorUtil.isContainingOnlyFieldPredicate;
import static com.speedment.runtime.core.internal.stream.builder.streamterminator.StreamTerminatorUtil.isSortedActionWithFieldPredicate;
import com.speedment.runtime.core.stream.Pipeline;
import com.speedment.runtime.core.stream.action.Action;
import com.speedment.runtime.field.comparator.CombinedComparator;
import com.speedment.runtime.field.comparator.FieldComparator;
import com.speedment.runtime.field.comparator.NullOrder;
import java.util.*;
import static java.util.Objects.requireNonNull;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
import java.util.function.Predicate;
import static java.util.stream.Collectors.toList;

/**
 * This Optimizer takes care of the following case:
 * <br>
 * <ul>
 * <li> a) Zero or more filter() operations
 * <li> b) Zero or more sorted() operations
 * <li> c) Zero or more skip() operations
 * <li> d) Zero or more limit() operations
 *
 * <em>No other operations</em> must be in the sequence a-d or within the
 * individual items a-d. <em>All</em> parameters in a and b must be obtained via
 * fields. Failure to any of these rules will make the Optimizer reject
 * optimization. Steps a) and b) may swap places.
 *
 * Thus, this optimizer can handle a (FILTER*, SORTED*, SKIP*, LIMIT*) or
 * (SORTED*, LIMIT*, SKIP*, LIMIT*) pattern where all non-primitive parameters
 * are all Field derived
 *
 *
 * @author <NAME>
 * @param <ENTITY> entity type
 */
public final class FilterSortedSkipOptimizer<ENTITY> implements SqlStreamOptimizer<ENTITY> {

    private final FilterOperation FILTER_OPERATION = new FilterOperation();
    private final SortedOperation SORTED_OPERATION = new SortedOperation();
    private final SkipOperation SKIP_OPERATION = new SkipOperation();
    private final LimitOperation LIMIT_OPERATION = new LimitOperation();

    private final List<Operation<ENTITY>> FILTER_SORTED_SKIP_LIMIT_PATH = Arrays.asList(
        FILTER_OPERATION,
        SORTED_OPERATION,
        SKIP_OPERATION,
        LIMIT_OPERATION
    );
    private final List<Operation<ENTITY>> SORTED_FILTER_SKIP_LIMIT_PATH = Arrays.asList(
        SORTED_OPERATION,
        FILTER_OPERATION,
        SKIP_OPERATION,
        LIMIT_OPERATION
    );

    // FILTER <-> SORTED
    // This optimizer can handle a (FILTER*,SORTED*,SKIP*, LIMIT*) pattern where filter and sorted parameters are all Field derived
    @Override
    public Metrics metrics(Pipeline initialPipeline, DbmsType dbmsType) {
        requireNonNull(initialPipeline);
        requireNonNull(dbmsType);
        final DbmsType.SkipLimitSupport skipLimitSupport = dbmsType.getSkipLimitSupport();
        final AtomicInteger filterCounter = new AtomicInteger();
        final AtomicInteger orderCounter = new AtomicInteger();
        final AtomicInteger skipCounter = new AtomicInteger();
        final AtomicInteger limitCounter = new AtomicInteger();

        traverse(initialPipeline,
            $ -> filterCounter.incrementAndGet(),
            $ -> orderCounter.incrementAndGet(),
            $ -> skipCounter.incrementAndGet(),
            $ -> limitCounter.incrementAndGet()
        );

        if (skipLimitSupport == ONLY_AFTER_SORTED && orderCounter.get() == 0) {
            // Just decline. There are other optimizer that handles just filtering better
            return Metrics.empty();
        }
        if (skipLimitSupport == NONE) {
            return Metrics.of(filterCounter.get() + orderCounter.get(), filterCounter.get(), orderCounter.get(), 0, 0);
        }

        return Metrics.of(
            filterCounter.get() + orderCounter.get() + skipCounter.get() + limitCounter.get(),
            filterCounter.get(),
            orderCounter.get(),
            skipCounter.get() > 0 ? 1 : 0,
            limitCounter.get() > 0 ? 1 : 0
        );
    }

    @Override
    public <P extends Pipeline> P optimize(
        final P initialPipeline,
        final SqlStreamOptimizerInfo<ENTITY> info,
        final AsynchronousQueryResult<ENTITY> query
    ) {
        requireNonNull(initialPipeline);
        requireNonNull(info);
        requireNonNull(query);
        final DbmsType dbmsType = info.getDbmsType();
        final DbmsType.SkipLimitSupport skipLimitSupport = dbmsType.getSkipLimitSupport();
        final List<FilterAction<ENTITY>> filters = new ArrayList<>();
        final List<SortedComparatorAction<ENTITY>> sorteds = new ArrayList<>();
        final List<SkipAction<ENTITY>> skips = new ArrayList<>();
        final List<LimitAction<ENTITY>> limits = new ArrayList<>();

        traverse(initialPipeline, filters::add, sorteds::add, skips::add, limits::add);

        final List<Object> values = new ArrayList<>();
        final StringBuilder sql = new StringBuilder();

        sql.append(info.getSqlSelect());

        if (!filters.isEmpty()) {
            @SuppressWarnings("unchecked")
            List<Predicate<ENTITY>> predicates = filters.stream()
                .map(FilterAction::getPredicate)
                .map(p -> (Predicate<ENTITY>) p)
                .collect(toList());

            final RenderResult rr = StreamTerminatorUtil.renderSqlWhere(
                dbmsType,
                info.getSqlColumnNamer(),
                info.getSqlDatabaseTypeFunction(),
                predicates
            );

            sql.append(" WHERE ").append(rr.getSql());
            values.addAll(rr.getValues());
        }

        if (!sorteds.isEmpty()) {

            final List<FieldComparator<ENTITY>> fieldComparators = new ArrayList<>();
            for (int i = sorteds.size() - 1; i >= 0; i--) {
                final SortedComparatorAction<ENTITY> sortedAction = sorteds.get(i);
                @SuppressWarnings("unchecked")
                final Comparator<? super ENTITY> comparator = sortedAction.getComparator();
                if (comparator instanceof FieldComparator) {
                    @SuppressWarnings("unchecked")
                    final FieldComparator<ENTITY> fieldComparator = (FieldComparator<ENTITY>) sortedAction.getComparator();
                    fieldComparators.add(fieldComparator);
                }
                if (comparator instanceof CombinedComparator) {
                    @SuppressWarnings("unchecked")
                    final CombinedComparator<ENTITY> combinedComparator = (CombinedComparator<ENTITY>) sortedAction.getComparator();
                    @SuppressWarnings("unchecked")
                    List<FieldComparator<ENTITY>> comparators = combinedComparator.stream()
                        .map(c -> (FieldComparator<ENTITY>)c)
                        .collect(toList());
                    Collections.reverse(comparators);
                    fieldComparators.addAll(comparators);
                }
            }

            if (!fieldComparators.isEmpty()) {

                sql.append(" ORDER BY ");
                // Iterate backwards
                final Set<ColumnIdentifier<ENTITY>> columns = new HashSet<>();
                int cnt = 0;
                for (FieldComparator<ENTITY> fieldComparator:fieldComparators) {
                    final ColumnIdentifier<ENTITY> columnIdentifier = fieldComparator.getField().identifier();

                    // Some databases (e.g. SQL Server) only allows distinct columns in ORDER BY 
                    if (columns.add(columnIdentifier)) {
                        if (cnt++ != 0) {
                            sql.append(", ");
                        }

                        boolean isReversed = fieldComparator.isReversed();
                        String fieldName = info.getSqlColumnNamer().apply(fieldComparator.getField());

                        final NullOrder effectiveNullOrder = isReversed
                            ? fieldComparator.getNullOrder().reversed()
                            : fieldComparator.getNullOrder();

                        // Specify NullOrder pre column if nulls are first
                        if (effectiveNullOrder == NullOrder.FIRST) {
                            if (dbmsType.getSortByNullOrderInsertion() == DbmsType.SortByNullOrderInsertion.PRE) {
                                sql.append(fieldName).append("IS NOT NULL, ");
                            }
                            if (dbmsType.getSortByNullOrderInsertion() == DbmsType.SortByNullOrderInsertion.PRE_WITH_CASE) {
                                sql.append("CASE WHEN ").append(fieldName).append(" IS NULL THEN 0 ELSE 1 END, ");
                            }
                        }

                        sql.append(fieldName);
                        if (isReversed) {
                            sql.append(" DESC");<commit_msg>runtime-core: Fix order in comparator<commit_after>/**
 *
 * Copyright (c) 2006-2017, Speedment, Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); You may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */
package com.speedment.runtime.core.internal.component.sql.optimizer;

import com.speedment.runtime.config.identifier.ColumnIdentifier;
import com.speedment.runtime.core.component.sql.Metrics;
import com.speedment.runtime.core.component.sql.SqlStreamOptimizer;
import com.speedment.runtime.core.component.sql.SqlStreamOptimizerInfo;
import com.speedment.runtime.core.db.AsynchronousQueryResult;
import com.speedment.runtime.core.db.DbmsType;
import static com.speedment.runtime.core.db.DbmsType.SkipLimitSupport.NONE;
import static com.speedment.runtime.core.db.DbmsType.SkipLimitSupport.ONLY_AFTER_SORTED;
import com.speedment.runtime.core.internal.stream.builder.action.reference.FilterAction;
import com.speedment.runtime.core.internal.stream.builder.action.reference.LimitAction;
import com.speedment.runtime.core.internal.stream.builder.action.reference.SkipAction;
import com.speedment.runtime.core.internal.stream.builder.action.reference.SortedComparatorAction;
import com.speedment.runtime.core.internal.stream.builder.streamterminator.StreamTerminatorUtil;
import com.speedment.runtime.core.internal.stream.builder.streamterminator.StreamTerminatorUtil.RenderResult;
import static com.speedment.runtime.core.internal.stream.builder.streamterminator.StreamTerminatorUtil.isContainingOnlyFieldPredicate;
import static com.speedment.runtime.core.internal.stream.builder.streamterminator.StreamTerminatorUtil.isSortedActionWithFieldPredicate;
import com.speedment.runtime.core.stream.Pipeline;
import com.speedment.runtime.core.stream.action.Action;
import com.speedment.runtime.field.comparator.CombinedComparator;
import com.speedment.runtime.field.comparator.FieldComparator;
import com.speedment.runtime.field.comparator.NullOrder;
import java.util.*;
import static java.util.Objects.requireNonNull;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
import java.util.function.Predicate;
import static java.util.stream.Collectors.toList;

/**
 * This Optimizer takes care of the following case:
 * <br>
 * <ul>
 * <li> a) Zero or more filter() operations
 * <li> b) Zero or more sorted() operations
 * <li> c) Zero or more skip() operations
 * <li> d) Zero or more limit() operations
 *
 * <em>No other operations</em> must be in the sequence a-d or within the
 * individual items a-d. <em>All</em> parameters in a and b must be obtained via
 * fields. Failure to any of these rules will make the Optimizer reject
 * optimization. Steps a) and b) may swap places.
 *
 * Thus, this optimizer can handle a (FILTER*, SORTED*, SKIP*, LIMIT*) or
 * (SORTED*, LIMIT*, SKIP*, LIMIT*) pattern where all non-primitive parameters
 * are all Field derived
 *
 *
 * @author <NAME>
 * @param <ENTITY> entity type
 */
public final class FilterSortedSkipOptimizer<ENTITY> implements SqlStreamOptimizer<ENTITY> {

    private final FilterOperation FILTER_OPERATION = new FilterOperation();
    private final SortedOperation SORTED_OPERATION = new SortedOperation();
    private final SkipOperation SKIP_OPERATION = new SkipOperation();
    private final LimitOperation LIMIT_OPERATION = new LimitOperation();

    private final List<Operation<ENTITY>> FILTER_SORTED_SKIP_LIMIT_PATH = Arrays.asList(
        FILTER_OPERATION,
        SORTED_OPERATION,
        SKIP_OPERATION,
        LIMIT_OPERATION
    );
    private final List<Operation<ENTITY>> SORTED_FILTER_SKIP_LIMIT_PATH = Arrays.asList(
        SORTED_OPERATION,
        FILTER_OPERATION,
        SKIP_OPERATION,
        LIMIT_OPERATION
    );

    // FILTER <-> SORTED
    // This optimizer can handle a (FILTER*,SORTED*,SKIP*, LIMIT*) pattern where filter and sorted parameters are all Field derived
    @Override
    public Metrics metrics(Pipeline initialPipeline, DbmsType dbmsType) {
        requireNonNull(initialPipeline);
        requireNonNull(dbmsType);
        final DbmsType.SkipLimitSupport skipLimitSupport = dbmsType.getSkipLimitSupport();
        final AtomicInteger filterCounter = new AtomicInteger();
        final AtomicInteger orderCounter = new AtomicInteger();
        final AtomicInteger skipCounter = new AtomicInteger();
        final AtomicInteger limitCounter = new AtomicInteger();

        traverse(initialPipeline,
            $ -> filterCounter.incrementAndGet(),
            $ -> orderCounter.incrementAndGet(),
            $ -> skipCounter.incrementAndGet(),
            $ -> limitCounter.incrementAndGet()
        );

        if (skipLimitSupport == ONLY_AFTER_SORTED && orderCounter.get() == 0) {
            // Just decline. There are other optimizer that handles just filtering better
            return Metrics.empty();
        }
        if (skipLimitSupport == NONE) {
            return Metrics.of(filterCounter.get() + orderCounter.get(), filterCounter.get(), orderCounter.get(), 0, 0);
        }

        return Metrics.of(
            filterCounter.get() + orderCounter.get() + skipCounter.get() + limitCounter.get(),
            filterCounter.get(),
            orderCounter.get(),
            skipCounter.get() > 0 ? 1 : 0,
            limitCounter.get() > 0 ? 1 : 0
        );
    }

    @Override
    @SuppressWarnings("unchecked")
    public <P extends Pipeline> P optimize(
        final P initialPipeline,
        final SqlStreamOptimizerInfo<ENTITY> info,
        final AsynchronousQueryResult<ENTITY> query
    ) {
        requireNonNull(initialPipeline);
        requireNonNull(info);
        requireNonNull(query);
        final DbmsType dbmsType = info.getDbmsType();
        final DbmsType.SkipLimitSupport skipLimitSupport = dbmsType.getSkipLimitSupport();
        final List<FilterAction<ENTITY>> filters = new ArrayList<>();
        final List<SortedComparatorAction<ENTITY>> sorteds = new ArrayList<>();
        final List<SkipAction<ENTITY>> skips = new ArrayList<>();
        final List<LimitAction<ENTITY>> limits = new ArrayList<>();

        traverse(initialPipeline, filters::add, sorteds::add, skips::add, limits::add);

        final List<Object> values = new ArrayList<>();
        final StringBuilder sql = new StringBuilder();

        sql.append(info.getSqlSelect());

        if (!filters.isEmpty()) {
            @SuppressWarnings("unchecked")
            List<Predicate<ENTITY>> predicates = filters.stream()
                .map(FilterAction::getPredicate)
                .map(p -> (Predicate<ENTITY>) p)
                .collect(toList());

            final RenderResult rr = StreamTerminatorUtil.renderSqlWhere(
                dbmsType,
                info.getSqlColumnNamer(),
                info.getSqlDatabaseTypeFunction(),
                predicates
            );

            sql.append(" WHERE ").append(rr.getSql());
            values.addAll(rr.getValues());
        }

        if (!sorteds.isEmpty()) {

            final List<FieldComparator<ENTITY>> fieldComparators = new ArrayList<>();
            for (int i = sorteds.size() - 1; i >= 0; i--) {
                final SortedComparatorAction<ENTITY> sortedAction = sorteds.get(i);
                @SuppressWarnings("unchecked")
                final Comparator<? super ENTITY> comparator = sortedAction.getComparator();
                if (comparator instanceof FieldComparator) {
                    @SuppressWarnings("unchecked")
                    final FieldComparator<ENTITY> fieldComparator = (FieldComparator<ENTITY>) sortedAction.getComparator();
                    fieldComparators.add(fieldComparator);
                }
                if (comparator instanceof CombinedComparator) {
                    @SuppressWarnings("unchecked")
                    final CombinedComparator<ENTITY> combinedComparator = (CombinedComparator<ENTITY>) sortedAction.getComparator();
                    combinedComparator.stream()
                        .map(c -> (FieldComparator<ENTITY>) c)
                        .forEachOrdered(fieldComparators::add);
                }
            }

            if (!fieldComparators.isEmpty()) {

                sql.append(" ORDER BY ");
                // Iterate backwards
                final Set<ColumnIdentifier<ENTITY>> columns = new HashSet<>();
                int cnt = 0;
                for (FieldComparator<ENTITY> fieldComparator : fieldComparators) {
                    final ColumnIdentifier<ENTITY> columnIdentifier = fieldComparator.getField().identifier();

                    // Some databases (e.g. SQL Server) only allows distinct columns in ORDER BY 
                    if (columns.add(columnIdentifier)) {
                        if (cnt++ != 0) {
                            sql.append(", ");
                        }

                        boolean isReversed = fieldComparator.isReversed();
                        String fieldName = info.getSqlColumnNamer().apply(fieldComparator.getField());

                        final NullOrder effectiveNullOrder = isReversed
                            ? fieldComparator.getNullOrder().reversed()
                            : fieldComparator.getNullOrder();

                        // Specify NullOrder pre column if nulls are first
                        if (effectiveNullOrder == NullOrder.FIRST) {
                            if (dbmsType.getSortByNullOrderInsertion() == DbmsType.SortByNullOrderInsertion.PRE) {
                                sql.append(fieldName).append("IS NOT NULL, ");
                            }
                            if (dbmsType.getSortByNullOrderInsertion() == DbmsType.SortByNullOrderInsertion.PRE_WITH_CASE) {
                                sql.append("CASE WHEN ").append(fieldName).append(" IS NULL THEN 0 ELSE 1 END, ");
                            }
                        }

                        sql.append(fieldName);
                        if (isReversed) {
                            sql.append(" DESC");
{'fraction_non_alphanumeric': 0.09809122341685218, 'fraction_numerical': 0.0025576658930516745, 'mean_word_length': 2.6278350515463917, 'pattern_counts': {'":': 0, '<': 113, '<?xml version=': 0, '>': 131, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1122966', 'n_tokens_mistral': 5515, 'n_tokens_neox': 5386, 'n_words': 1281}