<commit_before>/*
 * Licensed to CRATE Technology GmbH ("Crate") under one or more contributor
 * license agreements.  See the NOTICE file distributed with this work for
 * additional information regarding copyright ownership.  Crate licenses
 * this file to you under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.  You may
 * obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
 * License for the specific language governing permissions and limitations
 * under the License.
 *
 * However, if you have executed another commercial license agreement
 * with Crate these terms will supersede the license and you may use the
 * software solely pursuant to the terms of the relevant commercial agreement.
 */

package io.crate.operator.operations;

import io.crate.metadata.*;
import io.crate.operator.Input;
import io.crate.operator.InputCollectExpression;
import io.crate.operator.aggregation.AggregationFunction;
import io.crate.operator.aggregation.CollectExpression;
import io.crate.operator.aggregation.FunctionExpression;
import io.crate.operator.reference.doc.CollectorExpression;
import io.crate.operator.reference.doc.DocLevelExpressions;
import io.crate.planner.RowGranularity;
import io.crate.planner.node.CollectNode;
import io.crate.planner.symbol.*;
import org.cratedb.sql.CrateException;

import java.util.*;

/**
 * convert Symbols into Inputs for evaluation and CollectExpressions
 * that might be treated in a special way
 */
public class ImplementationSymbolVisitor extends SymbolVisitor<ImplementationSymbolVisitor.Context, Input<?>> {
    public static class Context {
        protected Set<CollectExpression<?>> collectExpressions = new LinkedHashSet<>(); // to keep insertion order
        protected List<Input<?>> topLevelInputs = new ArrayList<>();
        protected List<CollectorExpression<?>> docLevelExpressions = new ArrayList<>();
        protected RowGranularity maxGranularity = RowGranularity.CLUSTER;
        protected List<AggregationContext> aggregations = new ArrayList<>();

        public RowGranularity maxGranularity() {
            return maxGranularity;
        }

        private void setMaxGranularity(RowGranularity granularity) {
            if (granularity.ordinal() > this.maxGranularity.ordinal()) {
                this.maxGranularity = granularity;
            }
        }

        private void add(Input<?> input) {
            topLevelInputs.add(input);
        }

        public CollectorExpression<?>[] docLevelExpressions() {
            return docLevelExpressions.toArray(new CollectorExpression<?>[docLevelExpressions.size()]);
        }

        public Set<CollectExpression<?>> collectExpressions() {
            return collectExpressions;
        }

        // note: some implementations using this method expect a copy of the data
        // so that if the context / backing list is manipulated the output from this doesn't change
        public Input<?>[] topLevelInputs() {
            return topLevelInputs.toArray(new Input<?>[topLevelInputs.size()]);
        }

        public AggregationContext[] aggregations() {
            return aggregations.toArray(new AggregationContext[aggregations.size()]);
        }
    }

    protected final ReferenceResolver referenceResolver;
    protected final Functions functions;
    protected final RowGranularity rowGranularity;


    public ImplementationSymbolVisitor(ReferenceResolver referenceResolver, Functions functions, RowGranularity rowGranularity) {
        this.referenceResolver = referenceResolver;
        this.functions = functions;
        this.rowGranularity = rowGranularity;
    }


    public Context process(CollectNode node) {
        Context context = new Context();
        if (node.toCollect() != null) {
            for (Symbol symbol : node.toCollect()) {
                context.add(process(symbol, context));
            }
        }
        return context;
    }

    public Context process(Symbol... symbols) {
        Context context = new Context();
        for (Symbol symbol : symbols) {
            context.add(process(symbol, context));
        }
        return context;
    }

    public Context process(List<Symbol> symbols) {
        Context context = new Context();
        for (Symbol symbol : symbols) {
            context.add(process(symbol, context));
        }
        return context;
    }

    @Override
    public Input<?> visitFunction(Function function, Context context) {
        final FunctionImplementation functionImplementation = functions.get(function.info().ident());
        if (functionImplementation != null && functionImplementation instanceof Scalar<?>) {

            List<Symbol> arguments = function.arguments();
            Input[] argumentInputs = new Input[arguments.size()];
            int i = 0;
            for (Symbol argument : function.arguments()) {
                argumentInputs[i++] = process(argument, context);
            }
            return new FunctionExpression<>((Scalar<?>) functionImplementation, argumentInputs);
        } else {
            throw new CrateException("Unknown Function");
        }
    }

    @Override
    public Input<?> visitInputColumn(InputColumn inputColumn, Context context) {
        InputCollectExpression<?> input = new InputCollectExpression<>(inputColumn.index());
        context.collectExpressions.add(input);
        return input;
    }

    @Override
    public Input<?> visitReference(Reference symbol, Context context) {
        Input<?> result;

        if (symbol.info().granularity().ordinal() <= rowGranularity.ordinal()) {
            if (symbol.info().granularity() == RowGranularity.DOC) {
                CollectorExpression<?> docLevelExpression = DocLevelExpressions.getExpression(symbol.info());<commit_msg>throw more accurate exception if function implmentation is not found<commit_after>/*
 * Licensed to CRATE Technology GmbH ("Crate") under one or more contributor
 * license agreements.  See the NOTICE file distributed with this work for
 * additional information regarding copyright ownership.  Crate licenses
 * this file to you under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.  You may
 * obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
 * License for the specific language governing permissions and limitations
 * under the License.
 *
 * However, if you have executed another commercial license agreement
 * with Crate these terms will supersede the license and you may use the
 * software solely pursuant to the terms of the relevant commercial agreement.
 */

package io.crate.operator.operations;

import io.crate.metadata.*;
import io.crate.operator.Input;
import io.crate.operator.InputCollectExpression;
import io.crate.operator.aggregation.AggregationFunction;
import io.crate.operator.aggregation.CollectExpression;
import io.crate.operator.aggregation.FunctionExpression;
import io.crate.operator.reference.doc.CollectorExpression;
import io.crate.operator.reference.doc.DocLevelExpressions;
import io.crate.planner.RowGranularity;
import io.crate.planner.node.CollectNode;
import io.crate.planner.symbol.*;
import org.cratedb.sql.CrateException;

import java.util.*;

/**
 * convert Symbols into Inputs for evaluation and CollectExpressions
 * that might be treated in a special way
 */
public class ImplementationSymbolVisitor extends SymbolVisitor<ImplementationSymbolVisitor.Context, Input<?>> {
    public static class Context {
        protected Set<CollectExpression<?>> collectExpressions = new LinkedHashSet<>(); // to keep insertion order
        protected List<Input<?>> topLevelInputs = new ArrayList<>();
        protected List<CollectorExpression<?>> docLevelExpressions = new ArrayList<>();
        protected RowGranularity maxGranularity = RowGranularity.CLUSTER;
        protected List<AggregationContext> aggregations = new ArrayList<>();

        public RowGranularity maxGranularity() {
            return maxGranularity;
        }

        private void setMaxGranularity(RowGranularity granularity) {
            if (granularity.ordinal() > this.maxGranularity.ordinal()) {
                this.maxGranularity = granularity;
            }
        }

        private void add(Input<?> input) {
            topLevelInputs.add(input);
        }

        public CollectorExpression<?>[] docLevelExpressions() {
            return docLevelExpressions.toArray(new CollectorExpression<?>[docLevelExpressions.size()]);
        }

        public Set<CollectExpression<?>> collectExpressions() {
            return collectExpressions;
        }

        // note: some implementations using this method expect a copy of the data
        // so that if the context / backing list is manipulated the output from this doesn't change
        public Input<?>[] topLevelInputs() {
            return topLevelInputs.toArray(new Input<?>[topLevelInputs.size()]);
        }

        public AggregationContext[] aggregations() {
            return aggregations.toArray(new AggregationContext[aggregations.size()]);
        }
    }

    protected final ReferenceResolver referenceResolver;
    protected final Functions functions;
    protected final RowGranularity rowGranularity;


    public ImplementationSymbolVisitor(ReferenceResolver referenceResolver, Functions functions, RowGranularity rowGranularity) {
        this.referenceResolver = referenceResolver;
        this.functions = functions;
        this.rowGranularity = rowGranularity;
    }


    public Context process(CollectNode node) {
        Context context = new Context();
        if (node.toCollect() != null) {
            for (Symbol symbol : node.toCollect()) {
                context.add(process(symbol, context));
            }
        }
        return context;
    }

    public Context process(Symbol... symbols) {
        Context context = new Context();
        for (Symbol symbol : symbols) {
            context.add(process(symbol, context));
        }
        return context;
    }

    public Context process(List<Symbol> symbols) {
        Context context = new Context();
        for (Symbol symbol : symbols) {
            context.add(process(symbol, context));
        }
        return context;
    }

    @Override
    public Input<?> visitFunction(Function function, Context context) {
        final FunctionImplementation functionImplementation = functions.get(function.info().ident());
        if (functionImplementation != null && functionImplementation instanceof Scalar<?>) {

            List<Symbol> arguments = function.arguments();
            Input[] argumentInputs = new Input[arguments.size()];
            int i = 0;
            for (Symbol argument : function.arguments()) {
                argumentInputs[i++] = process(argument, context);
            }
            return new FunctionExpression<>((Scalar<?>) functionImplementation, argumentInputs);
        } else {
            throw new IllegalArgumentException(
                    String.format("Cannot find implementation for function %s", function));
        }
    }

    @Override
    public Input<?> visitInputColumn(InputColumn inputColumn, Context context) {
        InputCollectExpression<?> input = new InputCollectExpression<>(inputColumn.index());
        context.collectExpressions.add(input);
        return input;
    }

    @Override
    public Input<?> visitReference(Reference symbol, Context context) {
        Input<?> result;

        if (symbol.info().granularity().ordinal() <= rowGranularity.ordinal()) {
            if (symbol.info().granularity() == RowGranularity.DOC) {
                CollectorExpression<?> docLevelExpression = DocLevelExpressions.getExpression(symbol.info());
{'fraction_non_alphanumeric': 0.09102741397543318, 'fraction_numerical': 0.0008134710811030668, 'mean_word_length': 3.1380006731740155, 'pattern_counts': {'":': 0, '<': 69, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2896083', 'n_tokens_mistral': 2999, 'n_tokens_neox': 2964, 'n_words': 908}