<commit_before>/**
 * END USER LICENSE AGREEMENT (“EULA”)
 *
 * READ THIS AGREEMENT CAREFULLY (date: 9/13/2011):
 * http://www.akiban.com/licensing/20110913
 *
 * BY INSTALLING OR USING ALL OR ANY PORTION OF THE SOFTWARE, YOU ARE ACCEPTING
 * ALL OF THE TERMS AND CONDITIONS OF THIS AGREEMENT. YOU AGREE THAT THIS
 * AGREEMENT IS ENFORCEABLE LIKE ANY WRITTEN AGREEMENT SIGNED BY YOU.
 *
 * IF YOU HAVE PAID A LICENSE FEE FOR USE OF THE SOFTWARE AND DO NOT AGREE TO
 * THESE TERMS, YOU MAY RETURN THE SOFTWARE FOR A FULL REFUND PROVIDED YOU (A) DO
 * NOT USE THE SOFTWARE AND (B) RETURN THE SOFTWARE WITHIN THIRTY (30) DAYS OF
 * YOUR INITIAL PURCHASE.
 *
 * IF YOU WISH TO USE THE SOFTWARE AS AN EMPLOYEE, CONTRACTOR, OR AGENT OF A
 * CORPORATION, PARTNERSHIP OR SIMILAR ENTITY, THEN YOU MUST BE AUTHORIZED TO SIGN
 * FOR AND BIND THE ENTITY IN ORDER TO ACCEPT THE TERMS OF THIS AGREEMENT. THE
 * LICENSES GRANTED UNDER THIS AGREEMENT ARE EXPRESSLY CONDITIONED UPON ACCEPTANCE
 * BY SUCH AUTHORIZED PERSONNEL.
 *
 * IF YOU HAVE ENTERED INTO A SEPARATE WRITTEN LICENSE AGREEMENT WITH AKIBAN FOR
 * USE OF THE SOFTWARE, THE TERMS AND CONDITIONS OF SUCH OTHER AGREEMENT SHALL
 * PREVAIL OVER ANY CONFLICTING TERMS OR CONDITIONS IN THIS AGREEMENT.
 */

package com.akiban.sql.optimizer.rule;

import static com.akiban.sql.optimizer.rule.OldExpressionAssembler.*;

import com.akiban.ais.model.IndexColumn;
import com.akiban.qp.operator.API.InputPreservationOption;
import com.akiban.qp.operator.API.JoinType;
import com.akiban.server.collation.AkCollator;
import com.akiban.server.expression.std.FieldExpression;
import com.akiban.server.expression.subquery.ResultSetSubqueryExpression;
import com.akiban.server.expression.subquery.ScalarSubqueryExpression;
import com.akiban.server.types3.TInstance;
import com.akiban.server.types3.Types3Switch;
import com.akiban.server.types3.pvalue.PUnderlying;
import com.akiban.server.types3.pvalue.PValueSources;
import com.akiban.server.types3.texpressions.AnySubqueryTExpression;
import com.akiban.server.types3.texpressions.TNullExpression;
import com.akiban.server.types3.texpressions.TPreparedExpression;
import com.akiban.server.types3.texpressions.TPreparedField;
import com.akiban.server.types3.texpressions.TPreparedLiteral;
import com.akiban.sql.optimizer.*;
import com.akiban.sql.optimizer.plan.*;
import com.akiban.sql.optimizer.plan.ExpressionsSource.DistinctState;
import com.akiban.sql.optimizer.plan.PhysicalSelect.PhysicalResultColumn;
import com.akiban.sql.optimizer.plan.ResultSet.ResultField;
import com.akiban.sql.optimizer.plan.Sort.OrderByExpression;
import com.akiban.sql.optimizer.plan.UpdateStatement.UpdateColumn;

import com.akiban.sql.optimizer.rule.range.ColumnRanges;
import com.akiban.sql.optimizer.rule.range.RangeSegment;
import com.akiban.sql.types.DataTypeDescriptor;
import com.akiban.sql.parser.ParameterNode;

import com.akiban.server.error.AkibanInternalException;
import com.akiban.server.error.UnsupportedSQLException;

import com.akiban.server.expression.Expression;
import com.akiban.server.expression.std.Expressions;
import com.akiban.server.expression.std.LiteralExpression;
import com.akiban.server.expression.subquery.AnySubqueryExpression;
import com.akiban.server.expression.subquery.ExistsSubqueryExpression;
import com.akiban.server.types.AkType;

import com.akiban.qp.exec.UpdatePlannable;
import com.akiban.qp.operator.API;
import com.akiban.qp.operator.IndexScanSelector;
import com.akiban.qp.operator.Operator;
import com.akiban.qp.operator.UpdateFunction;
import com.akiban.qp.row.BindableRow;
import com.akiban.qp.row.Row;
import com.akiban.qp.rowtype.*;

import com.akiban.qp.expression.IndexBound;
import com.akiban.qp.expression.IndexKeyRange;
import com.akiban.qp.expression.RowBasedUnboundExpressions;
import com.akiban.qp.expression.UnboundExpressions;

import com.akiban.ais.model.Column;
import com.akiban.ais.model.Index;
import com.akiban.ais.model.GroupTable;

import com.akiban.server.api.dml.ColumnSelector;
import com.akiban.server.types3.texpressions.ExistsSubqueryTExpression;
import com.akiban.server.types3.texpressions.ScalarSubqueryTExpression;
import com.akiban.util.tap.PointTap;
import com.akiban.util.tap.Tap;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.*;

public class OperatorAssembler extends BaseRule
{
    private static final Logger logger = LoggerFactory.getLogger(OperatorAssembler.class);
    private static final PointTap SELECT_COUNT = Tap.createCount("sql: select");
    private static final PointTap INSERT_COUNT = Tap.createCount("sql: insert");
    private static final PointTap UPDATE_COUNT = Tap.createCount("sql: update");
    private static final PointTap DELETE_COUNT = Tap.createCount("sql: delete");

    public static final int INSERTION_SORT_MAX_LIMIT = 100;

    private final boolean usePValues;

    @SuppressWarnings("unused") // used by reflection in RulesTestHelper
    public OperatorAssembler() {
        this.usePValues = Types3Switch.ON;
    }
    
    public OperatorAssembler(boolean usePValues) {
        this.usePValues = usePValues;
    }

    @Override
    protected Logger getLogger() {
        return logger;
    }

    @Override
    public void apply(PlanContext plan) {
        new Assembler(plan, usePValues).apply();
    }

    interface PartialAssembler<T> extends SubqueryOperatorAssembler<T> {
        List<T> assembleExpressions(List<ExpressionNode> expressions,
                                    ColumnExpressionToIndex fieldOffsets);
        List<T> assembleExpressionsA(List<? extends AnnotatedExpression> expressions,
                                     ColumnExpressionToIndex fieldOffsets);
        T assembleExpression(ExpressionNode expr, ColumnExpressionToIndex fieldOffsets);
        void assembleExpressionInto(ExpressionNode expr, ColumnExpressionToIndex fieldOffsets, T[] arr, int i);
        Operator assembleAggregates(Operator inputOperator, RowType inputRowType, int inputsIndex, List<String> names);

        T field(RowType rowType, int position);
        
        RowType valuesRowType(ExpressionsSource expressionsSource);

        void fillNulls(Index index, T[] keys);
        List<T> assembleUpdates(UserTableRowType targetRowType, List<UpdateColumn> updateColumns,
                                         ColumnExpressionToIndex fieldOffsets);
        T[] createNulls(Index index, int nkeys);
        Operator ifEmptyNulls(Operator input, RowType rowType,
                              InputPreservationOption inputPreservation);

        API.Ordering createOrdering();
    }

    private static final PartialAssembler<?> NULL_PARTIAL_ASSEMBLER = new PartialAssembler<Object>() {
        @Override
        public List<Object> assembleExpressions(List<ExpressionNode> expressions,
                                                ColumnExpressionToIndex fieldOffsets) {
            return null;
        }

        @Override
        public void assembleExpressionInto(ExpressionNode expr, ColumnExpressionToIndex fieldOffsets, Object[] arr,
                                           int i)
        { // nothing; arr is null
        }

        @Override
        public void fillNulls(Index index, Object[] keys) {
            // nothing; keys are null
        }

        @Override
        public RowType valuesRowType(ExpressionsSource expressionsSource) {
            throw new UnsupportedOperationException(); // only the active assembler should be called for this
        }

        @Override
        public List<Object> assembleExpressionsA(List<? extends AnnotatedExpression> expressions,
                                                 ColumnExpressionToIndex fieldOffsets) {
            return null;
        }

        @Override
        public Object assembleExpression(ExpressionNode expr, ColumnExpressionToIndex fieldOffsets) {
            return null;
        }

        @Override
        public Object assembleSubqueryExpression(SubqueryExpression subqueryExpression) {
            return null;
        }

        @Override
        public Operator assembleAggregates(Operator inputOperator, RowType inputRowType, int inputsIndex,
                                           List<String> names) {
            throw new AssertionError();
        }

        @Override
        public List<Object> assembleUpdates(UserTableRowType targetRowType, List<UpdateColumn> updateColumns,
                                                ColumnExpressionToIndex fieldOffsets) {
            return null;
        }

        @Override
        public Object[] createNulls(Index index, int nkeys) {
            return null;
        }

        @Override
        public Operator ifEmptyNulls(Operator input, RowType rowType,
                                     InputPreservationOption inputPreservation) {
            return null;
        }

        @Override
        public API.Ordering createOrdering() {
            throw new UnsupportedOperationException();
        }

        @Override
        public Object field(RowType rowType, int position) {
            return null;
        }
    };

    @SuppressWarnings("unchecked")
    private static <T> PartialAssembler<T> nullAssembler() {
        return (PartialAssembler<T>) NULL_PARTIAL_ASSEMBLER;
    }

    static class Assembler {

        abstract class BasePartialAssembler<T> implements PartialAssembler<T> {

            protected BasePartialAssembler(ExpressionAssembler<T> expressionAssembler) {
                this.expressionAssembler = expressionAssembler;
            }

            private ExpressionAssembler<T> expressionAssembler;

            // Assemble a list of expressions from the given nodes.
            @Override
            public List<T> assembleExpressions(List<ExpressionNode> expressions,
                                                           ColumnExpressionToIndex fieldOffsets) {
                List<T> result = new ArrayList<T>(expressions.size());
                for (ExpressionNode expr : expressions) {
                    result.add(assembleExpression(expr, fieldOffsets));
                }
                return result;
            }

            @Override
            public void assembleExpressionInto(ExpressionNode expr, ColumnExpressionToIndex fieldOffsets, T[] arr,
                                               int i) {
                T result = assembleExpression(expr, fieldOffsets);
                arr[i] = result;
            }

            // Assemble a list of expressions from the given nodes.
            @Override
            public List<T> assembleExpressionsA(List<? extends AnnotatedExpression> expressions,
                                                            ColumnExpressionToIndex fieldOffsets) {
                List<T> result = new ArrayList<T>(expressions.size());
                for (AnnotatedExpression aexpr : expressions) {
                    result.add(assembleExpression(aexpr.getExpression(), fieldOffsets));
                }
                return result;
            }

            // Assemble an expression against the given row offsets.
            @Override
            public T assembleExpression(ExpressionNode expr, ColumnExpressionToIndex fieldOffsets) {
                ColumnExpressionContext context = getColumnExpressionContext(fieldOffsets);
                return expressionAssembler.assembleExpression(expr, context, this);
            }

            // Assemble an aggregate operator
            @Override
            public Operator assembleAggregates(Operator inputOperator, RowType inputRowType, int inputsIndex,
                                               List<String> names) {
                return expressionAssembler.assembleAggregates(inputOperator, inputRowType, inputsIndex, names);
            }

            protected abstract T existsExpression(Operator operator, RowType outerRowType,
                                                  RowType innerRowType,
                                                  int bindingPosition);
            protected abstract T anyExpression(Operator operator, T innerExpression, RowType outerRowType,
                                               RowType innerRowType,
                                               int bindingPosition);
            protected abstract T scalarSubqueryExpression(Operator operator, T innerExpression,
                                                          RowType outerRowType,
                                                          RowType innerRowType,
                                                          int bindingPosition);
            protected abstract T resultSetSubqueryExpression(Operator operator, RowType outerRowType,
                                                             RowType innerRowType,
                                                             int bindingPosition);
            protected abstract T nullExpression(RowType rowType, int i);


            protected List<? extends T> createNulls(RowType rowType) {
                int nfields = rowType.nFields();
                List<T> result = new ArrayList<T>(nfields);
                for (int i = 0; i < nfields; ++i)
                    result.add(nullExpression(rowType, i));
                return result;
            }

            @Override
            public T assembleSubqueryExpression(SubqueryExpression sexpr) {
                ColumnExpressionToIndex fieldOffsets = columnBoundRows.current;
                RowType outerRowType = null;
                if (fieldOffsets != null)
                    outerRowType = fieldOffsets.getRowType();
                pushBoundRow(fieldOffsets);
                PlanNode subquery = sexpr.getSubquery().getQuery();
                ExpressionNode expression = null;
                boolean fieldExpression = false;
                if ((sexpr instanceof AnyCondition) ||
                        (sexpr instanceof SubqueryValueExpression)) {
                    if (subquery instanceof ResultSet)
                        subquery = ((ResultSet)subquery).getInput();
                    if (subquery instanceof Project) {
                        Project project = (Project)subquery;
                        subquery = project.getInput();
                        expression = project.getFields().get(0);
                    }
                    else {
                        fieldExpression = true;
                    }
                }
                RowStream stream = assembleQuery(subquery);
                T innerExpression = null;
                if (fieldExpression)
                    innerExpression = field(stream.rowType, 0);
                else if (expression != null)
                    innerExpression = assembleExpression(expression, stream.fieldOffsets);
                T result = assembleSubqueryExpression(sexpr,
                        stream.operator,
                        innerExpression,
                        outerRowType,
                        stream.rowType,
                        currentBindingPosition());
                popBoundRow();
                columnBoundRows.current = fieldOffsets;
                return result;
            }

            private T assembleSubqueryExpression(SubqueryExpression sexpr,
                                                 Operator operator,
                                                 T innerExpression,
                                                 RowType outerRowType,
                                                 RowType innerRowType,
                                                 int bindingPosition) {
                if (sexpr instanceof ExistsCondition)
                    return existsExpression(operator, outerRowType,
                            innerRowType, bindingPosition);
                else if (sexpr instanceof AnyCondition)
                    return anyExpression(operator, innerExpression,
                            outerRowType, innerRowType, bindingPosition);
                else if (sexpr instanceof SubqueryValueExpression)
                    return scalarSubqueryExpression(operator, innerExpression,
                                                    outerRowType, innerRowType,
                                                    bindingPosition);
                else if (sexpr instanceof SubqueryResultSetExpression)
                    return resultSetSubqueryExpression(operator, outerRowType,
                                                       innerRowType, bindingPosition);
                else
                    throw new UnsupportedSQLException("Unknown subquery", sexpr.getSQLsource());
            }

            @Override
            public List<T> assembleUpdates(UserTableRowType targetRowType, List<UpdateColumn> updateColumns,
                                           ColumnExpressionToIndex fieldOffsets) {
                List<T> updates = assembleExpressionsA(updateColumns, fieldOffsets);
                // Have a list of expressions in the order specified.
                // Want a list as wide as the target row with Java nulls
                // for the gaps.
                // TODO: It might be simpler to have an update function
                // that knew about column offsets for ordered expressions.
                T[] row = array(targetRowType.nFields());
                for (int i = 0; i < updateColumns.size(); i++) {
                    UpdateColumn column = updateColumns.get(i);
                    row[column.getColumn().getPosition()] = updates.get(i);
                }
                updates = Arrays.asList(row);
                return updates;
            }

            @Override
            public T[] createNulls(Index index, int nkeys) {
                T[] arr = array(nkeys);
                fillNulls(index, arr);
                return arr;
            }

            protected abstract T[] array(int size);
        }

        private class OldPartialAssembler extends BasePartialAssembler<Expression> {

            private OldPartialAssembler(RulesContext context) {
                super(new OldExpressionAssembler(context));
            }

            @Override
            public void fillNulls(Index index, Expression[] keys) {
                for (int i = 0; i < keys.length; ++i) {
                    if (keys[i] == null)
                        keys[i] = LiteralExpression.forNull();
                }
            }

            @Override
            public RowType valuesRowType(ExpressionsSource expressionsSource) {
                AkType[] types = expressionsSource.getFieldTypes();
                return schema.newValuesType(types);
            }

            @Override
            protected Expression existsExpression(Operator operator, RowType outerRowType, RowType innerRowType,
                                                  int bindingPosition) {
                return new ExistsSubqueryExpression(operator, outerRowType, innerRowType, bindingPosition);
            }

            @Override
            protected Expression anyExpression(Operator operator, Expression innerExpression, RowType outerRowType,
                                               RowType innerRowType,
                                               int bindingPosition) {
                return new AnySubqueryExpression(operator, innerExpression, outerRowType, innerRowType,
                        bindingPosition);
            }

            @Override
            protected Expression scalarSubqueryExpression(Operator operator, Expression innerExpression,
                                                          RowType outerRowType, RowType innerRowType,
                                                          int bindingPosition) {
                return new ScalarSubqueryExpression(operator, innerExpression, outerRowType, innerRowType,
                        bindingPosition);
            }

            @Override
            protected Expression resultSetSubqueryExpression(Operator operator, RowType outerRowType,
                                                             RowType innerRowType,
                                                             int bindingPosition) {
                return new ResultSetSubqueryExpression(operator, outerRowType, innerRowType, bindingPosition);
            }

            @Override
            public Expression field(RowType rowType, int position) {
                return new FieldExpression(rowType, position);
            }

            @Override
            protected Expression[] array(int size) {
                return new Expression[size];
            }

            @Override
            public Operator ifEmptyNulls(Operator input, RowType rowType,
                                         InputPreservationOption inputPreservation) {
                return API.ifEmpty_Default(input, rowType, createNulls(rowType), null, inputPreservation);
            }

            @Override
            protected Expression nullExpression(RowType rowType, int i) {
                return LiteralExpression.forNull();
            }

            @Override
            public API.Ordering createOrdering() {
                return API.ordering(false);
            }
        }

        private class NewPartialAssembler extends BasePartialAssembler<TPreparedExpression> {
            private NewPartialAssembler(RulesContext context) {
                super(new NewExpressionAssembler(context));
            }

            @Override
            public void fillNulls(Index index, TPreparedExpression[] keys) {
                List<IndexColumn> indexColumns = index.getAllColumns();
                for (int i = 0; i < keys.length; ++i) {
                    if (keys[i] == null)
                        keys[i] = new TNullExpression(indexColumns.get(i).getColumn().tInstance());
                }
            }

            @Override
            public RowType valuesRowType(ExpressionsSource expressionsSource) {
                TInstance[] types = expressionsSource.getFieldTInstances();
                return schema.newValuesType(types);
            }

            @Override
            protected TPreparedExpression existsExpression(Operator operator, RowType outerRowType,
                                                           RowType innerRowType,
                                                           int bindingPosition) {
                return new ExistsSubqueryTExpression(operator, outerRowType, innerRowType, bindingPosition);
            }

            @Override
            protected TPreparedExpression anyExpression(Operator operator, TPreparedExpression innerExpression,
                                                        RowType outerRowType,
                                                        RowType innerRowType, int bindingPosition) {
                return new AnySubqueryTExpression(operator, innerExpression, outerRowType, innerRowType, bindingPosition);
            }

            @Override
            protected TPreparedExpression scalarSubqueryExpression(Operator operator, TPreparedExpression innerExpression,
                                                                   RowType outerRowType, RowType innerRowType,
                                                                   int bindingPosition) {
                return new ScalarSubqueryTExpression(operator, innerExpression, outerRowType, innerRowType, bindingPosition);
            }

            @Override
            protected TPreparedExpression resultSetSubqueryExpression(Operator operator, RowType outerRowType,
                                                                      RowType innerRowType, int bindingPosition) {
                throw new UnsupportedOperationException(); // TODO
            }

            @Override
            public TPreparedExpression field(RowType rowType, int position) {
                return new TPreparedField(rowType.typeInstanceAt(position), position);
            }

            @Override
            protected TPreparedExpression[] array(int size) {
                return new TPreparedExpression[size];
            }

            @Override
            public Operator ifEmptyNulls(Operator input, RowType rowType,
                                         InputPreservationOption inputPreservation) {
                return API.ifEmpty_Default(input, rowType, null, createNulls(rowType), inputPreservation);
            }

            @Override
            protected TPreparedExpression nullExpression(RowType rowType, int i) {
                return new TNullExpression(rowType.typeInstanceAt(i));
            }

            @Override
            public API.Ordering createOrdering() {
                return API.ordering(true);
            }
        }

        private PlanContext planContext;
        private SchemaRulesContext rulesContext;
        private Schema schema;
        private boolean usePValues;
        private final PartialAssembler<Expression> oldPartialAssembler;
        private final PartialAssembler<TPreparedExpression> newPartialAssembler;
        private final PartialAssembler<?> partialAssembler;

        public Assembler(PlanContext planContext, boolean usePValues) {
            this.usePValues = usePValues;
            this.planContext = planContext;
            rulesContext = (SchemaRulesContext)planContext.getRulesContext();
            schema = rulesContext.getSchema();
            if (usePValues) {
                newPartialAssembler = new NewPartialAssembler(rulesContext);
                oldPartialAssembler = nullAssembler();
                partialAssembler = newPartialAssembler;
            }
            else {
                newPartialAssembler = nullAssembler();
                oldPartialAssembler = new OldPartialAssembler(rulesContext);
                partialAssembler = oldPartialAssembler;
            }
            computeBindingsOffsets();
        }

        public void apply() {
            planContext.setPlan(assembleStatement((BaseStatement)planContext.getPlan()));
        }
        
        protected BasePlannable assembleStatement(BaseStatement plan) {
            if (plan instanceof SelectQuery) {
                SELECT_COUNT.hit();
                return selectQuery((SelectQuery)plan);
            } else if (plan instanceof InsertStatement) {
                INSERT_COUNT.hit();
                return insertStatement((InsertStatement)plan);
            } else if (plan instanceof UpdateStatement) {
                UPDATE_COUNT.hit();
                return updateStatement((UpdateStatement)plan);
            } else if (plan instanceof DeleteStatement) {
                DELETE_COUNT.hit();
                return deleteStatement((DeleteStatement)plan);
            } else
                throw new UnsupportedSQLException("Cannot assemble plan: " + plan, null);
        }

        protected PhysicalSelect selectQuery(SelectQuery selectQuery) {
            PlanNode planQuery = selectQuery.getQuery();
            RowStream stream = assembleQuery(planQuery);
            List<PhysicalResultColumn> resultColumns;
            if (planQuery instanceof ResultSet) {
                List<ResultField> results = ((ResultSet)planQuery).getFields();
                resultColumns = getResultColumns(results);
            }
            else {
                // VALUES results in column1, column2, ...
                resultColumns = getResultColumns(stream.rowType.nFields());
            }
            return new PhysicalSelect(stream.operator, stream.rowType, resultColumns, 
                                      getParameterTypes());
        }

        protected PhysicalUpdate insertStatement(InsertStatement insertStatement) {
            PlanNode planQuery = insertStatement.getQuery();
            List<ExpressionNode> projectFields = null;
            if (planQuery instanceof Project) {
                Project project = (Project)planQuery;
                projectFields = project.getFields();
                planQuery = project.getInput();
            }
            RowStream stream = assembleQuery(planQuery);
            UserTableRowType targetRowType = 
                tableRowType(insertStatement.getTargetTable());
            List<Expression> inserts = null;
            List<TPreparedExpression> insertsP = null;
            if (projectFields != null) {
                // In the common case, we can project into a wider row
                // of the correct type directly.
                insertsP = newPartialAssembler.assembleExpressions(projectFields, stream.fieldOffsets);
                inserts = oldPartialAssembler.assembleExpressions(projectFields, stream.fieldOffsets);
            }
            else {
                // VALUES just needs each field, which will get rearranged below.
                int nfields = stream.rowType.nFields();
                if (usePValues) {
                    insertsP = new ArrayList<TPreparedExpression>(nfields);
                    for (int i = 0; i < nfields; ++i) {
                        insertsP.add(new TPreparedField(stream.rowType.typeInstanceAt(i), i));
                    }
                }
                else {
                    inserts = new ArrayList<Expression>(nfields);
                    for (int i = 0; i < nfields; i++) {
                        inserts.add(Expressions.field(stream.rowType, i));
                    }
                }
            }
            // Have a list of expressions in the order specified.
            // Want a list as wide as the target row with NULL
            // literals for the gaps.
            // TODO: That doesn't seem right. How are explicit NULLs
            // to be distinguished from the column's default value?
            if (inserts != null) {
                Expression[] row = new Expression[targetRowType.nFields()];
                Arrays.fill(row, LiteralExpression.forNull());
                int ncols = inserts.size();
                for (int i = 0; i < ncols; i++) {
                    Column column = insertStatement.getTargetColumns().get(i);
                    row[column.getPosition()] = inserts.get(i);
                }
                inserts = Arrays.asList(row);
            }
            else {
                TPreparedExpression[] row = new TPreparedExpression[targetRowType.nFields()];
                int ncols = insertsP.size();
                for (int i = 0; i < ncols; i++) {
                    Column column = insertStatement.getTargetColumns().get(i);
                    row[column.getPosition()] = insertsP.get(i);
                }
                for (int i = 0, len = targetRowType.nFields(); i < len; ++i) {
                    if (row[i] == null) {
                        TInstance tinst = targetRowType.typeInstanceAt(i);
                        PUnderlying underlying = tinst.typeClass().underlyingType();
                        row[i] = new TPreparedLiteral(tinst, PValueSources.getNullSource(underlying));
                    }
                }
            }
            stream.operator = API.project_Table(stream.operator, stream.rowType,
                                                targetRowType, inserts, insertsP);
            UpdatePlannable plan = API.insert_Default(stream.operator, usePValues);
            return new PhysicalUpdate(plan, getParameterTypes());
        }

        protected PhysicalUpdate updateStatement(UpdateStatement updateStatement) {
            RowStream stream = assembleQuery(updateStatement.getQuery());
            UserTableRowType targetRowType = 
                tableRowType(updateStatement.getTargetTable());
            assert (stream.rowType == targetRowType);
            List<UpdateColumn> updateColumns = updateStatement.getUpdateColumns();
            List<Expression> updates = oldPartialAssembler.assembleUpdates(targetRowType, updateColumns,
                    stream.fieldOffsets);<commit_msg>forgot to write the insertsP having adjusted it<commit_after>/**
 * END USER LICENSE AGREEMENT (“EULA”)
 *
 * READ THIS AGREEMENT CAREFULLY (date: 9/13/2011):
 * http://www.akiban.com/licensing/20110913
 *
 * BY INSTALLING OR USING ALL OR ANY PORTION OF THE SOFTWARE, YOU ARE ACCEPTING
 * ALL OF THE TERMS AND CONDITIONS OF THIS AGREEMENT. YOU AGREE THAT THIS
 * AGREEMENT IS ENFORCEABLE LIKE ANY WRITTEN AGREEMENT SIGNED BY YOU.
 *
 * IF YOU HAVE PAID A LICENSE FEE FOR USE OF THE SOFTWARE AND DO NOT AGREE TO
 * THESE TERMS, YOU MAY RETURN THE SOFTWARE FOR A FULL REFUND PROVIDED YOU (A) DO
 * NOT USE THE SOFTWARE AND (B) RETURN THE SOFTWARE WITHIN THIRTY (30) DAYS OF
 * YOUR INITIAL PURCHASE.
 *
 * IF YOU WISH TO USE THE SOFTWARE AS AN EMPLOYEE, CONTRACTOR, OR AGENT OF A
 * CORPORATION, PARTNERSHIP OR SIMILAR ENTITY, THEN YOU MUST BE AUTHORIZED TO SIGN
 * FOR AND BIND THE ENTITY IN ORDER TO ACCEPT THE TERMS OF THIS AGREEMENT. THE
 * LICENSES GRANTED UNDER THIS AGREEMENT ARE EXPRESSLY CONDITIONED UPON ACCEPTANCE
 * BY SUCH AUTHORIZED PERSONNEL.
 *
 * IF YOU HAVE ENTERED INTO A SEPARATE WRITTEN LICENSE AGREEMENT WITH AKIBAN FOR
 * USE OF THE SOFTWARE, THE TERMS AND CONDITIONS OF SUCH OTHER AGREEMENT SHALL
 * PREVAIL OVER ANY CONFLICTING TERMS OR CONDITIONS IN THIS AGREEMENT.
 */

package com.akiban.sql.optimizer.rule;

import static com.akiban.sql.optimizer.rule.OldExpressionAssembler.*;

import com.akiban.ais.model.IndexColumn;
import com.akiban.qp.operator.API.InputPreservationOption;
import com.akiban.qp.operator.API.JoinType;
import com.akiban.server.collation.AkCollator;
import com.akiban.server.expression.std.FieldExpression;
import com.akiban.server.expression.subquery.ResultSetSubqueryExpression;
import com.akiban.server.expression.subquery.ScalarSubqueryExpression;
import com.akiban.server.types3.TInstance;
import com.akiban.server.types3.Types3Switch;
import com.akiban.server.types3.pvalue.PUnderlying;
import com.akiban.server.types3.pvalue.PValueSources;
import com.akiban.server.types3.texpressions.AnySubqueryTExpression;
import com.akiban.server.types3.texpressions.TNullExpression;
import com.akiban.server.types3.texpressions.TPreparedExpression;
import com.akiban.server.types3.texpressions.TPreparedField;
import com.akiban.server.types3.texpressions.TPreparedLiteral;
import com.akiban.sql.optimizer.*;
import com.akiban.sql.optimizer.plan.*;
import com.akiban.sql.optimizer.plan.ExpressionsSource.DistinctState;
import com.akiban.sql.optimizer.plan.PhysicalSelect.PhysicalResultColumn;
import com.akiban.sql.optimizer.plan.ResultSet.ResultField;
import com.akiban.sql.optimizer.plan.Sort.OrderByExpression;
import com.akiban.sql.optimizer.plan.UpdateStatement.UpdateColumn;

import com.akiban.sql.optimizer.rule.range.ColumnRanges;
import com.akiban.sql.optimizer.rule.range.RangeSegment;
import com.akiban.sql.types.DataTypeDescriptor;
import com.akiban.sql.parser.ParameterNode;

import com.akiban.server.error.AkibanInternalException;
import com.akiban.server.error.UnsupportedSQLException;

import com.akiban.server.expression.Expression;
import com.akiban.server.expression.std.Expressions;
import com.akiban.server.expression.std.LiteralExpression;
import com.akiban.server.expression.subquery.AnySubqueryExpression;
import com.akiban.server.expression.subquery.ExistsSubqueryExpression;
import com.akiban.server.types.AkType;

import com.akiban.qp.exec.UpdatePlannable;
import com.akiban.qp.operator.API;
import com.akiban.qp.operator.IndexScanSelector;
import com.akiban.qp.operator.Operator;
import com.akiban.qp.operator.UpdateFunction;
import com.akiban.qp.row.BindableRow;
import com.akiban.qp.row.Row;
import com.akiban.qp.rowtype.*;

import com.akiban.qp.expression.IndexBound;
import com.akiban.qp.expression.IndexKeyRange;
import com.akiban.qp.expression.RowBasedUnboundExpressions;
import com.akiban.qp.expression.UnboundExpressions;

import com.akiban.ais.model.Column;
import com.akiban.ais.model.Index;
import com.akiban.ais.model.GroupTable;

import com.akiban.server.api.dml.ColumnSelector;
import com.akiban.server.types3.texpressions.ExistsSubqueryTExpression;
import com.akiban.server.types3.texpressions.ScalarSubqueryTExpression;
import com.akiban.util.tap.PointTap;
import com.akiban.util.tap.Tap;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.*;

public class OperatorAssembler extends BaseRule
{
    private static final Logger logger = LoggerFactory.getLogger(OperatorAssembler.class);
    private static final PointTap SELECT_COUNT = Tap.createCount("sql: select");
    private static final PointTap INSERT_COUNT = Tap.createCount("sql: insert");
    private static final PointTap UPDATE_COUNT = Tap.createCount("sql: update");
    private static final PointTap DELETE_COUNT = Tap.createCount("sql: delete");

    public static final int INSERTION_SORT_MAX_LIMIT = 100;

    private final boolean usePValues;

    @SuppressWarnings("unused") // used by reflection in RulesTestHelper
    public OperatorAssembler() {
        this.usePValues = Types3Switch.ON;
    }
    
    public OperatorAssembler(boolean usePValues) {
        this.usePValues = usePValues;
    }

    @Override
    protected Logger getLogger() {
        return logger;
    }

    @Override
    public void apply(PlanContext plan) {
        new Assembler(plan, usePValues).apply();
    }

    interface PartialAssembler<T> extends SubqueryOperatorAssembler<T> {
        List<T> assembleExpressions(List<ExpressionNode> expressions,
                                    ColumnExpressionToIndex fieldOffsets);
        List<T> assembleExpressionsA(List<? extends AnnotatedExpression> expressions,
                                     ColumnExpressionToIndex fieldOffsets);
        T assembleExpression(ExpressionNode expr, ColumnExpressionToIndex fieldOffsets);
        void assembleExpressionInto(ExpressionNode expr, ColumnExpressionToIndex fieldOffsets, T[] arr, int i);
        Operator assembleAggregates(Operator inputOperator, RowType inputRowType, int inputsIndex, List<String> names);

        T field(RowType rowType, int position);
        
        RowType valuesRowType(ExpressionsSource expressionsSource);

        void fillNulls(Index index, T[] keys);
        List<T> assembleUpdates(UserTableRowType targetRowType, List<UpdateColumn> updateColumns,
                                         ColumnExpressionToIndex fieldOffsets);
        T[] createNulls(Index index, int nkeys);
        Operator ifEmptyNulls(Operator input, RowType rowType,
                              InputPreservationOption inputPreservation);

        API.Ordering createOrdering();
    }

    private static final PartialAssembler<?> NULL_PARTIAL_ASSEMBLER = new PartialAssembler<Object>() {
        @Override
        public List<Object> assembleExpressions(List<ExpressionNode> expressions,
                                                ColumnExpressionToIndex fieldOffsets) {
            return null;
        }

        @Override
        public void assembleExpressionInto(ExpressionNode expr, ColumnExpressionToIndex fieldOffsets, Object[] arr,
                                           int i)
        { // nothing; arr is null
        }

        @Override
        public void fillNulls(Index index, Object[] keys) {
            // nothing; keys are null
        }

        @Override
        public RowType valuesRowType(ExpressionsSource expressionsSource) {
            throw new UnsupportedOperationException(); // only the active assembler should be called for this
        }

        @Override
        public List<Object> assembleExpressionsA(List<? extends AnnotatedExpression> expressions,
                                                 ColumnExpressionToIndex fieldOffsets) {
            return null;
        }

        @Override
        public Object assembleExpression(ExpressionNode expr, ColumnExpressionToIndex fieldOffsets) {
            return null;
        }

        @Override
        public Object assembleSubqueryExpression(SubqueryExpression subqueryExpression) {
            return null;
        }

        @Override
        public Operator assembleAggregates(Operator inputOperator, RowType inputRowType, int inputsIndex,
                                           List<String> names) {
            throw new AssertionError();
        }

        @Override
        public List<Object> assembleUpdates(UserTableRowType targetRowType, List<UpdateColumn> updateColumns,
                                                ColumnExpressionToIndex fieldOffsets) {
            return null;
        }

        @Override
        public Object[] createNulls(Index index, int nkeys) {
            return null;
        }

        @Override
        public Operator ifEmptyNulls(Operator input, RowType rowType,
                                     InputPreservationOption inputPreservation) {
            return null;
        }

        @Override
        public API.Ordering createOrdering() {
            throw new UnsupportedOperationException();
        }

        @Override
        public Object field(RowType rowType, int position) {
            return null;
        }
    };

    @SuppressWarnings("unchecked")
    private static <T> PartialAssembler<T> nullAssembler() {
        return (PartialAssembler<T>) NULL_PARTIAL_ASSEMBLER;
    }

    static class Assembler {

        abstract class BasePartialAssembler<T> implements PartialAssembler<T> {

            protected BasePartialAssembler(ExpressionAssembler<T> expressionAssembler) {
                this.expressionAssembler = expressionAssembler;
            }

            private ExpressionAssembler<T> expressionAssembler;

            // Assemble a list of expressions from the given nodes.
            @Override
            public List<T> assembleExpressions(List<ExpressionNode> expressions,
                                                           ColumnExpressionToIndex fieldOffsets) {
                List<T> result = new ArrayList<T>(expressions.size());
                for (ExpressionNode expr : expressions) {
                    result.add(assembleExpression(expr, fieldOffsets));
                }
                return result;
            }

            @Override
            public void assembleExpressionInto(ExpressionNode expr, ColumnExpressionToIndex fieldOffsets, T[] arr,
                                               int i) {
                T result = assembleExpression(expr, fieldOffsets);
                arr[i] = result;
            }

            // Assemble a list of expressions from the given nodes.
            @Override
            public List<T> assembleExpressionsA(List<? extends AnnotatedExpression> expressions,
                                                            ColumnExpressionToIndex fieldOffsets) {
                List<T> result = new ArrayList<T>(expressions.size());
                for (AnnotatedExpression aexpr : expressions) {
                    result.add(assembleExpression(aexpr.getExpression(), fieldOffsets));
                }
                return result;
            }

            // Assemble an expression against the given row offsets.
            @Override
            public T assembleExpression(ExpressionNode expr, ColumnExpressionToIndex fieldOffsets) {
                ColumnExpressionContext context = getColumnExpressionContext(fieldOffsets);
                return expressionAssembler.assembleExpression(expr, context, this);
            }

            // Assemble an aggregate operator
            @Override
            public Operator assembleAggregates(Operator inputOperator, RowType inputRowType, int inputsIndex,
                                               List<String> names) {
                return expressionAssembler.assembleAggregates(inputOperator, inputRowType, inputsIndex, names);
            }

            protected abstract T existsExpression(Operator operator, RowType outerRowType,
                                                  RowType innerRowType,
                                                  int bindingPosition);
            protected abstract T anyExpression(Operator operator, T innerExpression, RowType outerRowType,
                                               RowType innerRowType,
                                               int bindingPosition);
            protected abstract T scalarSubqueryExpression(Operator operator, T innerExpression,
                                                          RowType outerRowType,
                                                          RowType innerRowType,
                                                          int bindingPosition);
            protected abstract T resultSetSubqueryExpression(Operator operator, RowType outerRowType,
                                                             RowType innerRowType,
                                                             int bindingPosition);
            protected abstract T nullExpression(RowType rowType, int i);


            protected List<? extends T> createNulls(RowType rowType) {
                int nfields = rowType.nFields();
                List<T> result = new ArrayList<T>(nfields);
                for (int i = 0; i < nfields; ++i)
                    result.add(nullExpression(rowType, i));
                return result;
            }

            @Override
            public T assembleSubqueryExpression(SubqueryExpression sexpr) {
                ColumnExpressionToIndex fieldOffsets = columnBoundRows.current;
                RowType outerRowType = null;
                if (fieldOffsets != null)
                    outerRowType = fieldOffsets.getRowType();
                pushBoundRow(fieldOffsets);
                PlanNode subquery = sexpr.getSubquery().getQuery();
                ExpressionNode expression = null;
                boolean fieldExpression = false;
                if ((sexpr instanceof AnyCondition) ||
                        (sexpr instanceof SubqueryValueExpression)) {
                    if (subquery instanceof ResultSet)
                        subquery = ((ResultSet)subquery).getInput();
                    if (subquery instanceof Project) {
                        Project project = (Project)subquery;
                        subquery = project.getInput();
                        expression = project.getFields().get(0);
                    }
                    else {
                        fieldExpression = true;
                    }
                }
                RowStream stream = assembleQuery(subquery);
                T innerExpression = null;
                if (fieldExpression)
                    innerExpression = field(stream.rowType, 0);
                else if (expression != null)
                    innerExpression = assembleExpression(expression, stream.fieldOffsets);
                T result = assembleSubqueryExpression(sexpr,
                        stream.operator,
                        innerExpression,
                        outerRowType,
                        stream.rowType,
                        currentBindingPosition());
                popBoundRow();
                columnBoundRows.current = fieldOffsets;
                return result;
            }

            private T assembleSubqueryExpression(SubqueryExpression sexpr,
                                                 Operator operator,
                                                 T innerExpression,
                                                 RowType outerRowType,
                                                 RowType innerRowType,
                                                 int bindingPosition) {
                if (sexpr instanceof ExistsCondition)
                    return existsExpression(operator, outerRowType,
                            innerRowType, bindingPosition);
                else if (sexpr instanceof AnyCondition)
                    return anyExpression(operator, innerExpression,
                            outerRowType, innerRowType, bindingPosition);
                else if (sexpr instanceof SubqueryValueExpression)
                    return scalarSubqueryExpression(operator, innerExpression,
                                                    outerRowType, innerRowType,
                                                    bindingPosition);
                else if (sexpr instanceof SubqueryResultSetExpression)
                    return resultSetSubqueryExpression(operator, outerRowType,
                                                       innerRowType, bindingPosition);
                else
                    throw new UnsupportedSQLException("Unknown subquery", sexpr.getSQLsource());
            }

            @Override
            public List<T> assembleUpdates(UserTableRowType targetRowType, List<UpdateColumn> updateColumns,
                                           ColumnExpressionToIndex fieldOffsets) {
                List<T> updates = assembleExpressionsA(updateColumns, fieldOffsets);
                // Have a list of expressions in the order specified.
                // Want a list as wide as the target row with Java nulls
                // for the gaps.
                // TODO: It might be simpler to have an update function
                // that knew about column offsets for ordered expressions.
                T[] row = array(targetRowType.nFields());
                for (int i = 0; i < updateColumns.size(); i++) {
                    UpdateColumn column = updateColumns.get(i);
                    row[column.getColumn().getPosition()] = updates.get(i);
                }
                updates = Arrays.asList(row);
                return updates;
            }

            @Override
            public T[] createNulls(Index index, int nkeys) {
                T[] arr = array(nkeys);
                fillNulls(index, arr);
                return arr;
            }

            protected abstract T[] array(int size);
        }

        private class OldPartialAssembler extends BasePartialAssembler<Expression> {

            private OldPartialAssembler(RulesContext context) {
                super(new OldExpressionAssembler(context));
            }

            @Override
            public void fillNulls(Index index, Expression[] keys) {
                for (int i = 0; i < keys.length; ++i) {
                    if (keys[i] == null)
                        keys[i] = LiteralExpression.forNull();
                }
            }

            @Override
            public RowType valuesRowType(ExpressionsSource expressionsSource) {
                AkType[] types = expressionsSource.getFieldTypes();
                return schema.newValuesType(types);
            }

            @Override
            protected Expression existsExpression(Operator operator, RowType outerRowType, RowType innerRowType,
                                                  int bindingPosition) {
                return new ExistsSubqueryExpression(operator, outerRowType, innerRowType, bindingPosition);
            }

            @Override
            protected Expression anyExpression(Operator operator, Expression innerExpression, RowType outerRowType,
                                               RowType innerRowType,
                                               int bindingPosition) {
                return new AnySubqueryExpression(operator, innerExpression, outerRowType, innerRowType,
                        bindingPosition);
            }

            @Override
            protected Expression scalarSubqueryExpression(Operator operator, Expression innerExpression,
                                                          RowType outerRowType, RowType innerRowType,
                                                          int bindingPosition) {
                return new ScalarSubqueryExpression(operator, innerExpression, outerRowType, innerRowType,
                        bindingPosition);
            }

            @Override
            protected Expression resultSetSubqueryExpression(Operator operator, RowType outerRowType,
                                                             RowType innerRowType,
                                                             int bindingPosition) {
                return new ResultSetSubqueryExpression(operator, outerRowType, innerRowType, bindingPosition);
            }

            @Override
            public Expression field(RowType rowType, int position) {
                return new FieldExpression(rowType, position);
            }

            @Override
            protected Expression[] array(int size) {
                return new Expression[size];
            }

            @Override
            public Operator ifEmptyNulls(Operator input, RowType rowType,
                                         InputPreservationOption inputPreservation) {
                return API.ifEmpty_Default(input, rowType, createNulls(rowType), null, inputPreservation);
            }

            @Override
            protected Expression nullExpression(RowType rowType, int i) {
                return LiteralExpression.forNull();
            }

            @Override
            public API.Ordering createOrdering() {
                return API.ordering(false);
            }
        }

        private class NewPartialAssembler extends BasePartialAssembler<TPreparedExpression> {
            private NewPartialAssembler(RulesContext context) {
                super(new NewExpressionAssembler(context));
            }

            @Override
            public void fillNulls(Index index, TPreparedExpression[] keys) {
                List<IndexColumn> indexColumns = index.getAllColumns();
                for (int i = 0; i < keys.length; ++i) {
                    if (keys[i] == null)
                        keys[i] = new TNullExpression(indexColumns.get(i).getColumn().tInstance());
                }
            }

            @Override
            public RowType valuesRowType(ExpressionsSource expressionsSource) {
                TInstance[] types = expressionsSource.getFieldTInstances();
                return schema.newValuesType(types);
            }

            @Override
            protected TPreparedExpression existsExpression(Operator operator, RowType outerRowType,
                                                           RowType innerRowType,
                                                           int bindingPosition) {
                return new ExistsSubqueryTExpression(operator, outerRowType, innerRowType, bindingPosition);
            }

            @Override
            protected TPreparedExpression anyExpression(Operator operator, TPreparedExpression innerExpression,
                                                        RowType outerRowType,
                                                        RowType innerRowType, int bindingPosition) {
                return new AnySubqueryTExpression(operator, innerExpression, outerRowType, innerRowType, bindingPosition);
            }

            @Override
            protected TPreparedExpression scalarSubqueryExpression(Operator operator, TPreparedExpression innerExpression,
                                                                   RowType outerRowType, RowType innerRowType,
                                                                   int bindingPosition) {
                return new ScalarSubqueryTExpression(operator, innerExpression, outerRowType, innerRowType, bindingPosition);
            }

            @Override
            protected TPreparedExpression resultSetSubqueryExpression(Operator operator, RowType outerRowType,
                                                                      RowType innerRowType, int bindingPosition) {
                throw new UnsupportedOperationException(); // TODO
            }

            @Override
            public TPreparedExpression field(RowType rowType, int position) {
                return new TPreparedField(rowType.typeInstanceAt(position), position);
            }

            @Override
            protected TPreparedExpression[] array(int size) {
                return new TPreparedExpression[size];
            }

            @Override
            public Operator ifEmptyNulls(Operator input, RowType rowType,
                                         InputPreservationOption inputPreservation) {
                return API.ifEmpty_Default(input, rowType, null, createNulls(rowType), inputPreservation);
            }

            @Override
            protected TPreparedExpression nullExpression(RowType rowType, int i) {
                return new TNullExpression(rowType.typeInstanceAt(i));
            }

            @Override
            public API.Ordering createOrdering() {
                return API.ordering(true);
            }
        }

        private PlanContext planContext;
        private SchemaRulesContext rulesContext;
        private Schema schema;
        private boolean usePValues;
        private final PartialAssembler<Expression> oldPartialAssembler;
        private final PartialAssembler<TPreparedExpression> newPartialAssembler;
        private final PartialAssembler<?> partialAssembler;

        public Assembler(PlanContext planContext, boolean usePValues) {
            this.usePValues = usePValues;
            this.planContext = planContext;
            rulesContext = (SchemaRulesContext)planContext.getRulesContext();
            schema = rulesContext.getSchema();
            if (usePValues) {
                newPartialAssembler = new NewPartialAssembler(rulesContext);
                oldPartialAssembler = nullAssembler();
                partialAssembler = newPartialAssembler;
            }
            else {
                newPartialAssembler = nullAssembler();
                oldPartialAssembler = new OldPartialAssembler(rulesContext);
                partialAssembler = oldPartialAssembler;
            }
            computeBindingsOffsets();
        }

        public void apply() {
            planContext.setPlan(assembleStatement((BaseStatement)planContext.getPlan()));
        }
        
        protected BasePlannable assembleStatement(BaseStatement plan) {
            if (plan instanceof SelectQuery) {
                SELECT_COUNT.hit();
                return selectQuery((SelectQuery)plan);
            } else if (plan instanceof InsertStatement) {
                INSERT_COUNT.hit();
                return insertStatement((InsertStatement)plan);
            } else if (plan instanceof UpdateStatement) {
                UPDATE_COUNT.hit();
                return updateStatement((UpdateStatement)plan);
            } else if (plan instanceof DeleteStatement) {
                DELETE_COUNT.hit();
                return deleteStatement((DeleteStatement)plan);
            } else
                throw new UnsupportedSQLException("Cannot assemble plan: " + plan, null);
        }

        protected PhysicalSelect selectQuery(SelectQuery selectQuery) {
            PlanNode planQuery = selectQuery.getQuery();
            RowStream stream = assembleQuery(planQuery);
            List<PhysicalResultColumn> resultColumns;
            if (planQuery instanceof ResultSet) {
                List<ResultField> results = ((ResultSet)planQuery).getFields();
                resultColumns = getResultColumns(results);
            }
            else {
                // VALUES results in column1, column2, ...
                resultColumns = getResultColumns(stream.rowType.nFields());
            }
            return new PhysicalSelect(stream.operator, stream.rowType, resultColumns, 
                                      getParameterTypes());
        }

        protected PhysicalUpdate insertStatement(InsertStatement insertStatement) {
            PlanNode planQuery = insertStatement.getQuery();
            List<ExpressionNode> projectFields = null;
            if (planQuery instanceof Project) {
                Project project = (Project)planQuery;
                projectFields = project.getFields();
                planQuery = project.getInput();
            }
            RowStream stream = assembleQuery(planQuery);
            UserTableRowType targetRowType = 
                tableRowType(insertStatement.getTargetTable());
            List<Expression> inserts = null;
            List<TPreparedExpression> insertsP = null;
            if (projectFields != null) {
                // In the common case, we can project into a wider row
                // of the correct type directly.
                insertsP = newPartialAssembler.assembleExpressions(projectFields, stream.fieldOffsets);
                inserts = oldPartialAssembler.assembleExpressions(projectFields, stream.fieldOffsets);
            }
            else {
                // VALUES just needs each field, which will get rearranged below.
                int nfields = stream.rowType.nFields();
                if (usePValues) {
                    insertsP = new ArrayList<TPreparedExpression>(nfields);
                    for (int i = 0; i < nfields; ++i) {
                        insertsP.add(new TPreparedField(stream.rowType.typeInstanceAt(i), i));
                    }
                }
                else {
                    inserts = new ArrayList<Expression>(nfields);
                    for (int i = 0; i < nfields; i++) {
                        inserts.add(Expressions.field(stream.rowType, i));
                    }
                }
            }
            // Have a list of expressions in the order specified.
            // Want a list as wide as the target row with NULL
            // literals for the gaps.
            // TODO: That doesn't seem right. How are explicit NULLs
            // to be distinguished from the column's default value?
            if (inserts != null) {
                Expression[] row = new Expression[targetRowType.nFields()];
                Arrays.fill(row, LiteralExpression.forNull());
                int ncols = inserts.size();
                for (int i = 0; i < ncols; i++) {
                    Column column = insertStatement.getTargetColumns().get(i);
                    row[column.getPosition()] = inserts.get(i);
                }
                inserts = Arrays.asList(row);
            }
            else {
                TPreparedExpression[] row = new TPreparedExpression[targetRowType.nFields()];
                int ncols = insertsP.size();
                for (int i = 0; i < ncols; i++) {
                    Column column = insertStatement.getTargetColumns().get(i);
                    row[column.getPosition()] = insertsP.get(i);
                }
                for (int i = 0, len = targetRowType.nFields(); i < len; ++i) {
                    if (row[i] == null) {
                        TInstance tinst = targetRowType.typeInstanceAt(i);
                        PUnderlying underlying = tinst.typeClass().underlyingType();
                        row[i] = new TPreparedLiteral(tinst, PValueSources.getNullSource(underlying));
                    }
                }
                insertsP = Arrays.asList(row);
            }
            stream.operator = API.project_Table(stream.operator, stream.rowType,
                                                targetRowType, inserts, insertsP);
            UpdatePlannable plan = API.insert_Default(stream.operator, usePValues);
            return new PhysicalUpdate(plan, getParameterTypes());
        }

        protected PhysicalUpdate updateStatement(UpdateStatement updateStatement) {
            RowStream stream = assembleQuery(updateStatement.getQuery());
            UserTableRowType targetRowType = 
                tableRowType(updateStatement.getTargetTable());
            assert (stream.rowType == targetRowType);
            List<UpdateColumn> updateColumns = updateStatement.getUpdateColumns();
            List<Expression> updates = oldPartialAssembler.assembleUpdates(targetRowType, updateColumns,
                    stream.fieldOffsets);
{'fraction_non_alphanumeric': 0.06822977725674091, 'fraction_numerical': 0.0015005861664712779, 'mean_word_length': 1.8025232171018049, 'pattern_counts': {'":': 0, '<': 131, '<?xml version=': 0, '>': 113, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 192, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1565157', 'n_tokens_mistral': 14920, 'n_tokens_neox': 14045, 'n_words': 3562}