<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.jackrabbit.spi.commons.query.sql;

import org.apache.jackrabbit.spi.commons.query.AndQueryNode;
import org.apache.jackrabbit.spi.commons.query.LocationStepQueryNode;
import org.apache.jackrabbit.spi.commons.query.NAryQueryNode;
import org.apache.jackrabbit.spi.commons.query.NodeTypeQueryNode;
import org.apache.jackrabbit.spi.commons.query.NotQueryNode;
import org.apache.jackrabbit.spi.commons.query.OrQueryNode;
import org.apache.jackrabbit.spi.commons.query.OrderQueryNode;
import org.apache.jackrabbit.spi.commons.query.PathQueryNode;
import org.apache.jackrabbit.spi.commons.query.QueryConstants;
import org.apache.jackrabbit.spi.commons.query.QueryNode;
import org.apache.jackrabbit.spi.commons.query.QueryRootNode;
import org.apache.jackrabbit.spi.commons.query.RelationQueryNode;
import org.apache.jackrabbit.spi.commons.query.TextsearchQueryNode;
import org.apache.jackrabbit.spi.commons.query.PropertyFunctionQueryNode;
import org.apache.jackrabbit.spi.commons.query.QueryNodeFactory;
import org.apache.jackrabbit.spi.commons.conversion.NameException;
import org.apache.jackrabbit.spi.commons.name.NameConstants;
import org.apache.jackrabbit.spi.commons.name.NameFactoryImpl;
import org.apache.jackrabbit.spi.commons.name.PathBuilder;
import org.apache.jackrabbit.spi.Name;
import org.apache.jackrabbit.spi.Path;
import org.apache.jackrabbit.spi.commons.conversion.MalformedPathException;
import org.apache.jackrabbit.spi.commons.conversion.NameResolver;
import org.apache.jackrabbit.util.ISO8601;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.commons.collections.map.ReferenceMap;

import javax.jcr.query.InvalidQueryException;
import javax.jcr.NamespaceException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Map;
import java.io.StringReader;

/**
 * Implements the query builder for the JCR SQL syntax.
 */
public class JCRSQLQueryBuilder implements JCRSQLParserVisitor {

    /**
     * logger instance for this class
     */
    private static final Logger log = LoggerFactory.getLogger(JCRSQLQueryBuilder.class);

    /**
     * DateFormat pattern for type
     * {@link org.apache.jackrabbit.spi.commons.query.QueryConstants#TYPE_DATE}.
     */
    private static final String DATE_PATTERN = "yyyy-MM-dd";

    /**
     * Map of reusable JCRSQL parser instances indexed by NamespaceResolver.
     */
    private static Map parsers = new ReferenceMap(ReferenceMap.WEAK, ReferenceMap.WEAK);

    /**
     * The root node of the sql query syntax tree
     */
    private final ASTQuery stmt;

    /**
     * The root query node
     */
    private QueryRootNode root;

    /**
     * To resolve QNames
     */
    private NameResolver resolver;

    /**
     * Query node to gather the constraints defined in the WHERE clause
     */
    private final AndQueryNode constraintNode;

    /**
     * The Name of the node type in the from clause.
     */
    private Name nodeTypeName;

    /**
     * List of PathQueryNode constraints that need to be merged
     */
    private final List pathConstraints = new ArrayList();

    /**
     * The query node factory.
     */
    private final QueryNodeFactory factory;

    /**
     * Creates a new <code>JCRSQLQueryBuilder</code>.
     *
     * @param statement the root node of the SQL syntax tree.
     * @param resolver  a namespace resolver to use for names in the
     *                  <code>statement</code>.
     * @param factory   the query node factory.
     */
    private JCRSQLQueryBuilder(ASTQuery statement,
                               NameResolver resolver,
                               QueryNodeFactory factory) {
        this.stmt = statement;
        this.resolver = resolver;
        this.factory = factory;
        this.constraintNode =  factory.createAndQueryNode(null);
    }

    /**
     * Creates a <code>QueryNode</code> tree from a SQL <code>statement</code>
     * using the passed query node <code>factory</code>.
     *
     * @param statement the SQL statement.
     * @param resolver  the namespace resolver to use.
     * @return the <code>QueryNode</code> tree.
     * @throws InvalidQueryException if <code>statement</code> is malformed.
     */
    public static QueryRootNode createQuery(String statement,
                                            NameResolver resolver,
                                            QueryNodeFactory factory)
            throws InvalidQueryException {
        try {
            // get parser
            JCRSQLParser parser;
            synchronized (parsers) {
                parser = (JCRSQLParser) parsers.get(resolver);
                if (parser == null) {
                    parser = new JCRSQLParser(new StringReader(statement));
                    parser.setNameResolver(resolver);
                    parsers.put(resolver, parser);
                }
            }

            JCRSQLQueryBuilder builder;
            // guard against concurrent use within same session
            synchronized (parser) {
                parser.ReInit(new StringReader(statement));
                builder = new JCRSQLQueryBuilder(parser.Query(), resolver, factory);
            }
            return builder.getRootNode();
        } catch (ParseException e) {
            throw new InvalidQueryException(e.getMessage());
        } catch (IllegalArgumentException e) {
            throw new InvalidQueryException(e.getMessage());
        } catch (Throwable t) {
            // javacc parser may also throw an error in some cases
            throw new InvalidQueryException(t.getMessage());
        }
    }

    /**
     * Creates a String representation of the query node tree in SQL syntax.
     *
     * @param root     the root of the query node tree.
     * @param resolver to resolve QNames.
     * @return a String representation of the query node tree.
     * @throws InvalidQueryException if the query node tree cannot be converted
     *                               into a String representation due to restrictions in SQL.
     */
    public static String toString(QueryRootNode root, NameResolver resolver)
            throws InvalidQueryException {
        return QueryFormat.toString(root, resolver);
    }

    /**
     * Parses the statement and returns the root node of the <code>QueryNode</code>
     * tree.
     *
     * @return the root node of the <code>QueryNode</code> tree.
     */
    private QueryRootNode getRootNode() {
        if (root == null) {
            stmt.jjtAccept(this, null);
        }
        return root;
    }

    //----------------< JCRSQLParserVisitor >------------------------------------

    public Object visit(SimpleNode node, Object data) {
        // do nothing, should never be called actually
        return data;
    }

    public Object visit(ASTQuery node, Object data) {
        root = factory.createQueryRootNode();
        root.setLocationNode(factory.createPathQueryNode(root));

        // pass to select, from, where, ...
        node.childrenAccept(this, root);

        // use //* if no path has been set
        PathQueryNode pathNode = root.getLocationNode();
        pathNode.setAbsolute(true);
        if (pathConstraints.size() == 0) {
            LocationStepQueryNode step = factory.createLocationStepQueryNode(pathNode);
            step.setNameTest(null);
            step.setIncludeDescendants(true);
            pathNode.addPathStep(step);
        } else {
            try {
                while (pathConstraints.size() > 1) {
                    // merge path nodes
                    MergingPathQueryNode path = null;
                    for (Iterator it = pathConstraints.iterator(); it.hasNext();) {
                        path = (MergingPathQueryNode) it.next();
                        if (path.needsMerge()) {
                            break;
                        } else {
                            path = null;
                        }
                    }
                    if (path == null) {
                        throw new IllegalArgumentException("Invalid combination of jcr:path clauses");
                    } else {
                        pathConstraints.remove(path);
                        MergingPathQueryNode[] paths = (MergingPathQueryNode[]) pathConstraints.toArray(new MergingPathQueryNode[pathConstraints.size()]);
                        paths = path.doMerge(paths);
                        pathConstraints.clear();
                        pathConstraints.addAll(Arrays.asList(paths));
                    }
                }
            } catch (NoSuchElementException e) {
                throw new IllegalArgumentException("Invalid combination of jcr:path clauses");
            }
            MergingPathQueryNode path = (MergingPathQueryNode) pathConstraints.get(0);
            LocationStepQueryNode[] steps = path.getPathSteps();
            for (int i = 0; i < steps.length; i++) {
                LocationStepQueryNode step = factory.createLocationStepQueryNode(pathNode);
                step.setNameTest(steps[i].getNameTest());
                step.setIncludeDescendants(steps[i].getIncludeDescendants());
                step.setIndex(steps[i].getIndex());
                pathNode.addPathStep(step);
            }
        }

        if (constraintNode.getNumOperands() == 1) {
            // attach operand to last path step
            LocationStepQueryNode[] steps = pathNode.getPathSteps();
            steps[steps.length - 1].addPredicate(constraintNode.getOperands()[0]);
        } else if (constraintNode.getNumOperands() > 1) {
            // attach constraint to last path step
            LocationStepQueryNode[] steps = pathNode.getPathSteps();
            steps[steps.length - 1].addPredicate(constraintNode);
        }

        if (nodeTypeName != null) {
            // add node type constraint
            LocationStepQueryNode[] steps = pathNode.getPathSteps();
            NodeTypeQueryNode nodeType
                    = factory.createNodeTypeQueryNode(steps[steps.length - 1], nodeTypeName);
            steps[steps.length - 1].addPredicate(nodeType);
        }

        return root;
    }

    public Object visit(ASTSelectList node, Object data) {
        final QueryRootNode root = (QueryRootNode) data;

        node.childrenAccept(new DefaultParserVisitor() {
            public Object visit(ASTIdentifier node, Object data) {
                root.addSelectProperty(node.getName());
                return data;
            }

            public Object visit(ASTExcerptFunction node, Object data) {
                root.addSelectProperty(NameFactoryImpl.getInstance().create(Name.NS_REP_URI, "excerpt(.)"));
                return data;
            }
        }, root);

        return data;
    }

    public Object visit(ASTFromClause node, Object data) {
        QueryRootNode root = (QueryRootNode) data;

        return node.childrenAccept(new DefaultParserVisitor() {
            public Object visit(ASTIdentifier node, Object data) {
                if (!node.getName().equals(NameConstants.NT_BASE)) {
                    // node is either primary or mixin node type
                    nodeTypeName = node.getName();
                }
                return data;
            }
        }, root);
    }

    public Object visit(ASTWhereClause node, Object data) {
        return node.childrenAccept(this, constraintNode);
    }

    public Object visit(ASTPredicate node, Object data) {
        NAryQueryNode parent = (NAryQueryNode) data;

        int type = node.getOperationType();
        QueryNode predicateNode;

        try {
            final Name[] tmp = new Name[2];
            final ASTLiteral[] value = new ASTLiteral[1];
            node.childrenAccept(new DefaultParserVisitor() {
                public Object visit(ASTIdentifier node, Object data) {
                    if (tmp[0] == null) {
                        tmp[0] = node.getName();
                    } else if (tmp[1] == null) {
                        tmp[1] = node.getName();
                    }
                    return data;
                }

                public Object visit(ASTLiteral node, Object data) {
                    value[0] = node;
                    return data;
                }

                public Object visit(ASTLowerFunction node, Object data) {
                    getIdentifier(node);
                    return data;
                }

                public Object visit(ASTUpperFunction node, Object data) {
                    getIdentifier(node);
                    return data;
                }

                private void getIdentifier(SimpleNode node) {
                    if (node.jjtGetNumChildren() > 0) {
                        Node n = node.jjtGetChild(0);
                        if (n instanceof ASTIdentifier) {
                            ASTIdentifier identifier = (ASTIdentifier) n;
                            if (tmp[0] == null) {
                                tmp[0] = identifier.getName();
                            } else if (tmp[1] == null) {
                                tmp[1] = identifier.getName();
                            }
                        }
                    }
                }
            }, data);
            Name identifier = tmp[0];

            if (identifier != null && identifier.equals(NameConstants.JCR_PATH)) {
                if (tmp[1] != null) {
                    // simply ignore, this is a join of a mixin node type
                } else {
                    createPathQuery(value[0].getValue(), parent.getType());
                }
                // done
                return data;
            }

            if (type == QueryConstants.OPERATION_BETWEEN) {
                AndQueryNode between = factory.createAndQueryNode(parent);
                RelationQueryNode rel = createRelationQueryNode(between,
                        identifier, QueryConstants.OPERATION_GE_GENERAL, (ASTLiteral) node.children[1]);
                node.childrenAccept(this, rel);
                between.addOperand(rel);
                rel = createRelationQueryNode(between,
                        identifier, QueryConstants.OPERATION_LE_GENERAL, (ASTLiteral) node.children[2]);
                node.childrenAccept(this, rel);
                between.addOperand(rel);
                predicateNode = between;
            } else if (type == QueryConstants.OPERATION_GE_GENERAL
                    || type == QueryConstants.OPERATION_GT_GENERAL
                    || type == QueryConstants.OPERATION_LE_GENERAL
                    || type == QueryConstants.OPERATION_LT_GENERAL
                    || type == QueryConstants.OPERATION_NE_GENERAL
                    || type == QueryConstants.OPERATION_EQ_GENERAL) {
                predicateNode = createRelationQueryNode(parent,
                        identifier, type, value[0]);
                node.childrenAccept(this, predicateNode);
            } else if (type == QueryConstants.OPERATION_LIKE) {
                ASTLiteral pattern = value[0];
                if (node.getEscapeString() != null) {
                    if (node.getEscapeString().length() == 1) {
                        // backslash is the escape character we use internally
                        pattern.setValue(translateEscaping(pattern.getValue(), node.getEscapeString().charAt(0), '\\'));
                    } else {
                        throw new IllegalArgumentException("ESCAPE string value must have length 1: '" + node.getEscapeString() + "'");
                    }
                } else {
                    // no escape character specified.
                    // if the pattern contains any backslash characters we need
                    // to escape them.
                    pattern.setValue(pattern.getValue().replaceAll("\\\\", "\\\\\\\\"));
                }
                predicateNode = createRelationQueryNode(parent,
                        identifier, type, pattern);
                node.childrenAccept(this, predicateNode);
            } else if (type == QueryConstants.OPERATION_IN) {
                OrQueryNode in = factory.createOrQueryNode(parent);
                for (int i = 1; i < node.children.length; i++) {
                    RelationQueryNode rel = createRelationQueryNode(in,
                            identifier, QueryConstants.OPERATION_EQ_VALUE, (ASTLiteral) node.children[i]);
                    node.childrenAccept(this, rel);
                    in.addOperand(rel);
                }
                predicateNode = in;
            } else if (type == QueryConstants.OPERATION_NULL
                    || type == QueryConstants.OPERATION_NOT_NULL) {
                predicateNode = createRelationQueryNode(parent,
                        identifier, type, null);
            } else if (type == QueryConstants.OPERATION_SIMILAR) {
                ASTLiteral literal;
                if (node.children.length == 1) {
                    literal = (ASTLiteral) node.children[0];
                } else {
                    literal = (ASTLiteral) node.children[1];
                }
                predicateNode = createRelationQueryNode(parent, identifier, type, literal);
            } else if (type == QueryConstants.OPERATION_SPELLCHECK) {
                predicateNode = createRelationQueryNode(parent,
                        NameConstants.JCR_PRIMARYTYPE, type,
                        (ASTLiteral) node.children[0]);
            } else {
                throw new IllegalArgumentException("Unknown operation type: " + type);
            }
        } catch (ArrayIndexOutOfBoundsException e) {
            throw new IllegalArgumentException("Too few arguments in predicate");
        }

        if (predicateNode != null) {
            parent.addOperand(predicateNode);
        }

        return data;
    }

    public Object visit(ASTOrExpression node, Object data) {
        NAryQueryNode parent = (NAryQueryNode) data;
        OrQueryNode orQuery = factory.createOrQueryNode(parent);
        // pass to operands
        node.childrenAccept(this, orQuery);

        if (orQuery.getNumOperands() > 0) {
            parent.addOperand(orQuery);
        }
        return parent;
    }

    public Object visit(ASTAndExpression node, Object data) {
        NAryQueryNode parent = (NAryQueryNode) data;
        AndQueryNode andQuery = factory.createAndQueryNode(parent);
        // pass to operands
        node.childrenAccept(this, andQuery);

        parent.addOperand(andQuery);
        return parent;
    }

    public Object visit(ASTNotExpression node, Object data) {
        NAryQueryNode parent = (NAryQueryNode) data;
        NotQueryNode notQuery = factory.createNotQueryNode(parent);
        // pass to operand
        node.childrenAccept(this, notQuery);

        parent.addOperand(notQuery);
        return parent;
    }<commit_msg>JCR-1688: Query parser builds invalid parse tree<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.jackrabbit.spi.commons.query.sql;

import org.apache.jackrabbit.spi.commons.query.AndQueryNode;
import org.apache.jackrabbit.spi.commons.query.LocationStepQueryNode;
import org.apache.jackrabbit.spi.commons.query.NAryQueryNode;
import org.apache.jackrabbit.spi.commons.query.NodeTypeQueryNode;
import org.apache.jackrabbit.spi.commons.query.NotQueryNode;
import org.apache.jackrabbit.spi.commons.query.OrQueryNode;
import org.apache.jackrabbit.spi.commons.query.OrderQueryNode;
import org.apache.jackrabbit.spi.commons.query.PathQueryNode;
import org.apache.jackrabbit.spi.commons.query.QueryConstants;
import org.apache.jackrabbit.spi.commons.query.QueryNode;
import org.apache.jackrabbit.spi.commons.query.QueryRootNode;
import org.apache.jackrabbit.spi.commons.query.RelationQueryNode;
import org.apache.jackrabbit.spi.commons.query.TextsearchQueryNode;
import org.apache.jackrabbit.spi.commons.query.PropertyFunctionQueryNode;
import org.apache.jackrabbit.spi.commons.query.QueryNodeFactory;
import org.apache.jackrabbit.spi.commons.conversion.NameException;
import org.apache.jackrabbit.spi.commons.name.NameConstants;
import org.apache.jackrabbit.spi.commons.name.NameFactoryImpl;
import org.apache.jackrabbit.spi.commons.name.PathBuilder;
import org.apache.jackrabbit.spi.Name;
import org.apache.jackrabbit.spi.Path;
import org.apache.jackrabbit.spi.commons.conversion.MalformedPathException;
import org.apache.jackrabbit.spi.commons.conversion.NameResolver;
import org.apache.jackrabbit.util.ISO8601;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.commons.collections.map.ReferenceMap;

import javax.jcr.query.InvalidQueryException;
import javax.jcr.NamespaceException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Map;
import java.io.StringReader;

/**
 * Implements the query builder for the JCR SQL syntax.
 */
public class JCRSQLQueryBuilder implements JCRSQLParserVisitor {

    /**
     * logger instance for this class
     */
    private static final Logger log = LoggerFactory.getLogger(JCRSQLQueryBuilder.class);

    /**
     * DateFormat pattern for type
     * {@link org.apache.jackrabbit.spi.commons.query.QueryConstants#TYPE_DATE}.
     */
    private static final String DATE_PATTERN = "yyyy-MM-dd";

    /**
     * Map of reusable JCRSQL parser instances indexed by NamespaceResolver.
     */
    private static Map parsers = new ReferenceMap(ReferenceMap.WEAK, ReferenceMap.WEAK);

    /**
     * The root node of the sql query syntax tree
     */
    private final ASTQuery stmt;

    /**
     * The root query node
     */
    private QueryRootNode root;

    /**
     * To resolve QNames
     */
    private NameResolver resolver;

    /**
     * Query node to gather the constraints defined in the WHERE clause
     */
    private final AndQueryNode constraintNode;

    /**
     * The Name of the node type in the from clause.
     */
    private Name nodeTypeName;

    /**
     * List of PathQueryNode constraints that need to be merged
     */
    private final List pathConstraints = new ArrayList();

    /**
     * The query node factory.
     */
    private final QueryNodeFactory factory;

    /**
     * Creates a new <code>JCRSQLQueryBuilder</code>.
     *
     * @param statement the root node of the SQL syntax tree.
     * @param resolver  a namespace resolver to use for names in the
     *                  <code>statement</code>.
     * @param factory   the query node factory.
     */
    private JCRSQLQueryBuilder(ASTQuery statement,
                               NameResolver resolver,
                               QueryNodeFactory factory) {
        this.stmt = statement;
        this.resolver = resolver;
        this.factory = factory;
        this.constraintNode =  factory.createAndQueryNode(null);
    }

    /**
     * Creates a <code>QueryNode</code> tree from a SQL <code>statement</code>
     * using the passed query node <code>factory</code>.
     *
     * @param statement the SQL statement.
     * @param resolver  the namespace resolver to use.
     * @return the <code>QueryNode</code> tree.
     * @throws InvalidQueryException if <code>statement</code> is malformed.
     */
    public static QueryRootNode createQuery(String statement,
                                            NameResolver resolver,
                                            QueryNodeFactory factory)
            throws InvalidQueryException {
        try {
            // get parser
            JCRSQLParser parser;
            synchronized (parsers) {
                parser = (JCRSQLParser) parsers.get(resolver);
                if (parser == null) {
                    parser = new JCRSQLParser(new StringReader(statement));
                    parser.setNameResolver(resolver);
                    parsers.put(resolver, parser);
                }
            }

            JCRSQLQueryBuilder builder;
            // guard against concurrent use within same session
            synchronized (parser) {
                parser.ReInit(new StringReader(statement));
                builder = new JCRSQLQueryBuilder(parser.Query(), resolver, factory);
            }
            return builder.getRootNode();
        } catch (ParseException e) {
            throw new InvalidQueryException(e.getMessage());
        } catch (IllegalArgumentException e) {
            throw new InvalidQueryException(e.getMessage());
        } catch (Throwable t) {
            // javacc parser may also throw an error in some cases
            throw new InvalidQueryException(t.getMessage());
        }
    }

    /**
     * Creates a String representation of the query node tree in SQL syntax.
     *
     * @param root     the root of the query node tree.
     * @param resolver to resolve QNames.
     * @return a String representation of the query node tree.
     * @throws InvalidQueryException if the query node tree cannot be converted
     *                               into a String representation due to restrictions in SQL.
     */
    public static String toString(QueryRootNode root, NameResolver resolver)
            throws InvalidQueryException {
        return QueryFormat.toString(root, resolver);
    }

    /**
     * Parses the statement and returns the root node of the <code>QueryNode</code>
     * tree.
     *
     * @return the root node of the <code>QueryNode</code> tree.
     */
    private QueryRootNode getRootNode() {
        if (root == null) {
            stmt.jjtAccept(this, null);
        }
        return root;
    }

    //----------------< JCRSQLParserVisitor >------------------------------------

    public Object visit(SimpleNode node, Object data) {
        // do nothing, should never be called actually
        return data;
    }

    public Object visit(ASTQuery node, Object data) {
        root = factory.createQueryRootNode();
        root.setLocationNode(factory.createPathQueryNode(root));

        // pass to select, from, where, ...
        node.childrenAccept(this, root);

        // use //* if no path has been set
        PathQueryNode pathNode = root.getLocationNode();
        pathNode.setAbsolute(true);
        if (pathConstraints.size() == 0) {
            LocationStepQueryNode step = factory.createLocationStepQueryNode(pathNode);
            step.setNameTest(null);
            step.setIncludeDescendants(true);
            pathNode.addPathStep(step);
        } else {
            try {
                while (pathConstraints.size() > 1) {
                    // merge path nodes
                    MergingPathQueryNode path = null;
                    for (Iterator it = pathConstraints.iterator(); it.hasNext();) {
                        path = (MergingPathQueryNode) it.next();
                        if (path.needsMerge()) {
                            break;
                        } else {
                            path = null;
                        }
                    }
                    if (path == null) {
                        throw new IllegalArgumentException("Invalid combination of jcr:path clauses");
                    } else {
                        pathConstraints.remove(path);
                        MergingPathQueryNode[] paths = (MergingPathQueryNode[]) pathConstraints.toArray(new MergingPathQueryNode[pathConstraints.size()]);
                        paths = path.doMerge(paths);
                        pathConstraints.clear();
                        pathConstraints.addAll(Arrays.asList(paths));
                    }
                }
            } catch (NoSuchElementException e) {
                throw new IllegalArgumentException("Invalid combination of jcr:path clauses");
            }
            MergingPathQueryNode path = (MergingPathQueryNode) pathConstraints.get(0);
            LocationStepQueryNode[] steps = path.getPathSteps();
            for (int i = 0; i < steps.length; i++) {
                LocationStepQueryNode step = factory.createLocationStepQueryNode(pathNode);
                step.setNameTest(steps[i].getNameTest());
                step.setIncludeDescendants(steps[i].getIncludeDescendants());
                step.setIndex(steps[i].getIndex());
                pathNode.addPathStep(step);
            }
        }

        if (constraintNode.getNumOperands() == 1) {
            // attach operand to last path step
            LocationStepQueryNode[] steps = pathNode.getPathSteps();
            steps[steps.length - 1].addPredicate(constraintNode.getOperands()[0]);
        } else if (constraintNode.getNumOperands() > 1) {
            // attach constraint to last path step
            LocationStepQueryNode[] steps = pathNode.getPathSteps();
            steps[steps.length - 1].addPredicate(constraintNode);
        }

        if (nodeTypeName != null) {
            // add node type constraint
            LocationStepQueryNode[] steps = pathNode.getPathSteps();
            NodeTypeQueryNode nodeType
                    = factory.createNodeTypeQueryNode(steps[steps.length - 1], nodeTypeName);
            steps[steps.length - 1].addPredicate(nodeType);
        }

        return root;
    }

    public Object visit(ASTSelectList node, Object data) {
        final QueryRootNode root = (QueryRootNode) data;

        node.childrenAccept(new DefaultParserVisitor() {
            public Object visit(ASTIdentifier node, Object data) {
                root.addSelectProperty(node.getName());
                return data;
            }

            public Object visit(ASTExcerptFunction node, Object data) {
                root.addSelectProperty(NameFactoryImpl.getInstance().create(Name.NS_REP_URI, "excerpt(.)"));
                return data;
            }
        }, root);

        return data;
    }

    public Object visit(ASTFromClause node, Object data) {
        QueryRootNode root = (QueryRootNode) data;

        return node.childrenAccept(new DefaultParserVisitor() {
            public Object visit(ASTIdentifier node, Object data) {
                if (!node.getName().equals(NameConstants.NT_BASE)) {
                    // node is either primary or mixin node type
                    nodeTypeName = node.getName();
                }
                return data;
            }
        }, root);
    }

    public Object visit(ASTWhereClause node, Object data) {
        return node.childrenAccept(this, constraintNode);
    }

    public Object visit(ASTPredicate node, Object data) {
        NAryQueryNode parent = (NAryQueryNode) data;

        int type = node.getOperationType();
        QueryNode predicateNode;

        try {
            final Name[] tmp = new Name[2];
            final ASTLiteral[] value = new ASTLiteral[1];
            node.childrenAccept(new DefaultParserVisitor() {
                public Object visit(ASTIdentifier node, Object data) {
                    if (tmp[0] == null) {
                        tmp[0] = node.getName();
                    } else if (tmp[1] == null) {
                        tmp[1] = node.getName();
                    }
                    return data;
                }

                public Object visit(ASTLiteral node, Object data) {
                    value[0] = node;
                    return data;
                }

                public Object visit(ASTLowerFunction node, Object data) {
                    getIdentifier(node);
                    return data;
                }

                public Object visit(ASTUpperFunction node, Object data) {
                    getIdentifier(node);
                    return data;
                }

                private void getIdentifier(SimpleNode node) {
                    if (node.jjtGetNumChildren() > 0) {
                        Node n = node.jjtGetChild(0);
                        if (n instanceof ASTIdentifier) {
                            ASTIdentifier identifier = (ASTIdentifier) n;
                            if (tmp[0] == null) {
                                tmp[0] = identifier.getName();
                            } else if (tmp[1] == null) {
                                tmp[1] = identifier.getName();
                            }
                        }
                    }
                }
            }, data);
            Name identifier = tmp[0];

            if (identifier != null && identifier.equals(NameConstants.JCR_PATH)) {
                if (tmp[1] != null) {
                    // simply ignore, this is a join of a mixin node type
                } else {
                    createPathQuery(value[0].getValue(), parent.getType());
                }
                // done
                return data;
            }

            if (type == QueryConstants.OPERATION_BETWEEN) {
                AndQueryNode between = factory.createAndQueryNode(parent);
                RelationQueryNode rel = createRelationQueryNode(between,
                        identifier, QueryConstants.OPERATION_GE_GENERAL, (ASTLiteral) node.children[1]);
                node.childrenAccept(this, rel);
                between.addOperand(rel);
                rel = createRelationQueryNode(between,
                        identifier, QueryConstants.OPERATION_LE_GENERAL, (ASTLiteral) node.children[2]);
                node.childrenAccept(this, rel);
                between.addOperand(rel);
                predicateNode = between;
            } else if (type == QueryConstants.OPERATION_GE_GENERAL
                    || type == QueryConstants.OPERATION_GT_GENERAL
                    || type == QueryConstants.OPERATION_LE_GENERAL
                    || type == QueryConstants.OPERATION_LT_GENERAL
                    || type == QueryConstants.OPERATION_NE_GENERAL
                    || type == QueryConstants.OPERATION_EQ_GENERAL) {
                predicateNode = createRelationQueryNode(parent,
                        identifier, type, value[0]);
                node.childrenAccept(this, predicateNode);
            } else if (type == QueryConstants.OPERATION_LIKE) {
                ASTLiteral pattern = value[0];
                if (node.getEscapeString() != null) {
                    if (node.getEscapeString().length() == 1) {
                        // backslash is the escape character we use internally
                        pattern.setValue(translateEscaping(pattern.getValue(), node.getEscapeString().charAt(0), '\\'));
                    } else {
                        throw new IllegalArgumentException("ESCAPE string value must have length 1: '" + node.getEscapeString() + "'");
                    }
                } else {
                    // no escape character specified.
                    // if the pattern contains any backslash characters we need
                    // to escape them.
                    pattern.setValue(pattern.getValue().replaceAll("\\\\", "\\\\\\\\"));
                }
                predicateNode = createRelationQueryNode(parent,
                        identifier, type, pattern);
                node.childrenAccept(this, predicateNode);
            } else if (type == QueryConstants.OPERATION_IN) {
                OrQueryNode in = factory.createOrQueryNode(parent);
                for (int i = 1; i < node.children.length; i++) {
                    RelationQueryNode rel = createRelationQueryNode(in,
                            identifier, QueryConstants.OPERATION_EQ_VALUE, (ASTLiteral) node.children[i]);
                    node.childrenAccept(this, rel);
                    in.addOperand(rel);
                }
                predicateNode = in;
            } else if (type == QueryConstants.OPERATION_NULL
                    || type == QueryConstants.OPERATION_NOT_NULL) {
                predicateNode = createRelationQueryNode(parent,
                        identifier, type, null);
            } else if (type == QueryConstants.OPERATION_SIMILAR) {
                ASTLiteral literal;
                if (node.children.length == 1) {
                    literal = (ASTLiteral) node.children[0];
                } else {
                    literal = (ASTLiteral) node.children[1];
                }
                predicateNode = createRelationQueryNode(parent, identifier, type, literal);
            } else if (type == QueryConstants.OPERATION_SPELLCHECK) {
                predicateNode = createRelationQueryNode(parent,
                        NameConstants.JCR_PRIMARYTYPE, type,
                        (ASTLiteral) node.children[0]);
            } else {
                throw new IllegalArgumentException("Unknown operation type: " + type);
            }
        } catch (ArrayIndexOutOfBoundsException e) {
            throw new IllegalArgumentException("Too few arguments in predicate");
        }

        if (predicateNode != null) {
            parent.addOperand(predicateNode);
        }

        return data;
    }

    public Object visit(ASTOrExpression node, Object data) {
        NAryQueryNode parent = (NAryQueryNode) data;
        OrQueryNode orQuery = factory.createOrQueryNode(parent);
        // pass to operands
        node.childrenAccept(this, orQuery);

        if (orQuery.getNumOperands() > 0) {
            parent.addOperand(orQuery);
        }
        return parent;
    }

    public Object visit(ASTAndExpression node, Object data) {
        NAryQueryNode parent = (NAryQueryNode) data;
        AndQueryNode andQuery = factory.createAndQueryNode(parent);
        // pass to operands
        node.childrenAccept(this, andQuery);

        if (andQuery.getNumOperands() > 0) {
            parent.addOperand(andQuery);
        }
        return parent;
    }

    public Object visit(ASTNotExpression node, Object data) {
        NAryQueryNode parent = (NAryQueryNode) data;
        NotQueryNode notQuery = factory.createNotQueryNode(parent);
        // pass to operand
        node.childrenAccept(this, notQuery);

        if (notQuery.getNumOperands() > 0) {
            parent.addOperand(notQuery);
        }
        return parent;
    }
{'fraction_non_alphanumeric': 0.09133060978971905, 'fraction_numerical': 0.0026567081881751424, 'mean_word_length': 1.8713298791018997, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4675437', 'n_tokens_mistral': 10092, 'n_tokens_neox': 9612, 'n_words': 2478}