<commit_before>/*
 * Copyright (c) 2010 Mysema Ltd.
 * All rights reserved.
 *
 */
package com.mysema.query.lucene;

import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;

import javax.annotation.Nullable;

import org.apache.commons.lang.StringUtils;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.util.NumericUtils;

import com.mysema.query.QueryMetadata;
import com.mysema.query.types.Constant;
import com.mysema.query.types.Expression;
import com.mysema.query.types.Operation;
import com.mysema.query.types.Operator;
import com.mysema.query.types.Ops;
import com.mysema.query.types.OrderSpecifier;
import com.mysema.query.types.ParamExpression;
import com.mysema.query.types.ParamNotSetException;
import com.mysema.query.types.Path;
import com.mysema.query.types.PathType;
/**
 * Serializes Querydsl queries to Lucene queries.
 *
 * @author vema
 *
 */
public class LuceneSerializer {
    private static final Map<Class<?>, Integer> sortFields = new HashMap<Class<?>, Integer>();

    static {
        sortFields.put(Integer.class, SortField.INT);
        sortFields.put(Float.class, SortField.FLOAT);
        sortFields.put(Long.class, SortField.LONG);
        sortFields.put(Double.class, SortField.DOUBLE);
        sortFields.put(Short.class, SortField.SHORT);
        sortFields.put(Byte.class, SortField.BYTE);
        sortFields.put(BigDecimal.class, SortField.DOUBLE);
        sortFields.put(BigInteger.class, SortField.LONG);
    }

    public static final LuceneSerializer DEFAULT = new LuceneSerializer(false, true);

    private final boolean lowerCase;

    private final boolean splitTerms;

    public LuceneSerializer(boolean lowerCase, boolean splitTerms) {
        this.lowerCase = lowerCase;
        this.splitTerms = splitTerms;
    }

    private Query toQuery(Operation<?> operation, QueryMetadata metadata) {
        Operator<?> op = operation.getOperator();
        if (op == Ops.OR) {
            return toTwoHandSidedQuery(operation, Occur.SHOULD, metadata);
        } else if (op == Ops.AND) {
            return toTwoHandSidedQuery(operation, Occur.MUST, metadata);
        } else if (op == Ops.NOT) {
            BooleanQuery bq = new BooleanQuery();
            bq.add(new BooleanClause(toQuery(operation.getArg(0), metadata), Occur.MUST_NOT));
            return bq;
        } else if (op == Ops.LIKE) {
            return like(operation, metadata);
        } else if (op == Ops.EQ_OBJECT || op == Ops.EQ_PRIMITIVE || op == Ops.EQ_IGNORE_CASE) {
            return eq(operation, metadata);
        } else if (op == Ops.NE_OBJECT || op == Ops.NE_PRIMITIVE) {
            return ne(operation, metadata);
        } else if (op == Ops.STARTS_WITH || op == Ops.STARTS_WITH_IC) {
            return startsWith(metadata, operation);
        } else if (op == Ops.ENDS_WITH || op == Ops.ENDS_WITH_IC) {
            return endsWith(operation, metadata);
        } else if (op == Ops.STRING_CONTAINS || op == Ops.STRING_CONTAINS_IC) {
            return stringContains(operation, metadata);
        } else if (op == Ops.BETWEEN) {
            return between(operation, metadata);
        } else if (op == Ops.IN) {
            return in(operation, metadata);
        } else if (op == Ops.LT || op == Ops.BEFORE) {
            return lt(operation, metadata);
        } else if (op == Ops.GT || op == Ops.AFTER) {
            return gt(operation, metadata);
        } else if (op == Ops.LOE || op == Ops.BOE) {
            return le(operation, metadata);
        } else if (op == Ops.GOE || op == Ops.AOE) {
            return ge(operation, metadata);
        } else if (op == Ops.DELEGATE){
            return toQuery(operation.getArg(0), metadata);
        }
        throw new UnsupportedOperationException("Illegal operation " + operation);
    }

    private Query toTwoHandSidedQuery(Operation<?> operation, Occur occur, QueryMetadata metadata) {
        Query lhs = toQuery(operation.getArg(0), metadata);
        Query rhs = toQuery(operation.getArg(1), metadata);
        BooleanQuery bq = new BooleanQuery();
        bq.add(createBooleanClause(lhs, occur));
        bq.add(createBooleanClause(rhs, occur));
        return bq;
    }

    /**
     * If the query is a BooleanQuery and it contains a single Occur.MUST_NOT
     * clause it will be returned as is. Otherwise it will be wrapped in a
     * BooleanClause with the given Occur.
     */
    private BooleanClause createBooleanClause(Query query, Occur occur) {
        if (query instanceof BooleanQuery) {
            BooleanClause[] clauses = ((BooleanQuery) query).getClauses();
            if (clauses.length == 1 && clauses[0].getOccur().equals(Occur.MUST_NOT)) {
                return clauses[0];
            }
        }
        return new BooleanClause(query, occur);
    }

    protected Query like(Operation<?> operation, QueryMetadata metadata) {
        verifyArguments(operation);
        String field = toField(operation.getArg(0));
        boolean tokenized = isTokenized(operation.getArg(0));
        
        String[] terms = createTerms(operation.getArg(1), tokenized, metadata);
        if (terms.length > 1) {
            BooleanQuery bq = new BooleanQuery();
            for (String s : terms) {
                bq.add(new WildcardQuery(new Term(field, "*" + normalize(s) + "*")), Occur.MUST);
            }
            return bq;
        }
        return new WildcardQuery(new Term(field, normalize(terms[0])));
    }

    @SuppressWarnings("unchecked")
    protected Query eq(Operation<?> operation, QueryMetadata metadata) {
        verifyArguments(operation);
        String field = toField(operation.getArg(0));
        boolean tokenized = isTokenized(operation.getArg(0));
        
        if (Number.class.isAssignableFrom(operation.getArg(1).getType())) {
            return new TermQuery(new Term(field, convertNumber(((Constant<Number>) operation
                    .getArg(1)).getConstant())));
        }
        return eq(field, createTerms(operation.getArg(1), tokenized, metadata), metadata);
    }

    protected boolean isTokenized(Expression<?> arg) {
        return splitTerms;
    }

    private String convertNumber(Number number) {
        if (Integer.class.isInstance(number)) {
            return NumericUtils.intToPrefixCoded(number.intValue());
        } else if (Double.class.isInstance(number)) {
            return NumericUtils.doubleToPrefixCoded(number.doubleValue());
        } else if (Long.class.isInstance(number)) {
            return NumericUtils.longToPrefixCoded(number.longValue());
        } else if (Float.class.isInstance(number)) {
            return NumericUtils.floatToPrefixCoded(number.floatValue());
        } else if (Byte.class.isInstance(number)) {
            return NumericUtils.intToPrefixCoded(number.intValue());
        } else if (Short.class.isInstance(number)) {
            return NumericUtils.intToPrefixCoded(number.intValue());
        } else if (BigDecimal.class.isInstance(number)) {
            return NumericUtils.doubleToPrefixCoded(number.doubleValue());
        } else if (BigInteger.class.isInstance(number)) {
            return NumericUtils.longToPrefixCoded(number.longValue());
        } else {
            throw new IllegalArgumentException("Unsupported numeric type "
                    + number.getClass().getName());
        }
    }

    protected Query eq(String field, String[] terms, QueryMetadata metadata) {
        if (terms.length > 1) {
            PhraseQuery pq = new PhraseQuery();
            for (String s : terms) {
                pq.add(new Term(field, normalize(s)));
            }
            return pq;
        }
        return new TermQuery(new Term(field, normalize(terms[0])));
    }

    @SuppressWarnings("unchecked")
    protected Query in(Operation<?> operation, QueryMetadata metadata) {
        String field = toField(operation.getArg(0));
        boolean tokenized = isTokenized(operation.getArg(0));
        Collection values = (Collection) ((Constant) operation.getArg(1)).getConstant();
        BooleanQuery bq = new BooleanQuery();
        for (Object value : values) {
            bq.add(eq(field, tokenized ? StringUtils.split(value.toString()) : new String[] { value.toString() }, metadata), Occur.SHOULD);
        }
        return bq;
    }

    protected Query ne(Operation<?> operation, QueryMetadata metadata) {
        BooleanQuery bq = new BooleanQuery();
        bq.add(new BooleanClause(eq(operation, metadata), Occur.MUST_NOT));
        return bq;
    }

    protected Query startsWith(QueryMetadata metadata, Operation<?> operation) {
        verifyArguments(operation);
        String field = toField(operation.getArg(0));
        boolean tokenized = isTokenized(operation.getArg(0));
        String[] terms = createEscapedTerms(operation.getArg(1), tokenized, metadata);
        if (terms.length > 1) {
            BooleanQuery bq = new BooleanQuery();
            for (int i = 0; i < terms.length; ++i) {
                String s = i == 0 ? terms[i] + "*" : "*" + terms[i] + "*";
                bq.add(new WildcardQuery(new Term(field, normalize(s))), Occur.MUST);
            }
            return bq;
        }
        return new PrefixQuery(new Term(field, normalize(terms[0])));
    }

    protected Query stringContains(Operation<?> operation, QueryMetadata metadata) {
        verifyArguments(operation);
        String field = toField(operation.getArg(0));
        boolean tokenized = isTokenized(operation.getArg(0));
        String[] terms = createEscapedTerms(operation.getArg(1), tokenized, metadata);
        if (terms.length > 1) {
            BooleanQuery bq = new BooleanQuery();
            for (String s : terms) {
                bq.add(new WildcardQuery(new Term(field, "*" + normalize(s) + "*")), Occur.MUST);
            }
            return bq;
        }
        return new WildcardQuery(new Term(field, "*" + normalize(terms[0]) + "*"));
    }

    protected Query endsWith(Operation<?> operation, QueryMetadata metadata) {
        verifyArguments(operation);
        String field = toField(operation.getArg(0));
        boolean tokenized = isTokenized(operation.getArg(0));
        String[] terms = createEscapedTerms(operation.getArg(1), tokenized, metadata);
        if (terms.length > 1) {
            BooleanQuery bq = new BooleanQuery();
            for (int i = 0; i < terms.length; ++i) {
                String s = i == terms.length - 1 ? "*" + terms[i] : "*" + terms[i] + "*";
                bq.add(new WildcardQuery(new Term(field, normalize(s))), Occur.MUST);
            }
            return bq;
        }
        return new WildcardQuery(new Term(field, "*" + normalize(terms[0])));
    }

    protected Query between(Operation<?> operation, QueryMetadata metadata) {
        verifyArguments(operation);
        // TODO Phrase not properly supported
        return range(toField(operation.getArg(0)), operation.getArg(1), operation.getArg(2), true, true, metadata);
    }

    protected Query lt(Operation<?> operation, QueryMetadata metadata) {
        verifyArguments(operation);
        return range(toField(operation.getArg(0)), null, operation.getArg(1), false, false, metadata);
    }

    protected Query gt(Operation<?> operation, QueryMetadata metadata) {
        verifyArguments(operation);
        return range(toField(operation.getArg(0)), operation.getArg(1), null, false, false, metadata);
    }

    protected Query le(Operation<?> operation, QueryMetadata metadata) {
        verifyArguments(operation);
        return range(toField(operation.getArg(0)), null, operation.getArg(1), true, true, metadata);
    }

    protected Query ge(Operation<?> operation, QueryMetadata metadata) {
        verifyArguments(operation);
        return range(toField(operation.getArg(0)), operation.getArg(1), null, true, true, metadata);
    }

    @SuppressWarnings("unchecked")
    protected Query range(String field, @Nullable Expression<?> min, @Nullable Expression<?> max, boolean minInc, boolean maxInc, QueryMetadata metadata) {
        if (min != null && Number.class.isAssignableFrom(min.getType()) || max != null
                && Number.class.isAssignableFrom(max.getType())) {
            Class<? extends Number> numType = (Class) (min != null ? min.getType() : max.getType());
            return numericRange((Class) numType, field, (Number) (min == null ? null
                    : ((Constant) min).getConstant()), (Number) (max == null ? null
                    : ((Constant) max).getConstant()), minInc, maxInc);
        }
        return stringRange(field, min, max, minInc, maxInc, metadata);
    }

    protected <N extends Number> NumericRangeQuery<?> numericRange(Class<N> clazz, String field,
            @Nullable N min, @Nullable N max, boolean minInc, boolean maxInc) {
        if (clazz.equals(Integer.class)) {
            return NumericRangeQuery.newIntRange(field, (Integer) min, (Integer) max, minInc, maxInc);
        } else if (clazz.equals(Double.class)) {
            return NumericRangeQuery.newDoubleRange(field, (Double) min, (Double) max, minInc, minInc);
        } else if (clazz.equals(Float.class)) {
            return NumericRangeQuery.newFloatRange(field, (Float) min, (Float) max, minInc, minInc);
        } else if (clazz.equals(Long.class)) {
            return NumericRangeQuery.newLongRange(field, (Long) min, (Long) max, minInc, minInc);
        } else if (clazz.equals(Byte.class) || clazz.equals(Short.class)) {
            return NumericRangeQuery.newIntRange(field, min != null ? min.intValue() : null,
                    max != null ? max.intValue() : null, minInc, maxInc);
        } else {
            throw new IllegalArgumentException("Unsupported numeric type " + clazz.getName());
        }
    }

    protected Query stringRange(String field, @Nullable Expression<?> min, @Nullable Expression<?> max,
            boolean minInc, boolean maxInc, QueryMetadata metadata) {
        if (min == null) {
            return new TermRangeQuery(field, null, normalize(createTerms(max, false, metadata)[0]), minInc, maxInc);
        } else if (max == null) {
            return new TermRangeQuery(field, normalize(createTerms(min, false, metadata)[0]), null, minInc, maxInc);
        } else {
            return new TermRangeQuery(field, normalize(createTerms(min, false, metadata)[0]), normalize(createTerms(max, false, metadata)[0]), minInc, maxInc);
        }
    }

    @SuppressWarnings("unchecked")
    protected String toField(Expression<?> expr) {
        if (expr instanceof Path) {
            return toField((Path<?>) expr);
        } else if (expr instanceof Operation) {
            Operation<?> operation = (Operation<?>) expr;
            if (operation.getOperator() == Ops.LOWER || operation.getOperator() == Ops.UPPER) {
                return toField(operation.getArg(0));
            }
        }
        throw new IllegalArgumentException("Unable to transform " + expr + " to field");
    }

    protected String toField(Path<?> path) {
        String rv = path.getMetadata().getExpression().toString();
        if (path.getMetadata().getParent() != null){
            Path<?> parent = path.getMetadata().getParent();
            if (parent.getMetadata().getPathType() != PathType.VARIABLE){
                rv = toField(parent) + "." + rv;
            }
        }
        return rv;
    }

    private void verifyArguments(Operation<?> operation) {
        List<Expression<?>> arguments = operation.getArgs();
        for (int i = 1; i < arguments.size(); ++i) {
            if (!(arguments.get(i) instanceof Constant<?>)
                    && !(arguments.get(i) instanceof ParamExpression<?>)
                    && !(arguments.get(i) instanceof PhraseElement)
                    && !(arguments.get(i) instanceof TermElement)) {
                throw new IllegalArgumentException("operand was of unsupported type "
                        + arguments.get(i).getClass().getName());
            }
        }
    }

    private String[] createTerms(Expression<?> expr, boolean tokenized, QueryMetadata metadata) {
        if (expr instanceof ParamExpression<?>){
            Object value = metadata.getParams().get(expr);
            if (value == null){
                throw new ParamNotSetException((ParamExpression<?>) expr);
            }
            return split(expr, tokenized, value.toString());
        }else{
            return split(expr, tokenized, expr.toString());<commit_msg>documented template methods<commit_after>/*
 * Copyright (c) 2010 Mysema Ltd.
 * All rights reserved.
 *
 */
package com.mysema.query.lucene;

import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;

import javax.annotation.Nullable;

import org.apache.commons.lang.StringUtils;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.util.NumericUtils;

import com.mysema.query.QueryMetadata;
import com.mysema.query.types.Constant;
import com.mysema.query.types.Expression;
import com.mysema.query.types.Operation;
import com.mysema.query.types.Operator;
import com.mysema.query.types.Ops;
import com.mysema.query.types.OrderSpecifier;
import com.mysema.query.types.ParamExpression;
import com.mysema.query.types.ParamNotSetException;
import com.mysema.query.types.Path;
import com.mysema.query.types.PathType;
/**
 * Serializes Querydsl queries to Lucene queries.
 *
 * @author vema
 *
 */
public class LuceneSerializer {
    private static final Map<Class<?>, Integer> sortFields = new HashMap<Class<?>, Integer>();

    static {
        sortFields.put(Integer.class, SortField.INT);
        sortFields.put(Float.class, SortField.FLOAT);
        sortFields.put(Long.class, SortField.LONG);
        sortFields.put(Double.class, SortField.DOUBLE);
        sortFields.put(Short.class, SortField.SHORT);
        sortFields.put(Byte.class, SortField.BYTE);
        sortFields.put(BigDecimal.class, SortField.DOUBLE);
        sortFields.put(BigInteger.class, SortField.LONG);
    }

    public static final LuceneSerializer DEFAULT = new LuceneSerializer(false, true);

    private final boolean lowerCase;

    private final boolean splitTerms;

    public LuceneSerializer(boolean lowerCase, boolean splitTerms) {
        this.lowerCase = lowerCase;
        this.splitTerms = splitTerms;
    }

    private Query toQuery(Operation<?> operation, QueryMetadata metadata) {
        Operator<?> op = operation.getOperator();
        if (op == Ops.OR) {
            return toTwoHandSidedQuery(operation, Occur.SHOULD, metadata);
        } else if (op == Ops.AND) {
            return toTwoHandSidedQuery(operation, Occur.MUST, metadata);
        } else if (op == Ops.NOT) {
            BooleanQuery bq = new BooleanQuery();
            bq.add(new BooleanClause(toQuery(operation.getArg(0), metadata), Occur.MUST_NOT));
            return bq;
        } else if (op == Ops.LIKE) {
            return like(operation, metadata);
        } else if (op == Ops.EQ_OBJECT || op == Ops.EQ_PRIMITIVE || op == Ops.EQ_IGNORE_CASE) {
            return eq(operation, metadata);
        } else if (op == Ops.NE_OBJECT || op == Ops.NE_PRIMITIVE) {
            return ne(operation, metadata);
        } else if (op == Ops.STARTS_WITH || op == Ops.STARTS_WITH_IC) {
            return startsWith(metadata, operation);
        } else if (op == Ops.ENDS_WITH || op == Ops.ENDS_WITH_IC) {
            return endsWith(operation, metadata);
        } else if (op == Ops.STRING_CONTAINS || op == Ops.STRING_CONTAINS_IC) {
            return stringContains(operation, metadata);
        } else if (op == Ops.BETWEEN) {
            return between(operation, metadata);
        } else if (op == Ops.IN) {
            return in(operation, metadata);
        } else if (op == Ops.LT || op == Ops.BEFORE) {
            return lt(operation, metadata);
        } else if (op == Ops.GT || op == Ops.AFTER) {
            return gt(operation, metadata);
        } else if (op == Ops.LOE || op == Ops.BOE) {
            return le(operation, metadata);
        } else if (op == Ops.GOE || op == Ops.AOE) {
            return ge(operation, metadata);
        } else if (op == Ops.DELEGATE){
            return toQuery(operation.getArg(0), metadata);
        }
        throw new UnsupportedOperationException("Illegal operation " + operation);
    }

    private Query toTwoHandSidedQuery(Operation<?> operation, Occur occur, QueryMetadata metadata) {
        Query lhs = toQuery(operation.getArg(0), metadata);
        Query rhs = toQuery(operation.getArg(1), metadata);
        BooleanQuery bq = new BooleanQuery();
        bq.add(createBooleanClause(lhs, occur));
        bq.add(createBooleanClause(rhs, occur));
        return bq;
    }

    /**
     * If the query is a BooleanQuery and it contains a single Occur.MUST_NOT
     * clause it will be returned as is. Otherwise it will be wrapped in a
     * BooleanClause with the given Occur.
     */
    private BooleanClause createBooleanClause(Query query, Occur occur) {
        if (query instanceof BooleanQuery) {
            BooleanClause[] clauses = ((BooleanQuery) query).getClauses();
            if (clauses.length == 1 && clauses[0].getOccur().equals(Occur.MUST_NOT)) {
                return clauses[0];
            }
        }
        return new BooleanClause(query, occur);
    }

    protected Query like(Operation<?> operation, QueryMetadata metadata) {
        verifyArguments(operation);
        String field = toField(operation.getArg(0));
        boolean tokenized = isTokenized(operation.getArg(0));
        
        String[] terms = createTerms(operation.getArg(1), tokenized, metadata);
        if (terms.length > 1) {
            BooleanQuery bq = new BooleanQuery();
            for (String s : terms) {
                bq.add(new WildcardQuery(new Term(field, "*" + normalize(s) + "*")), Occur.MUST);
            }
            return bq;
        }
        return new WildcardQuery(new Term(field, normalize(terms[0])));
    }

    @SuppressWarnings("unchecked")
    protected Query eq(Operation<?> operation, QueryMetadata metadata) {
        verifyArguments(operation);
        String field = toField(operation.getArg(0));
        boolean tokenized = isTokenized(operation.getArg(0));
        
        if (Number.class.isAssignableFrom(operation.getArg(1).getType())) {
            return new TermQuery(new Term(field, convertNumber(((Constant<Number>) operation
                    .getArg(1)).getConstant())));
        }
        return eq(field, createTerms(operation.getArg(1), tokenized, metadata), metadata);
    }


    private String convertNumber(Number number) {
        if (Integer.class.isInstance(number)) {
            return NumericUtils.intToPrefixCoded(number.intValue());
        } else if (Double.class.isInstance(number)) {
            return NumericUtils.doubleToPrefixCoded(number.doubleValue());
        } else if (Long.class.isInstance(number)) {
            return NumericUtils.longToPrefixCoded(number.longValue());
        } else if (Float.class.isInstance(number)) {
            return NumericUtils.floatToPrefixCoded(number.floatValue());
        } else if (Byte.class.isInstance(number)) {
            return NumericUtils.intToPrefixCoded(number.intValue());
        } else if (Short.class.isInstance(number)) {
            return NumericUtils.intToPrefixCoded(number.intValue());
        } else if (BigDecimal.class.isInstance(number)) {
            return NumericUtils.doubleToPrefixCoded(number.doubleValue());
        } else if (BigInteger.class.isInstance(number)) {
            return NumericUtils.longToPrefixCoded(number.longValue());
        } else {
            throw new IllegalArgumentException("Unsupported numeric type "
                    + number.getClass().getName());
        }
    }

    protected Query eq(String field, String[] terms, QueryMetadata metadata) {
        if (terms.length > 1) {
            PhraseQuery pq = new PhraseQuery();
            for (String s : terms) {
                pq.add(new Term(field, normalize(s)));
            }
            return pq;
        }
        return new TermQuery(new Term(field, normalize(terms[0])));
    }

    @SuppressWarnings("unchecked")
    protected Query in(Operation<?> operation, QueryMetadata metadata) {
        String field = toField(operation.getArg(0));
        boolean tokenized = isTokenized(operation.getArg(0));
        Collection values = (Collection) ((Constant) operation.getArg(1)).getConstant();
        BooleanQuery bq = new BooleanQuery();
        for (Object value : values) {
            bq.add(eq(field, tokenized ? StringUtils.split(value.toString()) : new String[] { value.toString() }, metadata), Occur.SHOULD);
        }
        return bq;
    }

    protected Query ne(Operation<?> operation, QueryMetadata metadata) {
        BooleanQuery bq = new BooleanQuery();
        bq.add(new BooleanClause(eq(operation, metadata), Occur.MUST_NOT));
        return bq;
    }

    protected Query startsWith(QueryMetadata metadata, Operation<?> operation) {
        verifyArguments(operation);
        String field = toField(operation.getArg(0));
        boolean tokenized = isTokenized(operation.getArg(0));
        String[] terms = createEscapedTerms(operation.getArg(1), tokenized, metadata);
        if (terms.length > 1) {
            BooleanQuery bq = new BooleanQuery();
            for (int i = 0; i < terms.length; ++i) {
                String s = i == 0 ? terms[i] + "*" : "*" + terms[i] + "*";
                bq.add(new WildcardQuery(new Term(field, normalize(s))), Occur.MUST);
            }
            return bq;
        }
        return new PrefixQuery(new Term(field, normalize(terms[0])));
    }

    protected Query stringContains(Operation<?> operation, QueryMetadata metadata) {
        verifyArguments(operation);
        String field = toField(operation.getArg(0));
        boolean tokenized = isTokenized(operation.getArg(0));
        String[] terms = createEscapedTerms(operation.getArg(1), tokenized, metadata);
        if (terms.length > 1) {
            BooleanQuery bq = new BooleanQuery();
            for (String s : terms) {
                bq.add(new WildcardQuery(new Term(field, "*" + normalize(s) + "*")), Occur.MUST);
            }
            return bq;
        }
        return new WildcardQuery(new Term(field, "*" + normalize(terms[0]) + "*"));
    }

    protected Query endsWith(Operation<?> operation, QueryMetadata metadata) {
        verifyArguments(operation);
        String field = toField(operation.getArg(0));
        boolean tokenized = isTokenized(operation.getArg(0));
        String[] terms = createEscapedTerms(operation.getArg(1), tokenized, metadata);
        if (terms.length > 1) {
            BooleanQuery bq = new BooleanQuery();
            for (int i = 0; i < terms.length; ++i) {
                String s = i == terms.length - 1 ? "*" + terms[i] : "*" + terms[i] + "*";
                bq.add(new WildcardQuery(new Term(field, normalize(s))), Occur.MUST);
            }
            return bq;
        }
        return new WildcardQuery(new Term(field, "*" + normalize(terms[0])));
    }

    protected Query between(Operation<?> operation, QueryMetadata metadata) {
        verifyArguments(operation);
        // TODO Phrase not properly supported
        return range(toField(operation.getArg(0)), operation.getArg(1), operation.getArg(2), true, true, metadata);
    }

    protected Query lt(Operation<?> operation, QueryMetadata metadata) {
        verifyArguments(operation);
        return range(toField(operation.getArg(0)), null, operation.getArg(1), false, false, metadata);
    }

    protected Query gt(Operation<?> operation, QueryMetadata metadata) {
        verifyArguments(operation);
        return range(toField(operation.getArg(0)), operation.getArg(1), null, false, false, metadata);
    }

    protected Query le(Operation<?> operation, QueryMetadata metadata) {
        verifyArguments(operation);
        return range(toField(operation.getArg(0)), null, operation.getArg(1), true, true, metadata);
    }

    protected Query ge(Operation<?> operation, QueryMetadata metadata) {
        verifyArguments(operation);
        return range(toField(operation.getArg(0)), operation.getArg(1), null, true, true, metadata);
    }

    @SuppressWarnings("unchecked")
    protected Query range(String field, @Nullable Expression<?> min, @Nullable Expression<?> max, boolean minInc, boolean maxInc, QueryMetadata metadata) {
        if (min != null && Number.class.isAssignableFrom(min.getType()) || max != null
                && Number.class.isAssignableFrom(max.getType())) {
            Class<? extends Number> numType = (Class) (min != null ? min.getType() : max.getType());
            return numericRange((Class) numType, field, (Number) (min == null ? null
                    : ((Constant) min).getConstant()), (Number) (max == null ? null
                    : ((Constant) max).getConstant()), minInc, maxInc);
        }
        return stringRange(field, min, max, minInc, maxInc, metadata);
    }

    protected <N extends Number> NumericRangeQuery<?> numericRange(Class<N> clazz, String field,
            @Nullable N min, @Nullable N max, boolean minInc, boolean maxInc) {
        if (clazz.equals(Integer.class)) {
            return NumericRangeQuery.newIntRange(field, (Integer) min, (Integer) max, minInc, maxInc);
        } else if (clazz.equals(Double.class)) {
            return NumericRangeQuery.newDoubleRange(field, (Double) min, (Double) max, minInc, minInc);
        } else if (clazz.equals(Float.class)) {
            return NumericRangeQuery.newFloatRange(field, (Float) min, (Float) max, minInc, minInc);
        } else if (clazz.equals(Long.class)) {
            return NumericRangeQuery.newLongRange(field, (Long) min, (Long) max, minInc, minInc);
        } else if (clazz.equals(Byte.class) || clazz.equals(Short.class)) {
            return NumericRangeQuery.newIntRange(field, min != null ? min.intValue() : null,
                    max != null ? max.intValue() : null, minInc, maxInc);
        } else {
            throw new IllegalArgumentException("Unsupported numeric type " + clazz.getName());
        }
    }

    protected Query stringRange(String field, @Nullable Expression<?> min, @Nullable Expression<?> max,
            boolean minInc, boolean maxInc, QueryMetadata metadata) {
        if (min == null) {
            return new TermRangeQuery(field, null, normalize(createTerms(max, false, metadata)[0]), minInc, maxInc);
        } else if (max == null) {
            return new TermRangeQuery(field, normalize(createTerms(min, false, metadata)[0]), null, minInc, maxInc);
        } else {
            return new TermRangeQuery(field, normalize(createTerms(min, false, metadata)[0]), normalize(createTerms(max, false, metadata)[0]), minInc, maxInc);
        }
    }

    protected String toField(Expression<?> expr) {
        if (expr instanceof Path<?>) {
            return toField((Path<?>) expr);
        } else if (expr instanceof Operation<?>) {
            Operation<?> operation = (Operation<?>) expr;
            if (operation.getOperator() == Ops.LOWER || operation.getOperator() == Ops.UPPER) {
                return toField(operation.getArg(0));
            }
        }
        throw new IllegalArgumentException("Unable to transform " + expr + " to field");
    }

    /**
     * template method, override to customize
     * 
     * @param path
     * @return
     */
    protected String toField(Path<?> path) {
        String rv = path.getMetadata().getExpression().toString();
        if (path.getMetadata().getParent() != null){
            Path<?> parent = path.getMetadata().getParent();
            if (parent.getMetadata().getPathType() != PathType.VARIABLE){
                rv = toField(parent) + "." + rv;
            }
        }
        return rv;
    }
    
    protected boolean isTokenized(Expression<?> expr) {
        if (expr instanceof Path<?>) {
            return isTokenized((Path<?>) expr);
        } else if (expr instanceof Operation<?>) {
            Operation<?> operation = (Operation<?>) expr;
            if (operation.getOperator() == Ops.LOWER || operation.getOperator() == Ops.UPPER) {
                return isTokenized(operation.getArg(0));
            }
        } 
        return splitTerms;        
    }

    /**
     * template method, override to customize
     * 
     * @param arg
     * @return
     */
    protected boolean isTokenized(Path<?> arg) {
        return splitTerms;
    }

    private void verifyArguments(Operation<?> operation) {
        List<Expression<?>> arguments = operation.getArgs();
        for (int i = 1; i < arguments.size(); ++i) {
            if (!(arguments.get(i) instanceof Constant<?>)
                    && !(arguments.get(i) instanceof ParamExpression<?>)
                    && !(arguments.get(i) instanceof PhraseElement)
                    && !(arguments.get(i) instanceof TermElement)) {
                throw new IllegalArgumentException("operand was of unsupported type "
                        + arguments.get(i).getClass().getName());
            }
        }
    }

    private String[] createTerms(Expression<?> expr, boolean tokenized, QueryMetadata metadata) {
        if (expr instanceof ParamExpression<?>){
            Object value = metadata.getParams().get(expr);
            if (value == null){
                throw new ParamNotSetException((ParamExpression<?>) expr);
            }
            return split(expr, tokenized, value.toString());
        }else{
            return split(expr, tokenized, expr.toString());
{'fraction_non_alphanumeric': 0.12610121168923735, 'fraction_numerical': 0.003563791874554526, 'mean_word_length': 2.7192238362845935, 'pattern_counts': {'":': 0, '<': 103, '<?xml version=': 0, '>': 107, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '19239', 'n_tokens_mistral': 9961, 'n_tokens_neox': 9775, 'n_words': 2291}