<commit_before>/* Generated By:JJTree: Do not edit this line. OWhereClause.java Version 4.3 */
/* JavaCCOptions:MULTI=true,NODE_USES_PARSER=false,VISITOR=true,TRACK_TOKENS=true,NODE_PREFIX=O,NODE_EXTENDS=,NODE_FACTORY=,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */
package com.orientechnologies.orient.core.sql.parser;

import com.orientechnologies.common.collection.OMultiCollectionIterator;
import com.orientechnologies.common.util.OSizeable;
import com.orientechnologies.orient.core.command.OCommandContext;
import com.orientechnologies.orient.core.db.ODatabaseDocumentInternal;
import com.orientechnologies.orient.core.db.record.OIdentifiable;
import com.orientechnologies.orient.core.index.*;
import com.orientechnologies.orient.core.metadata.schema.OClass;
import com.orientechnologies.orient.core.metadata.schema.OType;
import com.orientechnologies.orient.core.sql.executor.OResult;
import com.orientechnologies.orient.core.sql.executor.OResultInternal;

import java.util.*;
import java.util.stream.Collectors;

public class OWhereClause extends SimpleNode {
  protected OBooleanExpression baseExpression;

  protected List<OAndBlock> flattened;

  public OWhereClause(int id) {
    super(id);
  }

  public OWhereClause(OrientSql p, int id) {
    super(p, id);
  }

  /**
   * Accept the visitor. *
   */
  public Object jjtAccept(OrientSqlVisitor visitor, Object data) {
    return visitor.visit(this, data);
  }

  public boolean matchesFilters(OIdentifiable currentRecord, OCommandContext ctx) {
    if (baseExpression == null) {
      return true;
    }
    return baseExpression.evaluate(currentRecord, ctx);
  }

  public boolean matchesFilters(OResult currentRecord, OCommandContext ctx) {
    if (baseExpression == null) {
      return true;
    }
    return baseExpression.evaluate(currentRecord, ctx);
  }

  public void toString(Map<Object, Object> params, StringBuilder builder) {
    if (baseExpression == null) {
      return;
    }
    baseExpression.toString(params, builder);
  }

  /**
   * estimates how many items of this class will be returned applying this filter
   *
   * @param oClass
   *
   * @return an estimation of the number of records of this class returned applying this filter, 0 if and only if sure that no
   * records are returned
   */
  public long estimate(OClass oClass, long threshold, OCommandContext ctx) {
    long count = oClass.count();
    if (count > 1) {
      count = count / 2;
    }
    if (count < threshold) {
      return count;
    }

    long indexesCount = 0l;
    List<OAndBlock> flattenedConditions = flatten();
    Set<OIndex<?>> indexes = oClass.getIndexes();
    for (OAndBlock condition : flattenedConditions) {

      List<OBinaryCondition> indexedFunctConditions = condition
          .getIndexedFunctionConditions(oClass, (ODatabaseDocumentInternal) ctx.getDatabase());

      long conditionEstimation = Long.MAX_VALUE;

      if (indexedFunctConditions != null) {
        for (OBinaryCondition cond : indexedFunctConditions) {
          OFromClause from = new OFromClause(-1);
          OFromItem item = new OFromItem(-1);
          from.item = item;
          from.item.setIdentifier(new OIdentifier(oClass.getName()));
          long newCount = cond.estimateIndexed(from, ctx);
          if (newCount < conditionEstimation) {
            conditionEstimation = newCount;
          }
        }
      } else {
        Map<String, Object> conditions = getEqualityOperations(condition, ctx);

        for (OIndex index : indexes) {
          List<String> indexedFields = index.getDefinition().getFields();
          int nMatchingKeys = 0;
          for (String indexedField : indexedFields) {
            if (conditions.containsKey(indexedField)) {
              nMatchingKeys++;
            } else {
              break;
            }
          }
          if (nMatchingKeys > 0) {
            long newCount = estimateFromIndex(index, conditions, nMatchingKeys);
            if (newCount < conditionEstimation) {
              conditionEstimation = newCount;
            }
          }
        }
      }
      if (conditionEstimation > count) {
        return count;<commit_msg>Fix MATCH execution with full text indexes<commit_after>/* Generated By:JJTree: Do not edit this line. OWhereClause.java Version 4.3 */
/* JavaCCOptions:MULTI=true,NODE_USES_PARSER=false,VISITOR=true,TRACK_TOKENS=true,NODE_PREFIX=O,NODE_EXTENDS=,NODE_FACTORY=,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */
package com.orientechnologies.orient.core.sql.parser;

import com.orientechnologies.common.collection.OMultiCollectionIterator;
import com.orientechnologies.common.util.OSizeable;
import com.orientechnologies.orient.core.command.OCommandContext;
import com.orientechnologies.orient.core.db.ODatabaseDocumentInternal;
import com.orientechnologies.orient.core.db.record.OIdentifiable;
import com.orientechnologies.orient.core.index.*;
import com.orientechnologies.orient.core.metadata.schema.OClass;
import com.orientechnologies.orient.core.metadata.schema.OType;
import com.orientechnologies.orient.core.sql.executor.OResult;
import com.orientechnologies.orient.core.sql.executor.OResultInternal;

import java.util.*;
import java.util.stream.Collectors;

public class OWhereClause extends SimpleNode {
  protected OBooleanExpression baseExpression;

  protected List<OAndBlock> flattened;

  public OWhereClause(int id) {
    super(id);
  }

  public OWhereClause(OrientSql p, int id) {
    super(p, id);
  }

  /**
   * Accept the visitor. *
   */
  public Object jjtAccept(OrientSqlVisitor visitor, Object data) {
    return visitor.visit(this, data);
  }

  public boolean matchesFilters(OIdentifiable currentRecord, OCommandContext ctx) {
    if (baseExpression == null) {
      return true;
    }
    return baseExpression.evaluate(currentRecord, ctx);
  }

  public boolean matchesFilters(OResult currentRecord, OCommandContext ctx) {
    if (baseExpression == null) {
      return true;
    }
    return baseExpression.evaluate(currentRecord, ctx);
  }

  public void toString(Map<Object, Object> params, StringBuilder builder) {
    if (baseExpression == null) {
      return;
    }
    baseExpression.toString(params, builder);
  }

  /**
   * estimates how many items of this class will be returned applying this filter
   *
   * @param oClass
   *
   * @return an estimation of the number of records of this class returned applying this filter, 0 if and only if sure that no
   * records are returned
   */
  public long estimate(OClass oClass, long threshold, OCommandContext ctx) {
    long count = oClass.count();
    if (count > 1) {
      count = count / 2;
    }
    if (count < threshold) {
      return count;
    }

    long indexesCount = 0l;
    List<OAndBlock> flattenedConditions = flatten();
    Set<OIndex<?>> indexes = oClass.getIndexes();
    for (OAndBlock condition : flattenedConditions) {

      List<OBinaryCondition> indexedFunctConditions = condition
          .getIndexedFunctionConditions(oClass, (ODatabaseDocumentInternal) ctx.getDatabase());

      long conditionEstimation = Long.MAX_VALUE;

      if (indexedFunctConditions != null) {
        for (OBinaryCondition cond : indexedFunctConditions) {
          OFromClause from = new OFromClause(-1);
          OFromItem item = new OFromItem(-1);
          from.item = item;
          from.item.setIdentifier(new OIdentifier(oClass.getName()));
          long newCount = cond.estimateIndexed(from, ctx);
          if (newCount < conditionEstimation) {
            conditionEstimation = newCount;
          }
        }
      } else {
        Map<String, Object> conditions = getEqualityOperations(condition, ctx);

        for (OIndex index : indexes) {
          if (index.getType().equals(OClass.INDEX_TYPE.FULLTEXT.name()) || index.getType()
              .equals(OClass.INDEX_TYPE.FULLTEXT_HASH_INDEX.name())) {
            continue;
          }
          List<String> indexedFields = index.getDefinition().getFields();
          int nMatchingKeys = 0;
          for (String indexedField : indexedFields) {
            if (conditions.containsKey(indexedField)) {
              nMatchingKeys++;
            } else {
              break;
            }
          }
          if (nMatchingKeys > 0) {
            long newCount = estimateFromIndex(index, conditions, nMatchingKeys);
            if (newCount < conditionEstimation) {
              conditionEstimation = newCount;
            }
          }
        }
      }
      if (conditionEstimation > count) {
        return count;
{'fraction_non_alphanumeric': 0.09592579546788775, 'fraction_numerical': 0.0023482446870963956, 'mean_word_length': 3.1249394673123487, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '422422', 'n_tokens_mistral': 2492, 'n_tokens_neox': 2425, 'n_words': 548}