<commit_before>package org.verdictdb.core.coordinator;

import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.verdictdb.core.connection.CachedMetaDataProvider;
import org.verdictdb.core.connection.DataTypeConverter;
import org.verdictdb.core.connection.DbmsConnection;
import org.verdictdb.core.connection.StaticMetaData;
import org.verdictdb.core.execution.ExecutablePlanRunner;
import org.verdictdb.core.querying.QueryExecutionPlan;
import org.verdictdb.core.querying.QueryExecutionPlanSimplifier;
import org.verdictdb.core.querying.ola.AsyncQueryExecutionPlan;
import org.verdictdb.core.resulthandler.ExecutionResultReader;
import org.verdictdb.core.scrambling.ScrambleMetaSet;
import org.verdictdb.core.sqlobject.*;
import org.verdictdb.exception.VerdictDBException;
import org.verdictdb.sqlreader.NonValidatingSQLParser;
import org.verdictdb.sqlreader.RelationStandardizer;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;

public class SelectQueryCoordinator {

  DbmsConnection conn;

  ScrambleMetaSet scrambleMetaSet;

  String defaultSchema = "tpch";

  StaticMetaData staticMetaData = new StaticMetaData();

  CachedMetaDataProvider cachedMetaData;

  public SelectQueryCoordinator(DbmsConnection conn) {
    this.conn = conn;
  }

  public SelectQuery standardizeQuery(String query) throws VerdictDBException {
    // parse the query
    RelationStandardizer.resetItemID();
    NonValidatingSQLParser sqlToRelation = new NonValidatingSQLParser();
    SelectQuery relation = (SelectQuery) sqlToRelation.toRelation(query);
    setStaticMetaData(relation);
    RelationStandardizer gen = new RelationStandardizer(staticMetaData);
    relation = gen.standardize(relation);
    return relation;
  }

  public ScrambleMetaSet getScrambleMetaSet() {
    return scrambleMetaSet;
  }

  public void setScrambleMetaSet(ScrambleMetaSet scrambleMetaSet) {
    this.scrambleMetaSet = scrambleMetaSet;
  }

  public StaticMetaData getStaticMetaData() {
    return staticMetaData;
  }

  public void setDefaultSchema(String defaultSchema) {
    this.defaultSchema = defaultSchema;
  }

  public void setStaticMetaData(SelectQuery relation) throws VerdictDBException {
    cachedMetaData = new CachedMetaData(conn);
    staticMetaData.setDefaultSchema(defaultSchema);
    // Extract all tables appeared in the query
    HashSet<BaseTable> tables = new HashSet<>();
    List<SelectQuery> queries = new ArrayList<>();
    queries.add(relation);
    while (!queries.isEmpty()) {
      SelectQuery query = queries.get(0);
      queries.remove(0);
      for (AbstractRelation t : query.getFromList()) {
        if (t instanceof BaseTable) tables.add((BaseTable) t);
        else if (t instanceof SelectQuery) queries.add((SelectQuery) t);
        else if (t instanceof JoinTable) {
          for (AbstractRelation join:((JoinTable) t).getJoinList()) {
            if (join instanceof BaseTable) tables.add((BaseTable) join);
            else if (join instanceof SelectQuery) queries.add((SelectQuery) join);
          }
        }
      }
      if (query.getFilter().isPresent()) {
        UnnamedColumn where = query.getFilter().get();
        List<UnnamedColumn> toCheck = new ArrayList<>();
        toCheck.add(where);
        while (!toCheck.isEmpty()) {
          UnnamedColumn col = toCheck.get(0);
          toCheck.remove(0);
          if (col instanceof ColumnOp) {
            toCheck.addAll(((ColumnOp) col).getOperands());
          }<commit_msg>minor<commit_after>package org.verdictdb.core.coordinator;

import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.verdictdb.core.connection.CachedMetaDataProvider;
import org.verdictdb.core.connection.DataTypeConverter;
import org.verdictdb.core.connection.DbmsConnection;
import org.verdictdb.core.connection.StaticMetaData;
import org.verdictdb.core.execution.ExecutablePlanRunner;
import org.verdictdb.core.querying.QueryExecutionPlan;
import org.verdictdb.core.querying.QueryExecutionPlanSimplifier;
import org.verdictdb.core.querying.ola.AsyncQueryExecutionPlan;
import org.verdictdb.core.resulthandler.ExecutionResultReader;
import org.verdictdb.core.scrambling.ScrambleMetaSet;
import org.verdictdb.core.sqlobject.*;
import org.verdictdb.exception.VerdictDBException;
import org.verdictdb.sqlreader.NonValidatingSQLParser;
import org.verdictdb.sqlreader.RelationStandardizer;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;

public class SelectQueryCoordinator {

  DbmsConnection conn;

  ScrambleMetaSet scrambleMetaSet;

  String defaultSchema = "tpch";

  StaticMetaData staticMetaData = new StaticMetaData();

  CachedMetaDataProvider cachedMetaData;

  public SelectQueryCoordinator(DbmsConnection conn) {
    this.conn = conn;
  }

  public SelectQuery standardizeQuery(String query) throws VerdictDBException {
    // parse the query
    RelationStandardizer.resetItemID();
    NonValidatingSQLParser sqlToRelation = new NonValidatingSQLParser();
    SelectQuery relation = (SelectQuery) sqlToRelation.toRelation(query);
    setStaticMetaData(relation);
    RelationStandardizer gen = new RelationStandardizer(staticMetaData);
    relation = gen.standardize(relation);
    return relation;
  }

  public ScrambleMetaSet getScrambleMetaSet() {
    return scrambleMetaSet;
  }

  public void setScrambleMetaSet(ScrambleMetaSet scrambleMetaSet) {
    this.scrambleMetaSet = scrambleMetaSet;
  }

  public StaticMetaData getStaticMetaData() {
    return staticMetaData;
  }

  public void setDefaultSchema(String defaultSchema) {
    this.defaultSchema = defaultSchema;
  }

  public void setStaticMetaData(SelectQuery relation) throws VerdictDBException {
    cachedMetaData = new CachedMetaDataProvider(conn);
    staticMetaData.setDefaultSchema(defaultSchema);
    // Extract all tables appeared in the query
    HashSet<BaseTable> tables = new HashSet<>();
    List<SelectQuery> queries = new ArrayList<>();
    queries.add(relation);
    while (!queries.isEmpty()) {
      SelectQuery query = queries.get(0);
      queries.remove(0);
      for (AbstractRelation t : query.getFromList()) {
        if (t instanceof BaseTable) tables.add((BaseTable) t);
        else if (t instanceof SelectQuery) queries.add((SelectQuery) t);
        else if (t instanceof JoinTable) {
          for (AbstractRelation join:((JoinTable) t).getJoinList()) {
            if (join instanceof BaseTable) tables.add((BaseTable) join);
            else if (join instanceof SelectQuery) queries.add((SelectQuery) join);
          }
        }
      }
      if (query.getFilter().isPresent()) {
        UnnamedColumn where = query.getFilter().get();
        List<UnnamedColumn> toCheck = new ArrayList<>();
        toCheck.add(where);
        while (!toCheck.isEmpty()) {
          UnnamedColumn col = toCheck.get(0);
          toCheck.remove(0);
          if (col instanceof ColumnOp) {
            toCheck.addAll(((ColumnOp) col).getOperands());
          }
{'fraction_non_alphanumeric': 0.09600679694137638, 'fraction_numerical': 0.0016992353440951572, 'mean_word_length': 4.259121370067014, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6204443', 'n_tokens_mistral': 2027, 'n_tokens_neox': 2002, 'n_words': 406}