<commit_before>/*
 * Licensed to STRATIO (C) under one or more contributor license agreements.
 * See the NOTICE file distributed with this work for additional information
 * regarding copyright ownership.  The STRATIO (C) licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package com.stratio.crossdata.core.normalizer;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.lang3.tuple.Pair;
import org.apache.log4j.Logger;

import com.stratio.crossdata.common.data.ColumnName;
import com.stratio.crossdata.common.data.IndexName;
import com.stratio.crossdata.common.data.TableName;
import com.stratio.crossdata.common.exceptions.IgnoreQueryException;
import com.stratio.crossdata.common.exceptions.ValidationException;
import com.stratio.crossdata.common.exceptions.validation.AmbiguousNameException;
import com.stratio.crossdata.common.exceptions.validation.BadFormatException;
import com.stratio.crossdata.common.exceptions.validation.NotExistNameException;
import com.stratio.crossdata.common.exceptions.validation.NotMatchDataTypeException;
import com.stratio.crossdata.common.exceptions.validation.NotValidCatalogException;
import com.stratio.crossdata.common.exceptions.validation.NotValidColumnException;
import com.stratio.crossdata.common.exceptions.validation.NotValidTableException;
import com.stratio.crossdata.common.exceptions.validation.YodaConditionException;
import com.stratio.crossdata.common.metadata.ColumnMetadata;
import com.stratio.crossdata.common.metadata.ColumnType;
import com.stratio.crossdata.common.metadata.DataType;
import com.stratio.crossdata.common.metadata.IndexMetadata;
import com.stratio.crossdata.common.metadata.IndexType;
import com.stratio.crossdata.common.metadata.TableMetadata;
import com.stratio.crossdata.common.statements.structures.AbstractRelation;
import com.stratio.crossdata.common.statements.structures.AliasSelector;
import com.stratio.crossdata.common.statements.structures.CaseWhenSelector;
import com.stratio.crossdata.common.statements.structures.ColumnSelector;
import com.stratio.crossdata.common.statements.structures.FunctionSelector;
import com.stratio.crossdata.common.statements.structures.GroupSelector;
import com.stratio.crossdata.common.statements.structures.Operator;
import com.stratio.crossdata.common.statements.structures.OrderByClause;
import com.stratio.crossdata.common.statements.structures.Relation;
import com.stratio.crossdata.common.statements.structures.RelationDisjunction;
import com.stratio.crossdata.common.statements.structures.RelationSelector;
import com.stratio.crossdata.common.statements.structures.SelectExpression;
import com.stratio.crossdata.common.statements.structures.Selector;
import com.stratio.crossdata.common.statements.structures.SelectorType;
import com.stratio.crossdata.common.utils.Constants;
import com.stratio.crossdata.core.metadata.MetadataManager;
import com.stratio.crossdata.core.query.SelectParsedQuery;
import com.stratio.crossdata.core.query.SelectValidatedQuery;
import com.stratio.crossdata.core.statements.SelectStatement;
import com.stratio.crossdata.core.structures.ExtendedSelectSelector;
import com.stratio.crossdata.core.structures.GroupByClause;
import com.stratio.crossdata.core.structures.InnerJoin;
import com.stratio.crossdata.core.validator.Validator;

import static com.stratio.crossdata.common.statements.structures.SelectorType.FUNCTION;

/**
 * Normalizator Class.
 */
public class Normalizator {

    /**
     * Class logger.
     */
    private static final Logger LOG = Logger.getLogger(Normalizator.class);

    private NormalizedFields fields = new NormalizedFields();

    private SelectParsedQuery parsedQuery;

    private Normalizator subqueryNormalizator;

    /**
     * Class Constructor.
     *
     * @param parsedQuery The parsed query.
     */
    public Normalizator(SelectParsedQuery parsedQuery) {
        this.parsedQuery = parsedQuery;
    }

    /**
     * Get the parsed query to Normalize.
     *
     * @return com.stratio.crossdata.core.query.SelectParsedQuery;
     */
    public SelectParsedQuery getParsedQuery() {
        return parsedQuery;
    }

    /**
     * Get the Normalizator of the subquery.
     *
     * @return com.stratio.crossdata.core.normalizer.Normalizator;
     */
    public Normalizator getSubqueryNormalizator() {
        return subqueryNormalizator;
    }

    /**
     * Get the obtained fields normalized.
     *
     * @return com.stratio.crossdata.core.normalizer.NormalizerFields
     */
    public NormalizedFields getFields() {
        return fields;
    }

    /**
     * Execute the normalization of a parsed query.
     *
     * @throws ValidationException
     */
    public void execute(List<TableName> parentsTableNames) throws ValidationException {

        if (parsedQuery.getStatement().isSubqueryInc()) {
            subqueryNormalizator = new Normalizator(parsedQuery.getChildParsedQuery());
            subqueryNormalizator.execute(parentsTableNames);
            checkSubquerySelectors(subqueryNormalizator.getFields().getSelectors());
        }

        fields.getTableNames().addAll(parentsTableNames);
        fields.setPreferredTableNames(new HashSet<>(parsedQuery.getStatement().getFromTables()));

        normalizeTables();
        normalizeSelectExpression();
        normalizeWhere();
        normalizeJoins();
        normalizeOrderBy();
        normalizeGroupBy();
        normalizeHaving();
        validateColumnsScope();
    }

    private void checkSubquerySelectors(List<Selector> selectors) throws ValidationException {
        Set<String> uniqueSelectors = new HashSet<>();

        for (Selector selector : selectors) {
            if (selector.getAlias() != null) {
                if (!uniqueSelectors.add(selector.getAlias())) {
                    throw new AmbiguousNameException("The alias [" + selector.getAlias() + "] is duplicate");
                }
            } else if (selector instanceof ColumnSelector) {
                if (!uniqueSelectors.add(selector.getColumnName().getName())) {
                    throw new AmbiguousNameException(
                            "The selector [" + selector.getColumnName().getName() + "] is ambiguous. Try using alias");
                }
            } else {
                if (!uniqueSelectors.add(selector.getStringValue())) {
                    throw new AmbiguousNameException(
                            "The selector [" + selector.getStringValue() + "] is duplicate. Try using alias");
                }
            }
        }
    }

    /**
     * Normalize the tables of a query.
     *
     * @throws ValidationException
     */
    public void normalizeTables() throws ValidationException {
        List<TableName> tableNames = parsedQuery.getStatement().getFromTables();
        if (tableNames != null && !tableNames.isEmpty()) {
            normalizeTables(tableNames);
        }
    }

    /**
     * Normalize the "from" tables of a parsed query.
     *
     * @param fromTables List of the from clause tables of a parsed query
     * @throws ValidationException
     */
    public void normalizeTables(List<TableName> fromTables) throws ValidationException {
        for (TableName tableName : fromTables) {
            if (!tableName.isVirtual()) {
                checkTable(tableName);
            }
            fields.getCatalogNames().add(tableName.getCatalogName());
            fields.addTableName(tableName);
        }
    }

    /**
     * Normalize all the joins of a parsed query.
     *
     * @throws ValidationException
     */
    public void normalizeJoins() throws ValidationException {
        List<InnerJoin> innerJoinList = parsedQuery.getStatement().getJoinList();
        if (!innerJoinList.isEmpty()) {
            addImplicitJoins(innerJoinList);
            for (InnerJoin innerJoin: innerJoinList) {
                normalizeJoins(innerJoin);
                fields.addJoin(innerJoin);
            }
        }
    }

    private void addImplicitJoins(List<InnerJoin> innerJoinList) {

        if(fields.getImplicitWhere() == null || fields.getImplicitWhere().isEmpty()){
            return;
        }

        // Clone normalized tables
        List<TableName> tableNames = new ArrayList();
        tableNames.addAll(fields.getTableNames());

        // Remove the "regular" table
        TableName currentTable = tableNames.get(0);
        tableNames.remove(0);

        // Tables pending to be related to a implicit relation
        List<TableName> tablesToBeRelated = new ArrayList();
        tablesToBeRelated.addAll(tableNames);

        // Clone implicit relations from where clause
        List<Relation> implicitRelations = new ArrayList<>();
        implicitRelations.addAll(fields.getImplicitWhere());

        while(!tablesToBeRelated.isEmpty() || !implicitRelations.isEmpty()){
            // Assign relations to corresponding join
            Iterator<Relation> iter = implicitRelations.iterator();
            Relation rel = null;
            while(iter.hasNext()){
                rel = iter.next();
                boolean tableFound = false;
                if(rel.getLeftTerm().getTableName().equals(currentTable)){
                    currentTable = rel.getRightTerm().getTableName();
                    tableFound = true;
                } else if(rel.getRightTerm().getTableName().equals(currentTable)){
                    currentTable = rel.getLeftTerm().getTableName();
                    tableFound = true;
                }
                if(tableFound){
                    for(InnerJoin innerJoin: innerJoinList){
                        if(innerJoin.getTablename().equals(currentTable)){
                            innerJoin.addRelation(rel);
                            tablesToBeRelated.remove(currentTable);
                            break;
                        }
                    }
                }
            }
            implicitRelations.remove(rel);
        }
        /*for(TableName tn: tableNames){

            // Find implicit relations where the current table name is involved
            // and choose the one that it has also a table that was already
            // selected in a previous implicit relation
            Iterator<Relation> iter = implicitRelations.iterator();
            Relation rel = null;
            while(iter.hasNext()){
                rel = iter.next();
                if(rel.getLeftTerm().getTableName().equals(tn)){
                    if(!tablesToBeRelated.contains(rel.getRightTerm().getTableName())){
                        for(InnerJoin innerJoin: innerJoinList){
                            if(innerJoin.getTablename().equals(tn)){
                                innerJoin.addRelation(rel);
                                implicitRelations.remove(rel);
                                break;
                            }
                        }
                    }
                } else if(rel.getRightTerm().getTableName().equals(tn)){
                    if(!tablesToBeRelated.contains(rel.getLeftTerm().getTableName())){
                        for(InnerJoin innerJoin: innerJoinList){
                            if(innerJoin.getTablename().equals(tn)){
                                innerJoin.addRelation(rel);
                                implicitRelations.remove(rel);
                                break;
                            }
                        }
                    }
                }
            }
            implicitRelations.remove(rel);
            tablesToBeRelated.remove(tn);
        }*/

        // Assign the rest (if any) of unassigned implicit relations
        if(!implicitRelations.isEmpty()){
            for(Relation implicitRelation: implicitRelations){
                for(InnerJoin innerJoin: innerJoinList){
                    Relation alreadyAssignedRelation = (Relation) innerJoin.getRelations().get(0);
                    TableName leftTable = alreadyAssignedRelation.getLeftTerm().getTableName();
                    TableName rightTable = alreadyAssignedRelation.getRightTerm().getTableName();
                    // Check if tables of the join are equals to the tables of the implicit relation
                    if((leftTable.equals(implicitRelation.getLeftTerm().getTableName())
                            && rightTable.equals(implicitRelation.getRightTerm().getTableName()))
                        || (rightTable.equals(implicitRelation.getLeftTerm().getTableName())
                            && leftTable.equals(implicitRelation.getRightTerm().getTableName()))){
                        innerJoin.addRelation(implicitRelation);
                    }
                }
            }
        }
    }

    /**
     * Normalize a specific inner join of a parsed query.
     *
     * @param innerJoin The inner join
     * @throws ValidationException
     */
    public void normalizeJoins(InnerJoin innerJoin) throws ValidationException {
        checkJoinRelations(innerJoin.getRelations());
    }

    private void normalizeWhere() throws ValidationException {
        List<AbstractRelation> whereClauses = parsedQuery.getStatement().getWhere();
        if ((whereClauses != null) && (!whereClauses.isEmpty())) {
            normalizeWhere(whereClauses);
            fields.setWhere(whereClauses);
        }
    }

    private void normalizeWhere(List<AbstractRelation> where) throws ValidationException {
        checkWhereRelations(where);
    }

    /**
     * Normalize the order by clause of a parsed query.
     *
     * @throws ValidationException
     */
    public void normalizeOrderBy() throws ValidationException {
        List<OrderByClause> orderByClauseClauses = parsedQuery.getStatement().getOrderByClauses();
        if (orderByClauseClauses != null) {
            normalizeOrderBy(orderByClauseClauses);
            fields.setOrderByClauses(orderByClauseClauses);
        }
    }

    /**
     * Normalize an specific order by of a parsed query.
     *
     * @param orderByClauseClauses The order by
     * @throws ValidationException
     */
    public void normalizeOrderBy(List<OrderByClause> orderByClauseClauses) throws ValidationException {
        for (OrderByClause orderBy : orderByClauseClauses) {
            Selector selector = orderBy.getSelector();
            switch (selector.getType()) {
            case COLUMN:
                Selector referencedSelector = findReferencedSelector(((ColumnSelector) selector).getName().getName());
                if (referencedSelector != null ) {
                    orderBy.setSelector(new AliasSelector(referencedSelector));
                }else {
                    checkColumnSelector((ColumnSelector) selector);
                }
                break;
            case FUNCTION:
            case ASTERISK:
            case BOOLEAN:
            case STRING:
            case INTEGER:
            case FLOATING_POINT:
                throw new BadFormatException("Order by only accepts columns");
            }

        }
    }

    /**
     * Normalize de select clause of a parsed query.
     *
     * @throws ValidationException
     */
    public void normalizeSelectExpression() throws ValidationException {
        SelectExpression selectExpression = parsedQuery.getStatement().getSelectExpression();
        if (selectExpression != null) {
            normalizeSelectExpression(selectExpression);
        }
    }

    /**
     * Normalize an specific select expression.
     *
     * @param selectExpression The select expression
     * @throws ValidationException
     */
    public void normalizeSelectExpression(
            SelectExpression selectExpression) throws ValidationException {
        List<Selector> normalizeSelectors = checkListSelector(selectExpression.getSelectorList());
        fields.getSelectors().addAll(normalizeSelectors);
        selectExpression.getSelectorList().clear();
        selectExpression.getSelectorList().addAll(normalizeSelectors);
    }

    /**
     * Normalize the group by of a parsed query.
     *
     * @throws ValidationException
     */
    public void normalizeGroupBy() throws ValidationException {
        //Ensure that there is not any case-when selector
        for (Selector s : fields.getSelectors()) {
            if (CaseWhenSelector.class.isInstance(s)) {
                throw new BadFormatException("Group By clause is not possible with Case When Selector");
            }
        }

        GroupByClause groupByClause = parsedQuery.getStatement().getGroupByClause();
        if (groupByClause != null) {
            normalizeGroupBy(groupByClause);
            fields.setGroupByClause(groupByClause);
        }
    }


    /**
     * Normalize an specific group by of a parsed query.
     *
     * @param groupByClause
     * @throws ValidationException
     */
    public void normalizeGroupBy(GroupByClause groupByClause) throws ValidationException {
        Set<ColumnName> columnNames = new HashSet<>();

        checkGBFormatBySelectorIdentifier(groupByClause.getSelectorIdentifier(), columnNames);

        // Check if all columns are correct
        for (Selector selector : fields.getSelectors()) {
            checkGroupByColumns(selector, columnNames);
        }
    }


    private void checkGBFormatBySelectorIdentifier(List<Selector> selectorList, Set<ColumnName> columnNames)
                    throws ValidationException {
        int selectorIndex = 0;

        for (Selector selector : selectorList) {
            switch (selector.getType()) {
            case FUNCTION:
                throw new BadFormatException("Function include into groupBy is not valid");
            case COLUMN:
                Selector referencedSelector = findReferencedSelector(((ColumnSelector) selector).getName().getName());
                if (referencedSelector != null ) {
                    selectorList.set(selectorIndex, new AliasSelector(referencedSelector));
                }else{
                    checkColumnSelector((ColumnSelector) selector);
                    if (!columnNames.add(((ColumnSelector) selector).getName())) {
                        throw new BadFormatException("COLUMN into group by is repeated");
                    }
                }
                break;
            case ASTERISK:
                throw new BadFormatException("Asterisk include into groupBy is not valid");
            }
            selectorIndex++;
        }
    }

    private Selector findReferencedSelector(String selectorAlias) {
        Selector referencedSelector = null;
        boolean aliasFound = false;
        Iterator<Selector> normalizedSelectExpressionIterator = parsedQuery.getStatement().getSelectExpression().getSelectorList().iterator();

        while (normalizedSelectExpressionIterator.hasNext() && !aliasFound){
            referencedSelector = normalizedSelectExpressionIterator.next();
            if( referencedSelector.getType() != SelectorType.COLUMN && referencedSelector.getAlias().equals(selectorAlias)){
                aliasFound = true;
            }
        }
        return aliasFound ? referencedSelector : null;
    }

    /**
     * Normalize the having of a parsed query.
     *
     * @throws ValidationException
     */
    public void normalizeHaving() throws ValidationException {
        List<AbstractRelation> havingClause = parsedQuery.getStatement().getHavingClause();
        if (havingClause != null) {

            if (!parsedQuery.getStatement().isGroupInc()) {
                throw new BadFormatException("Having clause requires Group By clause.");
            }

            List<AbstractRelation> havingClauses = parsedQuery.getStatement().getHavingClause();
            if ((havingClauses != null) && (!havingClauses.isEmpty())) {
                normalizeHaving(havingClauses);
                fields.setHavingClause(havingClauses);
            }
        }
    }


    private void checkFormatBySelectorIdentifierHaving(Selector selector, Set<ColumnName> columnNames)
            throws ValidationException {
        switch (selector.getType()) {<commit_msg>bug fixed caused by aliasSelector feature/CROSSDATA-589<commit_after>/*
 * Licensed to STRATIO (C) under one or more contributor license agreements.
 * See the NOTICE file distributed with this work for additional information
 * regarding copyright ownership.  The STRATIO (C) licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package com.stratio.crossdata.core.normalizer;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.lang3.tuple.Pair;
import org.apache.log4j.Logger;

import com.stratio.crossdata.common.data.ColumnName;
import com.stratio.crossdata.common.data.IndexName;
import com.stratio.crossdata.common.data.TableName;
import com.stratio.crossdata.common.exceptions.IgnoreQueryException;
import com.stratio.crossdata.common.exceptions.ValidationException;
import com.stratio.crossdata.common.exceptions.validation.AmbiguousNameException;
import com.stratio.crossdata.common.exceptions.validation.BadFormatException;
import com.stratio.crossdata.common.exceptions.validation.NotExistNameException;
import com.stratio.crossdata.common.exceptions.validation.NotMatchDataTypeException;
import com.stratio.crossdata.common.exceptions.validation.NotValidCatalogException;
import com.stratio.crossdata.common.exceptions.validation.NotValidColumnException;
import com.stratio.crossdata.common.exceptions.validation.NotValidTableException;
import com.stratio.crossdata.common.exceptions.validation.YodaConditionException;
import com.stratio.crossdata.common.metadata.ColumnMetadata;
import com.stratio.crossdata.common.metadata.ColumnType;
import com.stratio.crossdata.common.metadata.DataType;
import com.stratio.crossdata.common.metadata.IndexMetadata;
import com.stratio.crossdata.common.metadata.IndexType;
import com.stratio.crossdata.common.metadata.TableMetadata;
import com.stratio.crossdata.common.statements.structures.AbstractRelation;
import com.stratio.crossdata.common.statements.structures.AliasSelector;
import com.stratio.crossdata.common.statements.structures.CaseWhenSelector;
import com.stratio.crossdata.common.statements.structures.ColumnSelector;
import com.stratio.crossdata.common.statements.structures.FunctionSelector;
import com.stratio.crossdata.common.statements.structures.GroupSelector;
import com.stratio.crossdata.common.statements.structures.Operator;
import com.stratio.crossdata.common.statements.structures.OrderByClause;
import com.stratio.crossdata.common.statements.structures.Relation;
import com.stratio.crossdata.common.statements.structures.RelationDisjunction;
import com.stratio.crossdata.common.statements.structures.RelationSelector;
import com.stratio.crossdata.common.statements.structures.SelectExpression;
import com.stratio.crossdata.common.statements.structures.Selector;
import com.stratio.crossdata.common.statements.structures.SelectorType;
import com.stratio.crossdata.common.utils.Constants;
import com.stratio.crossdata.core.metadata.MetadataManager;
import com.stratio.crossdata.core.query.SelectParsedQuery;
import com.stratio.crossdata.core.query.SelectValidatedQuery;
import com.stratio.crossdata.core.statements.SelectStatement;
import com.stratio.crossdata.core.structures.ExtendedSelectSelector;
import com.stratio.crossdata.core.structures.GroupByClause;
import com.stratio.crossdata.core.structures.InnerJoin;
import com.stratio.crossdata.core.validator.Validator;

import static com.stratio.crossdata.common.statements.structures.SelectorType.FUNCTION;

/**
 * Normalizator Class.
 */
public class Normalizator {

    /**
     * Class logger.
     */
    private static final Logger LOG = Logger.getLogger(Normalizator.class);

    private NormalizedFields fields = new NormalizedFields();

    private SelectParsedQuery parsedQuery;

    private Normalizator subqueryNormalizator;

    /**
     * Class Constructor.
     *
     * @param parsedQuery The parsed query.
     */
    public Normalizator(SelectParsedQuery parsedQuery) {
        this.parsedQuery = parsedQuery;
    }

    /**
     * Get the parsed query to Normalize.
     *
     * @return com.stratio.crossdata.core.query.SelectParsedQuery;
     */
    public SelectParsedQuery getParsedQuery() {
        return parsedQuery;
    }

    /**
     * Get the Normalizator of the subquery.
     *
     * @return com.stratio.crossdata.core.normalizer.Normalizator;
     */
    public Normalizator getSubqueryNormalizator() {
        return subqueryNormalizator;
    }

    /**
     * Get the obtained fields normalized.
     *
     * @return com.stratio.crossdata.core.normalizer.NormalizerFields
     */
    public NormalizedFields getFields() {
        return fields;
    }

    /**
     * Execute the normalization of a parsed query.
     *
     * @throws ValidationException
     */
    public void execute(List<TableName> parentsTableNames) throws ValidationException {

        if (parsedQuery.getStatement().isSubqueryInc()) {
            subqueryNormalizator = new Normalizator(parsedQuery.getChildParsedQuery());
            subqueryNormalizator.execute(parentsTableNames);
            checkSubquerySelectors(subqueryNormalizator.getFields().getSelectors());
        }

        fields.getTableNames().addAll(parentsTableNames);
        fields.setPreferredTableNames(new HashSet<>(parsedQuery.getStatement().getFromTables()));

        normalizeTables();
        normalizeSelectExpression();
        normalizeWhere();
        normalizeJoins();
        normalizeOrderBy();
        normalizeGroupBy();
        normalizeHaving();
        validateColumnsScope();
    }

    private void checkSubquerySelectors(List<Selector> selectors) throws ValidationException {
        Set<String> uniqueSelectors = new HashSet<>();

        for (Selector selector : selectors) {
            if (selector.getAlias() != null) {
                if (!uniqueSelectors.add(selector.getAlias())) {
                    throw new AmbiguousNameException("The alias [" + selector.getAlias() + "] is duplicate");
                }
            } else if (selector instanceof ColumnSelector) {
                if (!uniqueSelectors.add(selector.getColumnName().getName())) {
                    throw new AmbiguousNameException(
                            "The selector [" + selector.getColumnName().getName() + "] is ambiguous. Try using alias");
                }
            } else {
                if (!uniqueSelectors.add(selector.getStringValue())) {
                    throw new AmbiguousNameException(
                            "The selector [" + selector.getStringValue() + "] is duplicate. Try using alias");
                }
            }
        }
    }

    /**
     * Normalize the tables of a query.
     *
     * @throws ValidationException
     */
    public void normalizeTables() throws ValidationException {
        List<TableName> tableNames = parsedQuery.getStatement().getFromTables();
        if (tableNames != null && !tableNames.isEmpty()) {
            normalizeTables(tableNames);
        }
    }

    /**
     * Normalize the "from" tables of a parsed query.
     *
     * @param fromTables List of the from clause tables of a parsed query
     * @throws ValidationException
     */
    public void normalizeTables(List<TableName> fromTables) throws ValidationException {
        for (TableName tableName : fromTables) {
            if (!tableName.isVirtual()) {
                checkTable(tableName);
            }
            fields.getCatalogNames().add(tableName.getCatalogName());
            fields.addTableName(tableName);
        }
    }

    /**
     * Normalize all the joins of a parsed query.
     *
     * @throws ValidationException
     */
    public void normalizeJoins() throws ValidationException {
        List<InnerJoin> innerJoinList = parsedQuery.getStatement().getJoinList();
        if (!innerJoinList.isEmpty()) {
            addImplicitJoins(innerJoinList);
            for (InnerJoin innerJoin: innerJoinList) {
                normalizeJoins(innerJoin);
                fields.addJoin(innerJoin);
            }
        }
    }

    private void addImplicitJoins(List<InnerJoin> innerJoinList) {

        if(fields.getImplicitWhere() == null || fields.getImplicitWhere().isEmpty()){
            return;
        }

        // Clone normalized tables
        List<TableName> tableNames = new ArrayList();
        tableNames.addAll(fields.getTableNames());

        // Remove the "regular" table
        TableName currentTable = tableNames.get(0);
        tableNames.remove(0);

        // Tables pending to be related to a implicit relation
        List<TableName> tablesToBeRelated = new ArrayList();
        tablesToBeRelated.addAll(tableNames);

        // Clone implicit relations from where clause
        List<Relation> implicitRelations = new ArrayList<>();
        implicitRelations.addAll(fields.getImplicitWhere());

        while(!tablesToBeRelated.isEmpty() || !implicitRelations.isEmpty()){
            // Assign relations to corresponding join
            Iterator<Relation> iter = implicitRelations.iterator();
            Relation rel = null;
            while(iter.hasNext()){
                rel = iter.next();
                boolean tableFound = false;
                if(rel.getLeftTerm().getTableName().equals(currentTable)){
                    currentTable = rel.getRightTerm().getTableName();
                    tableFound = true;
                } else if(rel.getRightTerm().getTableName().equals(currentTable)){
                    currentTable = rel.getLeftTerm().getTableName();
                    tableFound = true;
                }
                if(tableFound){
                    for(InnerJoin innerJoin: innerJoinList){
                        if(innerJoin.getTablename().equals(currentTable)){
                            innerJoin.addRelation(rel);
                            tablesToBeRelated.remove(currentTable);
                            break;
                        }
                    }
                }
            }
            implicitRelations.remove(rel);
        }
        /*for(TableName tn: tableNames){

            // Find implicit relations where the current table name is involved
            // and choose the one that it has also a table that was already
            // selected in a previous implicit relation
            Iterator<Relation> iter = implicitRelations.iterator();
            Relation rel = null;
            while(iter.hasNext()){
                rel = iter.next();
                if(rel.getLeftTerm().getTableName().equals(tn)){
                    if(!tablesToBeRelated.contains(rel.getRightTerm().getTableName())){
                        for(InnerJoin innerJoin: innerJoinList){
                            if(innerJoin.getTablename().equals(tn)){
                                innerJoin.addRelation(rel);
                                implicitRelations.remove(rel);
                                break;
                            }
                        }
                    }
                } else if(rel.getRightTerm().getTableName().equals(tn)){
                    if(!tablesToBeRelated.contains(rel.getLeftTerm().getTableName())){
                        for(InnerJoin innerJoin: innerJoinList){
                            if(innerJoin.getTablename().equals(tn)){
                                innerJoin.addRelation(rel);
                                implicitRelations.remove(rel);
                                break;
                            }
                        }
                    }
                }
            }
            implicitRelations.remove(rel);
            tablesToBeRelated.remove(tn);
        }*/

        // Assign the rest (if any) of unassigned implicit relations
        if(!implicitRelations.isEmpty()){
            for(Relation implicitRelation: implicitRelations){
                for(InnerJoin innerJoin: innerJoinList){
                    Relation alreadyAssignedRelation = (Relation) innerJoin.getRelations().get(0);
                    TableName leftTable = alreadyAssignedRelation.getLeftTerm().getTableName();
                    TableName rightTable = alreadyAssignedRelation.getRightTerm().getTableName();
                    // Check if tables of the join are equals to the tables of the implicit relation
                    if((leftTable.equals(implicitRelation.getLeftTerm().getTableName())
                            && rightTable.equals(implicitRelation.getRightTerm().getTableName()))
                        || (rightTable.equals(implicitRelation.getLeftTerm().getTableName())
                            && leftTable.equals(implicitRelation.getRightTerm().getTableName()))){
                        innerJoin.addRelation(implicitRelation);
                    }
                }
            }
        }
    }

    /**
     * Normalize a specific inner join of a parsed query.
     *
     * @param innerJoin The inner join
     * @throws ValidationException
     */
    public void normalizeJoins(InnerJoin innerJoin) throws ValidationException {
        checkJoinRelations(innerJoin.getRelations());
    }

    private void normalizeWhere() throws ValidationException {
        List<AbstractRelation> whereClauses = parsedQuery.getStatement().getWhere();
        if ((whereClauses != null) && (!whereClauses.isEmpty())) {
            normalizeWhere(whereClauses);
            fields.setWhere(whereClauses);
        }
    }

    private void normalizeWhere(List<AbstractRelation> where) throws ValidationException {
        checkWhereRelations(where);
    }

    /**
     * Normalize the order by clause of a parsed query.
     *
     * @throws ValidationException
     */
    public void normalizeOrderBy() throws ValidationException {
        List<OrderByClause> orderByClauseClauses = parsedQuery.getStatement().getOrderByClauses();
        if (orderByClauseClauses != null) {
            normalizeOrderBy(orderByClauseClauses);
            fields.setOrderByClauses(orderByClauseClauses);
        }
    }

    /**
     * Normalize an specific order by of a parsed query.
     *
     * @param orderByClauseClauses The order by
     * @throws ValidationException
     */
    public void normalizeOrderBy(List<OrderByClause> orderByClauseClauses) throws ValidationException {
        for (OrderByClause orderBy : orderByClauseClauses) {
            Selector selector = orderBy.getSelector();
            switch (selector.getType()) {
            case COLUMN:
                Selector referencedSelector = findReferencedSelector(((ColumnSelector) selector).getName().getName());
                if (referencedSelector != null ) {
                    orderBy.setSelector(new AliasSelector(referencedSelector));
                }else {
                    checkColumnSelector((ColumnSelector) selector);
                }
                break;
            case FUNCTION:
            case ASTERISK:
            case BOOLEAN:
            case STRING:
            case INTEGER:
            case FLOATING_POINT:
                throw new BadFormatException("Order by only accepts columns");
            }

        }
    }

    /**
     * Normalize de select clause of a parsed query.
     *
     * @throws ValidationException
     */
    public void normalizeSelectExpression() throws ValidationException {
        SelectExpression selectExpression = parsedQuery.getStatement().getSelectExpression();
        if (selectExpression != null) {
            normalizeSelectExpression(selectExpression);
        }
    }

    /**
     * Normalize an specific select expression.
     *
     * @param selectExpression The select expression
     * @throws ValidationException
     */
    public void normalizeSelectExpression(
            SelectExpression selectExpression) throws ValidationException {
        List<Selector> normalizeSelectors = checkListSelector(selectExpression.getSelectorList());
        fields.getSelectors().addAll(normalizeSelectors);
        selectExpression.getSelectorList().clear();
        selectExpression.getSelectorList().addAll(normalizeSelectors);
    }

    /**
     * Normalize the group by of a parsed query.
     *
     * @throws ValidationException
     */
    public void normalizeGroupBy() throws ValidationException {
        //Ensure that there is not any case-when selector
        for (Selector s : fields.getSelectors()) {
            if (CaseWhenSelector.class.isInstance(s)) {
                throw new BadFormatException("Group By clause is not possible with Case When Selector");
            }
        }

        GroupByClause groupByClause = parsedQuery.getStatement().getGroupByClause();
        if (groupByClause != null) {
            normalizeGroupBy(groupByClause);
            fields.setGroupByClause(groupByClause);
        }
    }


    /**
     * Normalize an specific group by of a parsed query.
     *
     * @param groupByClause
     * @throws ValidationException
     */
    public void normalizeGroupBy(GroupByClause groupByClause) throws ValidationException {
        Set<ColumnName> columnNames = new HashSet<>();

        checkGBFormatBySelectorIdentifier(groupByClause.getSelectorIdentifier(), columnNames);

        // Check if all columns are correct
        for (Selector selector : fields.getSelectors()) {
            checkGroupByColumns(selector, columnNames);
        }
    }


    private void checkGBFormatBySelectorIdentifier(List<Selector> selectorList, Set<ColumnName> columnNames)
                    throws ValidationException {
        int selectorIndex = 0;

        for (Selector selector : selectorList) {
            switch (selector.getType()) {
            case FUNCTION:
                throw new BadFormatException("Function include into groupBy is not valid");
            case COLUMN:
                Selector referencedSelector = findReferencedSelector(((ColumnSelector) selector).getName().getName());
                if (referencedSelector != null ) {
                    selectorList.set(selectorIndex, new AliasSelector(referencedSelector));
                }else{
                    checkColumnSelector((ColumnSelector) selector);
                    if (!columnNames.add(((ColumnSelector) selector).getName())) {
                        throw new BadFormatException("COLUMN into group by is repeated");
                    }
                }
                break;
            case ASTERISK:
                throw new BadFormatException("Asterisk include into groupBy is not valid");
            }
            selectorIndex++;
        }
    }

    private Selector findReferencedSelector(String selectorAlias) {
        Selector referencedSelector = null;
        boolean aliasFound = false;
        Iterator<Selector> normalizedSelectExpressionIterator = parsedQuery.getStatement().getSelectExpression().getSelectorList().iterator();

        while (normalizedSelectExpressionIterator.hasNext() && !aliasFound){
            referencedSelector = normalizedSelectExpressionIterator.next();
            if( referencedSelector.getType() != SelectorType.COLUMN && referencedSelector.getAlias() != null && referencedSelector.getAlias().equals(selectorAlias)){
                aliasFound = true;
            }
        }
        return aliasFound ? referencedSelector : null;
    }

    /**
     * Normalize the having of a parsed query.
     *
     * @throws ValidationException
     */
    public void normalizeHaving() throws ValidationException {
        List<AbstractRelation> havingClause = parsedQuery.getStatement().getHavingClause();
        if (havingClause != null) {

            if (!parsedQuery.getStatement().isGroupInc()) {
                throw new BadFormatException("Having clause requires Group By clause.");
            }

            List<AbstractRelation> havingClauses = parsedQuery.getStatement().getHavingClause();
            if ((havingClauses != null) && (!havingClauses.isEmpty())) {
                normalizeHaving(havingClauses);
                fields.setHavingClause(havingClauses);
            }
        }
    }


    private void checkFormatBySelectorIdentifierHaving(Selector selector, Set<ColumnName> columnNames)
            throws ValidationException {
        switch (selector.getType()) {
{'fraction_non_alphanumeric': 0.09083487406055658, 'fraction_numerical': 0.0005522606670348404, 'mean_word_length': 2.4235922729140977, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5664346', 'n_tokens_mistral': 10472, 'n_tokens_neox': 10032, 'n_words': 2159}