<commit_before>package org.jkiss.dbeaver.model.sql.format.tokenized;

import org.jkiss.dbeaver.Log;
import org.jkiss.dbeaver.ModelPreferences;
import org.jkiss.dbeaver.model.sql.SQLDialect;
import org.jkiss.dbeaver.model.sql.SQLUtils;
import org.jkiss.dbeaver.model.sql.format.SQLFormatterConfiguration;
import org.jkiss.dbeaver.utils.GeneralUtils;
import org.jkiss.utils.ArrayUtils;
import org.jkiss.utils.CommonUtils;
import org.jkiss.utils.Pair;

import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;

class IndentFormatter {
    private static final Log log = Log.getLog(SQLFormatterTokenized.class);

    private final SQLFormatterConfiguration formatterCfg;
    private final boolean isCompact;
    private final SQLDialect dialect;
    private List<String> statementDelimiters = new LinkedList<>();
    private String delimiterRedefiner;
    private int indent = 0;
    private int bracketsDepth = 0;
    private boolean encounterBetween = false;
    private List<Boolean> functionBracket = new ArrayList<>();
    private final String[] blockHeaderStrings;

    private static final String[] JOIN_BEGIN = {"LEFT", "RIGHT", "INNER", "OUTER", "FULL", "CROSS", "JOIN"};

    IndentFormatter(SQLFormatterConfiguration formatterCfg, boolean isCompact) {
        this.formatterCfg = formatterCfg;
        delimiterRedefiner = formatterCfg.getSyntaxManager().getDialect().getScriptDelimiterRedefiner();
        if (delimiterRedefiner != null) {
            delimiterRedefiner = delimiterRedefiner.toUpperCase(Locale.ENGLISH);
        }
        for (String delim : formatterCfg.getSyntaxManager().getStatementDelimiters()) {
            statementDelimiters.add(delim.toUpperCase(Locale.ENGLISH));
        }
        this.isCompact = isCompact;
        dialect = formatterCfg.getSyntaxManager().getDialect();
        blockHeaderStrings = dialect.getBlockHeaderStrings();
    }

    private int formatSymbol(String tokenString, List<Integer> bracketIndent, List<FormatterToken> argList, Integer index, FormatterToken prev) {
        int result = index;

        switch (tokenString) {
            case "(":
                functionBracket.add(formatterCfg.isFunction(prev.getString()) ? Boolean.TRUE : Boolean.FALSE);
                bracketIndent.add(indent);
                bracketsDepth++;
                // Adding indent after ( makes result too verbose and too multiline
                if (!isCompact && formatterCfg.getPreferenceStore().getBoolean(ModelPreferences.SQL_FORMAT_BREAK_BEFORE_CLOSE_BRACKET)) {
                    indent++;
                    index += insertReturnAndIndent(argList, index + 1, indent);
                }
                break;
            case ")":
                if (!bracketIndent.isEmpty() && !functionBracket.isEmpty()) {
                    indent = bracketIndent.remove(bracketIndent.size() - 1);
                    if (!isCompact && formatterCfg.getPreferenceStore().getBoolean(ModelPreferences.SQL_FORMAT_BREAK_BEFORE_CLOSE_BRACKET)) {
                        result += insertReturnAndIndent(argList, index, indent);
                    }
                    functionBracket.remove(functionBracket.size() - 1);
                    bracketsDepth--;
                }
                break;
            case ",":
                if (!isCompact) {
                    /*if (bracketsDepth <= 0 || "SELECT".equals(getPrevDMLKeyword(argList, index)))*/
                    {
                        boolean lfBeforeComma = formatterCfg.getPreferenceStore().getBoolean(ModelPreferences.SQL_FORMAT_LF_BEFORE_COMMA);
                        result += insertReturnAndIndent(
                            argList,
                            lfBeforeComma ? index : index + 1,
                            indent);
                    }
                }
                break;
            default:
                if (statementDelimiters.contains(tokenString)) {
                    indent = 0;
                    result += insertReturnAndIndent(argList, index, indent);
                }
        }
        return result;
    }

    private int formatKeyword(List<FormatterToken> argList, String tokenString, int index) {
        int result = index;
        if (statementDelimiters.contains(tokenString)) { //$NON-NLS-1$
            indent = 0;
            if (index > 0) {
                result += insertReturnAndIndent(argList, index - 1, indent);
            }
            result += insertReturnAndIndent(argList, index + 1, indent);
        } else {
            if (blockHeaderStrings != null && ArrayUtils.contains(blockHeaderStrings, tokenString) || SQLUtils.isBlockStartKeyword(dialect, tokenString)) {
                if (index > 0) {
                    result += insertReturnAndIndent(argList, index, indent - 1);
                }
                indent++;
                result += insertReturnAndIndent(argList, index + 1, indent);
            } else if (SQLUtils.isBlockEndKeyword(dialect, tokenString)) {
                indent--;
                result += insertReturnAndIndent(argList, index, indent - 1);
            } else switch (tokenString) {
                case "CREATE":
                    if (!isCompact) {
                        int nextIndex = getNextKeywordIndex(argList, index);
                        if (nextIndex > 0 && "OR".equals(argList.get(nextIndex).getString().toUpperCase(Locale.ENGLISH))) {
                            nextIndex = getNextKeywordIndex(argList, nextIndex);
                            if (nextIndex > 0 && "REPLACE".equals(argList.get(nextIndex).getString().toUpperCase(Locale.ENGLISH))) {
                                insertReturnAndIndent(argList, nextIndex + 1, indent);
                                break;
                            }
                        }
                    }
                case "DROP": //$NON-NLS-1$
                case "ALTER": //$NON-NLS-1$
                    break;
                case "DELETE": //$NON-NLS-1$
                case "SELECT": //$NON-NLS-1$
                case "UPDATE": //$NON-NLS-1$
                case "INSERT": //$NON-NLS-1$
                case "INTO": //$NON-NLS-1$
                case "TRUNCATE": //$NON-NLS-1$
                case "TABLE": //$NON-NLS-1$
                    if (!isCompact) {
                        if (!"TABLE".equals(tokenString)) {
                            if (bracketsDepth > 0) {
                                result += insertReturnAndIndent(argList, index, indent);
                            } else if (index > 0) {
                                // just add lf before keyword
                                indent = 0;
                                result += insertReturnAndIndent(argList, index - 1, indent);
                            }
                            indent++;
                            result += insertReturnAndIndent(argList, result + 1, indent);
                        }
                    }
                    break;
                case "CASE":  //$NON-NLS-1$
                    if (!isCompact) {
                        result += insertReturnAndIndent(argList, index - 1, indent);
                        if (!"WHEN".equals(getNextKeyword(argList, index))) {
                            indent++;
                            result += insertReturnAndIndent(argList, index + 1, indent);
                        }
                    }
                    break;
                case "FROM":
                case "WHERE":
                case "START WITH":
                case "CONNECT BY":
                case "ORDER BY":
                case "GROUP BY":
                case "HAVING":  //$NON-NLS-1$
                    result += insertReturnAndIndent(argList, index, indent - 1);
                    if (!isCompact) {
                        result += insertReturnAndIndent(argList, index + 1, indent);
                    }
                    break;
                case "LEFT":
                case "RIGHT":
                case "INNER":
                case "OUTER":
                case "FULL":
                case "CROSS":
                case "JOIN":
                    if (isJoinStart(argList, index)) {
                        result += insertReturnAndIndent(argList, index, indent - 1);
                    }
                    if (tokenString.equals("JOIN")) {
                        //index += insertReturnAndIndent(argList, index + 1, indent);
                    }
                    break;
                case "VALUES":  //$NON-NLS-1$
                case "LIMIT":  //$NON-NLS-1$
                    indent--;
                    result += insertReturnAndIndent(argList, index, indent);
                    break;
                case "OR":
                    if ("CREATE".equalsIgnoreCase(getPrevKeyword(argList, index))) {
                        break;
                    }
                case "WHEN":
                    if ("CASE".equals(getPrevKeyword(argList, index))) {
                        break;
                    }
                case "ELSE":  //$NON-NLS-1$
                    result += insertReturnAndIndent(argList, index, indent);
                    break;
                case "SET": {
                    if (index > 1) {
                        if ("UPDATE".equalsIgnoreCase(getPrevKeyword(argList, index))) {<commit_msg>fix CASE ... WHEN ... END indent<commit_after>package org.jkiss.dbeaver.model.sql.format.tokenized;

import org.jkiss.dbeaver.Log;
import org.jkiss.dbeaver.ModelPreferences;
import org.jkiss.dbeaver.model.sql.SQLDialect;
import org.jkiss.dbeaver.model.sql.SQLUtils;
import org.jkiss.dbeaver.model.sql.format.SQLFormatterConfiguration;
import org.jkiss.dbeaver.utils.GeneralUtils;
import org.jkiss.utils.ArrayUtils;
import org.jkiss.utils.CommonUtils;
import org.jkiss.utils.Pair;

import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;

class IndentFormatter {
    private static final Log log = Log.getLog(SQLFormatterTokenized.class);

    private final SQLFormatterConfiguration formatterCfg;
    private final boolean isCompact;
    private final SQLDialect dialect;
    private List<String> statementDelimiters = new LinkedList<>();
    private String delimiterRedefiner;
    private int indent = 0;
    private int bracketsDepth = 0;
    private boolean encounterBetween = false;
    private List<Boolean> functionBracket = new ArrayList<>();
    private final String[] blockHeaderStrings;

    private static final String[] JOIN_BEGIN = {"LEFT", "RIGHT", "INNER", "OUTER", "FULL", "CROSS", "JOIN"};

    IndentFormatter(SQLFormatterConfiguration formatterCfg, boolean isCompact) {
        this.formatterCfg = formatterCfg;
        delimiterRedefiner = formatterCfg.getSyntaxManager().getDialect().getScriptDelimiterRedefiner();
        if (delimiterRedefiner != null) {
            delimiterRedefiner = delimiterRedefiner.toUpperCase(Locale.ENGLISH);
        }
        for (String delim : formatterCfg.getSyntaxManager().getStatementDelimiters()) {
            statementDelimiters.add(delim.toUpperCase(Locale.ENGLISH));
        }
        this.isCompact = isCompact;
        dialect = formatterCfg.getSyntaxManager().getDialect();
        blockHeaderStrings = dialect.getBlockHeaderStrings();
    }

    private int formatSymbol(String tokenString, List<Integer> bracketIndent, List<FormatterToken> argList, Integer index, FormatterToken prev) {
        int result = index;

        switch (tokenString) {
            case "(":
                functionBracket.add(formatterCfg.isFunction(prev.getString()) ? Boolean.TRUE : Boolean.FALSE);
                bracketIndent.add(indent);
                bracketsDepth++;
                // Adding indent after ( makes result too verbose and too multiline
                if (!isCompact && formatterCfg.getPreferenceStore().getBoolean(ModelPreferences.SQL_FORMAT_BREAK_BEFORE_CLOSE_BRACKET)) {
                    indent++;
                    index += insertReturnAndIndent(argList, index + 1, indent);
                }
                break;
            case ")":
                if (!bracketIndent.isEmpty() && !functionBracket.isEmpty()) {
                    indent = bracketIndent.remove(bracketIndent.size() - 1);
                    if (!isCompact && formatterCfg.getPreferenceStore().getBoolean(ModelPreferences.SQL_FORMAT_BREAK_BEFORE_CLOSE_BRACKET)) {
                        result += insertReturnAndIndent(argList, index, indent);
                    }
                    functionBracket.remove(functionBracket.size() - 1);
                    bracketsDepth--;
                }
                break;
            case ",":
                if (!isCompact) {
                    /*if (bracketsDepth <= 0 || "SELECT".equals(getPrevDMLKeyword(argList, index)))*/
                    {
                        boolean lfBeforeComma = formatterCfg.getPreferenceStore().getBoolean(ModelPreferences.SQL_FORMAT_LF_BEFORE_COMMA);
                        result += insertReturnAndIndent(
                            argList,
                            lfBeforeComma ? index : index + 1,
                            indent);
                    }
                }
                break;
            default:
                if (statementDelimiters.contains(tokenString)) {
                    indent = 0;
                    result += insertReturnAndIndent(argList, index, indent);
                }
        }
        return result;
    }

    private int formatKeyword(List<FormatterToken> argList, String tokenString, int index) {
        int result = index;
        if (statementDelimiters.contains(tokenString)) { //$NON-NLS-1$
            indent = 0;
            if (index > 0) {
                result += insertReturnAndIndent(argList, index - 1, indent);
            }
            result += insertReturnAndIndent(argList, index + 1, indent);
        } else {
            if (blockHeaderStrings != null && ArrayUtils.contains(blockHeaderStrings, tokenString) || SQLUtils.isBlockStartKeyword(dialect, tokenString)) {
                if (index > 0) {
                    result += insertReturnAndIndent(argList, index, indent - 1);
                }
                indent++;
                result += insertReturnAndIndent(argList, index + 1, indent);
            } else if (SQLUtils.isBlockEndKeyword(dialect, tokenString)) {
                indent--;
                result += insertReturnAndIndent(argList, index, indent);
            } else switch (tokenString) {
                case "CREATE":
                    if (!isCompact) {
                        int nextIndex = getNextKeywordIndex(argList, index);
                        if (nextIndex > 0 && "OR".equals(argList.get(nextIndex).getString().toUpperCase(Locale.ENGLISH))) {
                            nextIndex = getNextKeywordIndex(argList, nextIndex);
                            if (nextIndex > 0 && "REPLACE".equals(argList.get(nextIndex).getString().toUpperCase(Locale.ENGLISH))) {
                                insertReturnAndIndent(argList, nextIndex + 1, indent);
                                break;
                            }
                        }
                    }
                case "DROP": //$NON-NLS-1$
                case "ALTER": //$NON-NLS-1$
                    break;
                case "DELETE": //$NON-NLS-1$
                case "SELECT": //$NON-NLS-1$
                case "UPDATE": //$NON-NLS-1$
                case "INSERT": //$NON-NLS-1$
                case "INTO": //$NON-NLS-1$
                case "TRUNCATE": //$NON-NLS-1$
                case "TABLE": //$NON-NLS-1$
                    if (!isCompact) {
                        if (!"TABLE".equals(tokenString)) {
                            if (bracketsDepth > 0) {
                                result += insertReturnAndIndent(argList, index, indent);
                            } else if (index > 0) {
                                // just add lf before keyword
                                indent = 0;
                                result += insertReturnAndIndent(argList, index - 1, indent);
                            }
                            indent++;
                            result += insertReturnAndIndent(argList, result + 1, indent);
                        }
                    }
                    break;
                case "CASE":  //$NON-NLS-1$
                    if (!isCompact) {
                        result += insertReturnAndIndent(argList, index - 1, indent);
                        if ("WHEN".equalsIgnoreCase(getNextKeyword(argList, index))) {
                            indent++;
                            result += insertReturnAndIndent(argList, index + 1, indent);
                        }
                    }
                    break;
                case "END": // CASE ... END
                    if (!isCompact) {
	                    indent--;
	                    result += insertReturnAndIndent(argList, index, indent);
                    }
                	break;
                case "FROM":
                case "WHERE":
                case "START WITH":
                case "CONNECT BY":
                case "ORDER BY":
                case "GROUP BY":
                case "HAVING":  //$NON-NLS-1$
                    result += insertReturnAndIndent(argList, index, indent - 1);
                    if (!isCompact) {
                        result += insertReturnAndIndent(argList, index + 1, indent);
                    }
                    break;
                case "LEFT":
                case "RIGHT":
                case "INNER":
                case "OUTER":
                case "FULL":
                case "CROSS":
                case "JOIN":
                    if (isJoinStart(argList, index)) {
                        result += insertReturnAndIndent(argList, index, indent - 1);
                    }
                    if (tokenString.equals("JOIN")) {
                        //index += insertReturnAndIndent(argList, index + 1, indent);
                    }
                    break;
                case "VALUES":  //$NON-NLS-1$
                case "LIMIT":  //$NON-NLS-1$
                    indent--;
                    result += insertReturnAndIndent(argList, index, indent);
                    break;
                case "OR":
                    if ("CREATE".equalsIgnoreCase(getPrevKeyword(argList, index))) {
                        break;
                    }
                case "WHEN":
                    if ("CASE".equalsIgnoreCase(getPrevKeyword(argList, index))) {
                        break;
                    }
                case "ELSE":  //$NON-NLS-1$
                    result += insertReturnAndIndent(argList, index, indent);
                    break;
                case "SET": {
                    if (index > 1) {
                        if ("UPDATE".equalsIgnoreCase(getPrevKeyword(argList, index))) {
{'fraction_non_alphanumeric': 0.10256680546039108, 'fraction_numerical': 0.004796289463975123, 'mean_word_length': 1.4347491338380598, 'pattern_counts': {'":': 69, '<': 19, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6005480', 'n_tokens_mistral': 5017, 'n_tokens_neox': 4663, 'n_words': 969}