<commit_before>/*
 * DBeaver - Universal Database Manager
 * Copyright (C) 2010-2021 DBeaver Corp and others
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.jkiss.dbeaver.model.sql;

import org.eclipse.core.resources.IFile;
import org.eclipse.core.runtime.Platform;
import org.jkiss.code.NotNull;
import org.jkiss.code.Nullable;
import org.jkiss.dbeaver.DBException;
import org.jkiss.dbeaver.Log;
import org.jkiss.dbeaver.model.*;
import org.jkiss.dbeaver.model.data.*;
import org.jkiss.dbeaver.model.edit.DBEPersistAction;
import org.jkiss.dbeaver.model.exec.DBCLogicalOperator;
import org.jkiss.dbeaver.model.exec.DBCSession;
import org.jkiss.dbeaver.model.impl.sql.BasicSQLDialect;
import org.jkiss.dbeaver.model.runtime.DBRFinder;
import org.jkiss.dbeaver.model.runtime.DBRProgressMonitor;
import org.jkiss.dbeaver.model.runtime.VoidProgressMonitor;
import org.jkiss.dbeaver.model.struct.*;
import org.jkiss.dbeaver.model.struct.rdb.DBSTableForeignKey;
import org.jkiss.dbeaver.model.struct.rdb.DBSTableForeignKeyColumn;
import org.jkiss.dbeaver.utils.ContentUtils;
import org.jkiss.dbeaver.utils.GeneralUtils;
import org.jkiss.utils.ArrayUtils;
import org.jkiss.utils.CommonUtils;
import org.jkiss.utils.Pair;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.lang.reflect.Array;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * SQL Utils
 */
public final class SQLUtils {

    private static final Log log = Log.getLog(SQLUtils.class);

    public static final Pattern PATTERN_OUT_PARAM = Pattern.compile("((\\?)|(:[a-z0-9]+))\\s*:=");
    public static final Pattern PATTERN_SIMPLE_NAME = Pattern.compile("[a-z][a-z0-9_]*", Pattern.CASE_INSENSITIVE);

    private static final Pattern CREATE_PREFIX_PATTERN = Pattern.compile("(CREATE (:OR REPLACE)?).+", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE);

    private static final int MIN_SQL_DESCRIPTION_LENGTH = 512;
    private static final int MAX_SQL_DESCRIPTION_LENGTH = 500;

    private static final String DBEAVER_DDL_COMMENT = "-- DDL generated by ";
    private static final String DBEAVER_DDL_WARNING = "-- WARNING: It may differ from actual native database DDL";
    private static final String DBEAVER_SCRIPT_DELIMITER = "$$";

    public static String stripTransformations(String query)
    {
        return query;
//        if (!query.contains(TOKEN_TRANSFORM_START)) {
//            return query;
//        } else {
//            return PATTERN_XFORM.matcher(query).replaceAll("");
//        }
    }

    public static String stripComments(@NotNull SQLDialect dialect, @NotNull String query)
    {
        Pair<String, String> multiLineComments = dialect.getMultiLineComments();
        return stripComments(
            query,
            multiLineComments == null ? null : multiLineComments.getFirst(),
            multiLineComments == null ? null : multiLineComments.getSecond(),
            dialect.getSingleLineComments());
    }

    public static boolean isCommentLine(SQLDialect dialect, String line) {
        for (String slc : dialect.getSingleLineComments()) {
            if (line.startsWith(slc)) {
                return true;
            }
        }
        return false;
    }

    public static String stripComments(@NotNull String query, @Nullable String mlCommentStart, @Nullable String mlCommentEnd, String[] slComments)
    {
        String leading = "", trailing = "";
        {
            int startPos, endPos;
            for (startPos = 0; startPos < query.length(); startPos++) {
                if (!Character.isWhitespace(query.charAt(startPos))) {
                    break;
                }
            }
            for (endPos = query.length() - 1; endPos > startPos; endPos--) {
                if (!Character.isWhitespace(query.charAt(endPos))) {
                    break;
                }
            }
            if (startPos > 0) {
                leading = query.substring(0, startPos);
            }
            if (endPos < query.length() - 1) {
                trailing = query.substring(endPos + 1);
            }
        }
        query = query.trim();
        if (mlCommentStart != null && mlCommentEnd != null && query.startsWith(mlCommentStart)) {
            int endPos = query.indexOf(mlCommentEnd);
            if (endPos != -1) {
                query = query.substring(endPos + mlCommentEnd.length());
            }
        }
        for (int i = 0; i < slComments.length; i++) {
            while (query.startsWith(slComments[i])) {
                int crPos = query.indexOf('\n');
                if (crPos == -1) {
                    // Query is comment line - return empty
                    query = "";
                    break;
                } else {
                    query = query.substring(crPos).trim();
                }
            }
        }
        return leading + query + trailing;
    }

    public static List<String> splitFilter(String filter)
    {
        if (CommonUtils.isEmpty(filter)) {
            return Collections.emptyList();
        }
        return CommonUtils.splitString(filter, ',');
    }

    public static boolean matchesAnyLike(String string, Collection<String> likes)
    {
        for (String like : likes) {
            if (matchesLike(string, like)) {
                return true;
            }
        }
        return false;
    }

    public static boolean isLikePattern(String like)
    {
        return like.indexOf('%') != -1 || like.indexOf('*') != -1 || like.indexOf('_') != -1 || like.indexOf('?') != -1;// || like.indexOf('_') != -1;
    }

    public static String makeLikePattern(String like)
    {
        StringBuilder result = new StringBuilder();

        for (int i = 0; i < like.length(); i++) {
            char c = like.charAt(i);
            if (c == '*') result.append(".*");
            else if (c == '?' || c == '_') result.append(".");
            else if (c == '%') result.append(".*");
            else if (Character.isLetterOrDigit(c)) result.append(c);
            else if (c == '(' || c == ')' || c == '[' || c == ']') result.append('\\').append(c);
            else if (c == '\\') {
                if (i < like.length() - 1) {
                    char nc = like.charAt(i + 1);
                    if (nc == '_' || nc == '*' || nc == '?' || nc == '.') {
                        result.append("\\").append(nc);
                        i++;
                    } else {<commit_msg>#11883 escape percent sign when constructing Java Pattern from like pattern<commit_after>/*
 * DBeaver - Universal Database Manager
 * Copyright (C) 2010-2021 DBeaver Corp and others
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.jkiss.dbeaver.model.sql;

import org.eclipse.core.resources.IFile;
import org.eclipse.core.runtime.Platform;
import org.jkiss.code.NotNull;
import org.jkiss.code.Nullable;
import org.jkiss.dbeaver.DBException;
import org.jkiss.dbeaver.Log;
import org.jkiss.dbeaver.model.*;
import org.jkiss.dbeaver.model.data.*;
import org.jkiss.dbeaver.model.edit.DBEPersistAction;
import org.jkiss.dbeaver.model.exec.DBCLogicalOperator;
import org.jkiss.dbeaver.model.exec.DBCSession;
import org.jkiss.dbeaver.model.impl.sql.BasicSQLDialect;
import org.jkiss.dbeaver.model.runtime.DBRFinder;
import org.jkiss.dbeaver.model.runtime.DBRProgressMonitor;
import org.jkiss.dbeaver.model.runtime.VoidProgressMonitor;
import org.jkiss.dbeaver.model.struct.*;
import org.jkiss.dbeaver.model.struct.rdb.DBSTableForeignKey;
import org.jkiss.dbeaver.model.struct.rdb.DBSTableForeignKeyColumn;
import org.jkiss.dbeaver.utils.ContentUtils;
import org.jkiss.dbeaver.utils.GeneralUtils;
import org.jkiss.utils.ArrayUtils;
import org.jkiss.utils.CommonUtils;
import org.jkiss.utils.Pair;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.lang.reflect.Array;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * SQL Utils
 */
public final class SQLUtils {

    private static final Log log = Log.getLog(SQLUtils.class);

    public static final Pattern PATTERN_OUT_PARAM = Pattern.compile("((\\?)|(:[a-z0-9]+))\\s*:=");
    public static final Pattern PATTERN_SIMPLE_NAME = Pattern.compile("[a-z][a-z0-9_]*", Pattern.CASE_INSENSITIVE);

    private static final Pattern CREATE_PREFIX_PATTERN = Pattern.compile("(CREATE (:OR REPLACE)?).+", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE);

    private static final int MIN_SQL_DESCRIPTION_LENGTH = 512;
    private static final int MAX_SQL_DESCRIPTION_LENGTH = 500;

    private static final String DBEAVER_DDL_COMMENT = "-- DDL generated by ";
    private static final String DBEAVER_DDL_WARNING = "-- WARNING: It may differ from actual native database DDL";
    private static final String DBEAVER_SCRIPT_DELIMITER = "$$";

    public static String stripTransformations(String query)
    {
        return query;
//        if (!query.contains(TOKEN_TRANSFORM_START)) {
//            return query;
//        } else {
//            return PATTERN_XFORM.matcher(query).replaceAll("");
//        }
    }

    public static String stripComments(@NotNull SQLDialect dialect, @NotNull String query)
    {
        Pair<String, String> multiLineComments = dialect.getMultiLineComments();
        return stripComments(
            query,
            multiLineComments == null ? null : multiLineComments.getFirst(),
            multiLineComments == null ? null : multiLineComments.getSecond(),
            dialect.getSingleLineComments());
    }

    public static boolean isCommentLine(SQLDialect dialect, String line) {
        for (String slc : dialect.getSingleLineComments()) {
            if (line.startsWith(slc)) {
                return true;
            }
        }
        return false;
    }

    public static String stripComments(@NotNull String query, @Nullable String mlCommentStart, @Nullable String mlCommentEnd, String[] slComments)
    {
        String leading = "", trailing = "";
        {
            int startPos, endPos;
            for (startPos = 0; startPos < query.length(); startPos++) {
                if (!Character.isWhitespace(query.charAt(startPos))) {
                    break;
                }
            }
            for (endPos = query.length() - 1; endPos > startPos; endPos--) {
                if (!Character.isWhitespace(query.charAt(endPos))) {
                    break;
                }
            }
            if (startPos > 0) {
                leading = query.substring(0, startPos);
            }
            if (endPos < query.length() - 1) {
                trailing = query.substring(endPos + 1);
            }
        }
        query = query.trim();
        if (mlCommentStart != null && mlCommentEnd != null && query.startsWith(mlCommentStart)) {
            int endPos = query.indexOf(mlCommentEnd);
            if (endPos != -1) {
                query = query.substring(endPos + mlCommentEnd.length());
            }
        }
        for (int i = 0; i < slComments.length; i++) {
            while (query.startsWith(slComments[i])) {
                int crPos = query.indexOf('\n');
                if (crPos == -1) {
                    // Query is comment line - return empty
                    query = "";
                    break;
                } else {
                    query = query.substring(crPos).trim();
                }
            }
        }
        return leading + query + trailing;
    }

    public static List<String> splitFilter(String filter)
    {
        if (CommonUtils.isEmpty(filter)) {
            return Collections.emptyList();
        }
        return CommonUtils.splitString(filter, ',');
    }

    public static boolean matchesAnyLike(String string, Collection<String> likes)
    {
        for (String like : likes) {
            if (matchesLike(string, like)) {
                return true;
            }
        }
        return false;
    }

    public static boolean isLikePattern(String like)
    {
        return like.indexOf('%') != -1 || like.indexOf('*') != -1 || like.indexOf('_') != -1 || like.indexOf('?') != -1;// || like.indexOf('_') != -1;
    }

    public static String makeLikePattern(String like)
    {
        StringBuilder result = new StringBuilder();

        for (int i = 0; i < like.length(); i++) {
            char c = like.charAt(i);
            if (c == '*') result.append(".*");
            else if (c == '?' || c == '_') result.append(".");
            else if (c == '%') result.append(".*");
            else if (Character.isLetterOrDigit(c)) result.append(c);
            else if (c == '(' || c == ')' || c == '[' || c == ']') result.append('\\').append(c);
            else if (c == '\\') {
                if (i < like.length() - 1) {
                    char nc = like.charAt(i + 1);
                    if (nc == '_' || nc == '*' || nc == '?' || nc == '.' || nc == '%') {
                        result.append("\\").append(nc);
                        i++;
                    } else {
{'fraction_non_alphanumeric': 0.12703838211208432, 'fraction_numerical': 0.005910418001851456, 'mean_word_length': 2.2805419294557345, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5454445', 'n_tokens_mistral': 4139, 'n_tokens_neox': 3985, 'n_words': 981}