<commit_before>/*
 * DBeaver - Universal Database Manager
 * Copyright (C) 2010-2020 DBeaver Corp and others
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.jkiss.dbeaver.tools.transfer.stream.exporter;

import org.jkiss.dbeaver.DBException;
import org.jkiss.dbeaver.Log;
import org.jkiss.dbeaver.model.DBPNamedObject;
import org.jkiss.dbeaver.model.DBUtils;
import org.jkiss.dbeaver.model.data.*;
import org.jkiss.dbeaver.model.exec.DBCResultSet;
import org.jkiss.dbeaver.model.exec.DBCSession;
import org.jkiss.dbeaver.model.runtime.DBRProgressMonitor;
import org.jkiss.dbeaver.model.sql.SQLConstants;
import org.jkiss.dbeaver.model.sql.SQLDialect;
import org.jkiss.dbeaver.model.sql.SQLUtils;
import org.jkiss.dbeaver.tools.transfer.DTUtils;
import org.jkiss.dbeaver.tools.transfer.stream.IStreamDataExporterSite;
import org.jkiss.dbeaver.utils.ContentUtils;
import org.jkiss.dbeaver.utils.GeneralUtils;
import org.jkiss.utils.CommonUtils;

import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.Reader;
import java.util.Locale;
import java.util.Map;

/**
 * SQL Exporter
 */
public class DataExporterSQL extends StreamExporterAbstract {

    private static final Log log = Log.getLog(DataExporterSQL.class);

    private static final String PROP_INCLUDE_AUTO_GENERATED = "includeAutoGenerated";
    private static final String PROP_OMIT_SCHEMA = "omitSchema";
    private static final String PROP_ROWS_IN_STATEMENT = "rowsInStatement";
    private static final String PROP_DATA_FORMAT = "nativeFormat";
    private static final char STRING_QUOTE = '\'';
    private static final String PROP_LINE_BEFORE_VALUES = "lineBeforeValues";
    private static final String PROP_LINE_BEFORE_ROWS = "lineBeforeRows";
    private static final String PROP_KEYWORD_CASE = "keywordCase";

    private boolean includeAutoGenerated;
    private String rowDelimiter;
    private boolean omitSchema;
    private int rowsInStatement;
    private boolean useNativeDataFormat = true;
    private boolean lineBeforeValues;
    private boolean lineBeforeRows = true;
    private String tableName;
    private DBDAttributeBinding[] columns;
    private final String KEYWORD_INSERT_INTO = "INSERT INTO";
    private final String KEYWORD_VALUES = "VALUES";

    private transient StringBuilder sqlBuffer = new StringBuilder(100);
    private transient long rowCount;
    private SQLDialect dialect;

    enum KeywordCase {
        upper,
        lower
    }

    private KeywordCase keywordCase;

    private boolean isSkipColumn(DBDAttributeBinding attr) {
        return attr.isPseudoAttribute() || (!includeAutoGenerated && attr.isAutoGenerated()) ||
            attr instanceof DBDAttributeBindingCustom;
    }

    @Override
    public void init(IStreamDataExporterSite site) throws DBException {
        super.init(site);

        Map<String, Object> properties = site.getProperties();

        if (properties.containsKey(PROP_INCLUDE_AUTO_GENERATED)) {
            includeAutoGenerated = CommonUtils.toBoolean(properties.get(PROP_INCLUDE_AUTO_GENERATED));
        }
        if (properties.containsKey(PROP_OMIT_SCHEMA)) {
            omitSchema = CommonUtils.toBoolean(properties.get(PROP_OMIT_SCHEMA));
        }
        try {
            rowsInStatement = Integer.parseInt(String.valueOf(properties.get(PROP_ROWS_IN_STATEMENT)));
        } catch (NumberFormatException e) {
            rowsInStatement = 10;
        }
        useNativeDataFormat = CommonUtils.toBoolean(properties.get(PROP_DATA_FORMAT));
        lineBeforeValues = CommonUtils.toBoolean(properties.get(PROP_LINE_BEFORE_VALUES));
        lineBeforeRows = CommonUtils.toBoolean(properties.get(PROP_LINE_BEFORE_ROWS));
        rowDelimiter = GeneralUtils.getDefaultLineSeparator();
        dialect = SQLUtils.getDialectFromObject(site.getSource());

        keywordCase = CommonUtils.valueOf(KeywordCase.class, String.valueOf(properties.get(PROP_KEYWORD_CASE)), KeywordCase.upper);
    }

    @Override
    public void dispose() {
        super.dispose();
    }

    @Override
    public void exportHeader(DBCSession session) throws DBException, IOException {
        if (useNativeDataFormat) {
            if (session instanceof DBDFormatSettingsExt) {
                ((DBDFormatSettingsExt) session).setUseNativeDateTimeFormat(true);
            }
        }
        columns = getSite().getAttributes();
        DBPNamedObject source = getSite().getSource();
        tableName = DTUtils.getTableName(session.getDataSource(), source, omitSchema);

        rowCount = 0;
    }

    @Override
    public void exportRow(DBCSession session, DBCResultSet resultSet, Object[] row) throws DBException, IOException {
        PrintWriter out = getWriter();
        boolean upperCase = keywordCase == KeywordCase.upper;
        SQLDialect.MultiValueInsertMode insertMode = rowsInStatement == 1 ? SQLDialect.MultiValueInsertMode.NOT_SUPPORTED : getMultiValueInsertMode();
        if (insertMode == SQLDialect.MultiValueInsertMode.NOT_SUPPORTED) {
            rowsInStatement = 1;
        }
        int columnsSize = columns.length;
        boolean firstRow = false;
        if (insertMode == SQLDialect.MultiValueInsertMode.NOT_SUPPORTED || rowCount % rowsInStatement == 0) {
            sqlBuffer.setLength(0);
            if (rowCount > 0) {
                if (insertMode == SQLDialect.MultiValueInsertMode.PLAIN) {
                    sqlBuffer.append(");");
                } else if (insertMode == SQLDialect.MultiValueInsertMode.GROUP_ROWS) {
                    sqlBuffer.append(";");
                }
                if (lineBeforeRows) {
                    sqlBuffer.append(rowDelimiter);
                }
            }
            sqlBuffer.append(upperCase ? KEYWORD_INSERT_INTO : KEYWORD_INSERT_INTO.toLowerCase(Locale.ENGLISH));
            sqlBuffer.append(" ").append(tableName).append(" (");
            boolean hasColumn = false;
            for (int i = 0; i < columnsSize; i++) {
                DBDAttributeBinding column = columns[i];
                if (isSkipColumn(column)) {
                    continue;
                }
                if (hasColumn) {
                    sqlBuffer.append(',');
                }
                hasColumn = true;
                sqlBuffer.append(DBUtils.getQuotedIdentifier(column));
            }
            sqlBuffer.append(") ");
            if (lineBeforeValues && insertMode != SQLDialect.MultiValueInsertMode.GROUP_ROWS) {
                sqlBuffer.append(rowDelimiter).append("\t");
            }
            sqlBuffer.append(upperCase ? KEYWORD_VALUES : KEYWORD_VALUES.toLowerCase(Locale.ENGLISH));
            if (insertMode != SQLDialect.MultiValueInsertMode.GROUP_ROWS) {
                sqlBuffer.append(" (");
            }
            if (rowsInStatement > 1 && lineBeforeRows) {
                sqlBuffer.append(rowDelimiter);
            }
            out.write(sqlBuffer.toString());
            firstRow = true;
        }
        if (insertMode != SQLDialect.MultiValueInsertMode.NOT_SUPPORTED && !firstRow) {
            out.write(",");
            if (lineBeforeRows) {
                out.write(rowDelimiter);
            }
        }
        if (insertMode == SQLDialect.MultiValueInsertMode.GROUP_ROWS) {
            if (lineBeforeRows) {
                out.write("\t");
            }
            out.write(" (");
        }
        rowCount++;
        boolean hasValue = false;
        for (int i = 0; i < columnsSize; i++) {
            DBDAttributeBinding column = columns[i];
            if (isSkipColumn(column)) {
                continue;
            }

            if (hasValue) {
                out.write(',');
            }
            hasValue = true;
            Object value = row[i];

            if (DBUtils.isNullValue(value)) {
                // just skip it
                out.write(SQLConstants.NULL_VALUE);
            } else if (row[i] instanceof DBDContent) {
                DBDContent content = (DBDContent) row[i];
                try {
                    if (column.getValueHandler() instanceof DBDContentValueHandler) {
                        ((DBDContentValueHandler) column.getValueHandler()).writeStreamValue(session.getProgressMonitor(), session.getDataSource(), column, content, out);
                    } else {
                        // Content
                        // Inline textual content and handle binaries in some special way
                        DBDContentStorage cs = content.getContents(session.getProgressMonitor());
                        if (cs != null) {
                            if (ContentUtils.isTextContent(content)) {
                                try (Reader contentReader = cs.getContentReader()) {
                                    writeStringValue(contentReader);
                                }
                            } else {
                                getSite().writeBinaryData(cs);
                            }
                        }
                    }
                } catch (Exception e) {
                    log.warn(e);
                } finally {
                    content.release();
                }
            } else if (value instanceof File) {
                out.write("@");
                out.write(((File) value).getAbsolutePath());
            } else {
                out.write(SQLUtils.convertValueToSQL(session.getDataSource(), column, row[i]));
            }
        }
        if (insertMode != SQLDialect.MultiValueInsertMode.PLAIN) {
            out.write(")");
        }
        if (insertMode == SQLDialect.MultiValueInsertMode.NOT_SUPPORTED) {
            out.write(";" + rowDelimiter);
        }
    }

    @Override
    public void exportFooter(DBRProgressMonitor monitor) {
        switch (getMultiValueInsertMode()) {
            case GROUP_ROWS:
                if (rowCount > 0) {
                    getWriter().write(";");
                }
                break;
            case PLAIN:
                if (rowCount > 0) {
                    getWriter().write(");");
                }
                break;
            default:
                break;
        }
    }

    private void writeStringValue(String value) {
        PrintWriter out = getWriter();
        out.write(STRING_QUOTE);
        if (dialect != null) {
            out.write(dialect.escapeString(value));
        } else {
            out.write(value);
        }
        out.write(STRING_QUOTE);
    }

    private void writeStringValue(Reader reader) throws IOException {
        try {
            PrintWriter out = getWriter();
            out.write(STRING_QUOTE);
            // Copy reader
            char buffer[] = new char[2000];
            for (; ; ) {
                int count = reader.read(buffer);
                if (count <= 0) {
                    break;
                }
                if (dialect != null) {
                    out.write(dialect.escapeString(String.valueOf(buffer, 0, count)));
                } else {
                    out.write(buffer, 0, count);
                }
            }
            out.write(STRING_QUOTE);
        } finally {
            ContentUtils.close(reader);
        }
    }

    private SQLDialect.MultiValueInsertMode getMultiValueInsertMode() {
        SQLDialect.MultiValueInsertMode insertMode = SQLDialect.MultiValueInsertMode.NOT_SUPPORTED;
        if (dialect != null && rowsInStatement != 1) {
            insertMode = dialect.getMultiValueInsertMode();
        }
        return insertMode;
    }

}
<commit_msg>#9696 Integer.parseInt + extra rowdelimiter fix, DBPIdentifierCase added<commit_after>/*
 * DBeaver - Universal Database Manager
 * Copyright (C) 2010-2020 DBeaver Corp and others
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.jkiss.dbeaver.tools.transfer.stream.exporter;

import org.jkiss.dbeaver.DBException;
import org.jkiss.dbeaver.Log;
import org.jkiss.dbeaver.model.DBPIdentifierCase;
import org.jkiss.dbeaver.model.DBPNamedObject;
import org.jkiss.dbeaver.model.DBUtils;
import org.jkiss.dbeaver.model.data.*;
import org.jkiss.dbeaver.model.exec.DBCResultSet;
import org.jkiss.dbeaver.model.exec.DBCSession;
import org.jkiss.dbeaver.model.runtime.DBRProgressMonitor;
import org.jkiss.dbeaver.model.sql.SQLConstants;
import org.jkiss.dbeaver.model.sql.SQLDialect;
import org.jkiss.dbeaver.model.sql.SQLUtils;
import org.jkiss.dbeaver.tools.transfer.DTUtils;
import org.jkiss.dbeaver.tools.transfer.stream.IStreamDataExporterSite;
import org.jkiss.dbeaver.utils.ContentUtils;
import org.jkiss.dbeaver.utils.GeneralUtils;
import org.jkiss.utils.CommonUtils;

import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.Reader;
import java.util.Locale;
import java.util.Map;

/**
 * SQL Exporter
 */
public class DataExporterSQL extends StreamExporterAbstract {

    private static final Log log = Log.getLog(DataExporterSQL.class);

    private static final String PROP_INCLUDE_AUTO_GENERATED = "includeAutoGenerated";
    private static final String PROP_OMIT_SCHEMA = "omitSchema";
    private static final String PROP_ROWS_IN_STATEMENT = "rowsInStatement";
    private static final String PROP_DATA_FORMAT = "nativeFormat";
    private static final char STRING_QUOTE = '\'';
    private static final String PROP_LINE_BEFORE_VALUES = "lineBeforeValues";
    private static final String PROP_LINE_BEFORE_ROWS = "lineBeforeRows";
    private static final String PROP_KEYWORD_CASE = "keywordCase";

    private boolean includeAutoGenerated;
    private String rowDelimiter;
    private boolean omitSchema;
    private int rowsInStatement;
    private boolean useNativeDataFormat = true;
    private boolean lineBeforeValues;
    private boolean lineBeforeRows = true;
    private String tableName;
    private DBDAttributeBinding[] columns;
    private final String KEYWORD_INSERT_INTO = "INSERT INTO";
    private final String KEYWORD_VALUES = "VALUES";
    private DBPIdentifierCase identifierCase = DBPIdentifierCase.LOWER;

    private transient StringBuilder sqlBuffer = new StringBuilder(100);
    private transient long rowCount;
    private SQLDialect dialect;

    enum KeywordCase {
        upper,
        lower
    }

    private KeywordCase keywordCase;

    private boolean isSkipColumn(DBDAttributeBinding attr) {
        return attr.isPseudoAttribute() || (!includeAutoGenerated && attr.isAutoGenerated()) ||
            attr instanceof DBDAttributeBindingCustom;
    }

    @Override
    public void init(IStreamDataExporterSite site) throws DBException {
        super.init(site);

        Map<String, Object> properties = site.getProperties();

        if (properties.containsKey(PROP_INCLUDE_AUTO_GENERATED)) {
            includeAutoGenerated = CommonUtils.toBoolean(properties.get(PROP_INCLUDE_AUTO_GENERATED));
        }
        if (properties.containsKey(PROP_OMIT_SCHEMA)) {
            omitSchema = CommonUtils.toBoolean(properties.get(PROP_OMIT_SCHEMA));
        }
        try {
            rowsInStatement = CommonUtils.toInt(properties.get(PROP_ROWS_IN_STATEMENT));
        } catch (NumberFormatException e) {
            rowsInStatement = 10;
        }
        useNativeDataFormat = CommonUtils.toBoolean(properties.get(PROP_DATA_FORMAT));
        lineBeforeValues = CommonUtils.toBoolean(properties.get(PROP_LINE_BEFORE_VALUES));
        lineBeforeRows = CommonUtils.toBoolean(properties.get(PROP_LINE_BEFORE_ROWS));
        rowDelimiter = GeneralUtils.getDefaultLineSeparator();
        dialect = SQLUtils.getDialectFromObject(site.getSource());

        keywordCase = CommonUtils.valueOf(KeywordCase.class, CommonUtils.toString(properties.get(PROP_KEYWORD_CASE)), KeywordCase.upper);
    }

    @Override
    public void dispose() {
        super.dispose();
    }

    @Override
    public void exportHeader(DBCSession session) throws DBException, IOException {
        if (useNativeDataFormat) {
            if (session instanceof DBDFormatSettingsExt) {
                ((DBDFormatSettingsExt) session).setUseNativeDateTimeFormat(true);
            }
        }
        columns = getSite().getAttributes();
        DBPNamedObject source = getSite().getSource();
        tableName = DTUtils.getTableName(session.getDataSource(), source, omitSchema);

        rowCount = 0;
    }

    @Override
    public void exportRow(DBCSession session, DBCResultSet resultSet, Object[] row) throws DBException, IOException {
        PrintWriter out = getWriter();
        boolean upperCase = keywordCase == KeywordCase.upper;
        SQLDialect.MultiValueInsertMode insertMode = rowsInStatement == 1 ? SQLDialect.MultiValueInsertMode.NOT_SUPPORTED : getMultiValueInsertMode();
        if (insertMode == SQLDialect.MultiValueInsertMode.NOT_SUPPORTED) {
            rowsInStatement = 1;
        }
        int columnsSize = columns.length;
        boolean firstRow = false;
        if (insertMode == SQLDialect.MultiValueInsertMode.NOT_SUPPORTED || rowCount % rowsInStatement == 0) {
            sqlBuffer.setLength(0);
            if (rowCount > 0) {
                if (insertMode == SQLDialect.MultiValueInsertMode.PLAIN) {
                    sqlBuffer.append(");");
                } else if (insertMode == SQLDialect.MultiValueInsertMode.GROUP_ROWS) {
                    sqlBuffer.append(";");
                }
                if (lineBeforeRows) {
                    sqlBuffer.append(rowDelimiter);
                }
            }
            sqlBuffer.append(upperCase ? KEYWORD_INSERT_INTO : identifierCase.transform(KEYWORD_INSERT_INTO));
            sqlBuffer.append(" ").append(tableName).append(" (");
            boolean hasColumn = false;
            for (int i = 0; i < columnsSize; i++) {
                DBDAttributeBinding column = columns[i];
                if (isSkipColumn(column)) {
                    continue;
                }
                if (hasColumn) {
                    sqlBuffer.append(',');
                }
                hasColumn = true;
                sqlBuffer.append(DBUtils.getQuotedIdentifier(column));
            }
            sqlBuffer.append(") ");
            if (lineBeforeValues && insertMode != SQLDialect.MultiValueInsertMode.GROUP_ROWS) {
                sqlBuffer.append(rowDelimiter).append("\t");
            }
            sqlBuffer.append(upperCase ? KEYWORD_VALUES : identifierCase.transform(KEYWORD_VALUES));
            if (insertMode != SQLDialect.MultiValueInsertMode.GROUP_ROWS) {
                sqlBuffer.append(" (");
            }
            if (rowsInStatement > 1 && lineBeforeRows) {
                sqlBuffer.append(rowDelimiter);
            }
            out.write(sqlBuffer.toString());
            firstRow = true;
        }
        if (insertMode != SQLDialect.MultiValueInsertMode.NOT_SUPPORTED && !firstRow) {
            out.write(",");
            if (lineBeforeRows) {
                out.write(rowDelimiter);
            }
        }
        if (insertMode == SQLDialect.MultiValueInsertMode.GROUP_ROWS) {
            if (lineBeforeRows) {
                out.write("\t");
            }
            out.write(" (");
        }
        rowCount++;
        boolean hasValue = false;
        for (int i = 0; i < columnsSize; i++) {
            DBDAttributeBinding column = columns[i];
            if (isSkipColumn(column)) {
                continue;
            }

            if (hasValue) {
                out.write(',');
            }
            hasValue = true;
            Object value = row[i];

            if (DBUtils.isNullValue(value)) {
                // just skip it
                out.write(SQLConstants.NULL_VALUE);
            } else if (row[i] instanceof DBDContent) {
                DBDContent content = (DBDContent) row[i];
                try {
                    if (column.getValueHandler() instanceof DBDContentValueHandler) {
                        ((DBDContentValueHandler) column.getValueHandler()).writeStreamValue(session.getProgressMonitor(), session.getDataSource(), column, content, out);
                    } else {
                        // Content
                        // Inline textual content and handle binaries in some special way
                        DBDContentStorage cs = content.getContents(session.getProgressMonitor());
                        if (cs != null) {
                            if (ContentUtils.isTextContent(content)) {
                                try (Reader contentReader = cs.getContentReader()) {
                                    writeStringValue(contentReader);
                                }
                            } else {
                                getSite().writeBinaryData(cs);
                            }
                        }
                    }
                } catch (Exception e) {
                    log.warn(e);
                } finally {
                    content.release();
                }
            } else if (value instanceof File) {
                out.write("@");
                out.write(((File) value).getAbsolutePath());
            } else {
                out.write(SQLUtils.convertValueToSQL(session.getDataSource(), column, row[i]));
            }
        }
        if (insertMode != SQLDialect.MultiValueInsertMode.PLAIN) {
            out.write(")");
        }
        if (insertMode == SQLDialect.MultiValueInsertMode.NOT_SUPPORTED) {
            out.write(";");
        }
    }

    @Override
    public void exportFooter(DBRProgressMonitor monitor) {
        switch (getMultiValueInsertMode()) {
            case GROUP_ROWS:
                if (rowCount > 0) {
                    getWriter().write(";");
                }
                break;
            case PLAIN:
                if (rowCount > 0) {
                    getWriter().write(");");
                }
                break;
            default:
                break;
        }
    }

    private void writeStringValue(String value) {
        PrintWriter out = getWriter();
        out.write(STRING_QUOTE);
        if (dialect != null) {
            out.write(dialect.escapeString(value));
        } else {
            out.write(value);
        }
        out.write(STRING_QUOTE);
    }

    private void writeStringValue(Reader reader) throws IOException {
        try {
            PrintWriter out = getWriter();
            out.write(STRING_QUOTE);
            // Copy reader
            char buffer[] = new char[2000];
            for (; ; ) {
                int count = reader.read(buffer);
                if (count <= 0) {
                    break;
                }
                if (dialect != null) {
                    out.write(dialect.escapeString(String.valueOf(buffer, 0, count)));
                } else {
                    out.write(buffer, 0, count);
                }
            }
            out.write(STRING_QUOTE);
        } finally {
            ContentUtils.close(reader);
        }
    }

    private SQLDialect.MultiValueInsertMode getMultiValueInsertMode() {
        SQLDialect.MultiValueInsertMode insertMode = SQLDialect.MultiValueInsertMode.NOT_SUPPORTED;
        if (dialect != null && rowsInStatement != 1) {
            insertMode = dialect.getMultiValueInsertMode();
        }
        return insertMode;
    }

}

{'fraction_non_alphanumeric': 0.0944195418842785, 'fraction_numerical': 0.003125385532754863, 'mean_word_length': 2.06503655155029, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2568033', 'n_tokens_mistral': 6597, 'n_tokens_neox': 6333, 'n_words': 1327}