<commit_before>/*
 * DBeaver - Universal Database Manager
 * Copyright (C) 2010-2019 <NAME> (<EMAIL>)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.jkiss.dbeaver.ui.editors.text.handlers;

import org.eclipse.core.commands.ExecutionEvent;
import org.eclipse.core.commands.ExecutionException;
import org.eclipse.jface.dialogs.Dialog;
import org.eclipse.jface.dialogs.IDialogConstants;
import org.eclipse.jface.dialogs.IDialogSettings;
import org.eclipse.jface.text.BadLocationException;
import org.eclipse.jface.text.IDocument;
import org.eclipse.jface.text.ITextSelection;
import org.eclipse.jface.viewers.ISelection;
import org.eclipse.jface.viewers.ISelectionProvider;
import org.eclipse.swt.SWT;
import org.eclipse.swt.layout.GridData;
import org.eclipse.swt.layout.GridLayout;
import org.eclipse.swt.widgets.*;
import org.eclipse.ui.handlers.HandlerUtil;
import org.jkiss.dbeaver.runtime.DBWorkbench;
import org.jkiss.dbeaver.ui.UIUtils;
import org.jkiss.dbeaver.ui.editors.text.BaseTextEditor;
import org.jkiss.utils.CommonUtils;

import java.util.ArrayList;
import java.util.List;
import java.util.StringTokenizer;

public final class MorphDelimitedListHandler extends AbstractTextHandler {

    @Override
    public Object execute(ExecutionEvent executionEvent) throws ExecutionException {
        Shell activeShell = HandlerUtil.getActiveShell(executionEvent);
        BaseTextEditor textEditor = BaseTextEditor.getTextEditor(HandlerUtil.getActiveEditor(executionEvent));
        if (textEditor != null) {
            IDocument document = textEditor.getDocumentProvider().getDocument(textEditor.getEditorInput());
            ISelectionProvider provider = textEditor.getSelectionProvider();
            if (provider != null) {
                ISelection selection = provider.getSelection();
                if (selection instanceof ITextSelection) {
                    ITextSelection textSelection = (ITextSelection) selection;
                    if (textSelection.getLength() <= 0) {
                        UIUtils.showMessageBox(activeShell, "Morph text", "Text selection is empty. You need to select some text to morph", SWT.ICON_INFORMATION);
                        return null;
                    }
                    String formattedText = morphText(activeShell, textSelection.getText());
                    if (formattedText != null) {
                        try {
                            document.replace(textSelection.getOffset(), textSelection.getLength(), formattedText);
                        } catch (BadLocationException e) {
                            DBWorkbench.getPlatformUI().showError("Morph text", "Error replacing text", e);
                        }
                    }
                }
            }
        }

        return null;
    }

    private String morphText(Shell activeShell, String text) {
        ConfigDialog configDialog = new ConfigDialog(activeShell);
        if (configDialog.open() != IDialogConstants.OK_ID) {
            return null;
        }

        List<String> tokens = new ArrayList<>();
        MorphDelimitedListSettings settings = configDialog.morphSettings;
        String sourceDelimiter = settings.getSourceDelimiter();

        // Fix line feed
        if (sourceDelimiter.contains("\n") && !sourceDelimiter.contains("\r")) {
            sourceDelimiter += "\r";
        }

        StringTokenizer st = new StringTokenizer(text, sourceDelimiter);
        while (st.hasMoreTokens()) {
            tokens.add(st.nextToken());
        }
        StringBuilder buf = new StringBuilder();
        if (!CommonUtils.isEmpty(settings.getLeadingText())) {
            buf.append(settings.getLeadingText());
        }
        int lastLineFeed = 0;
        for (int i = 0; i < tokens.size(); i++) {
            String token = tokens.get(i);
            if (!CommonUtils.isEmpty(settings.getQuoteString())) {
                buf.append(settings.getQuoteString()).append(token).append(settings.getQuoteString());
                lastLineFeed += settings.getQuoteString().length() * 2 + token.length();
            } else {
                buf.append(token);
                lastLineFeed += token.length();
            }
            if (i < tokens.size() - 1) {
                buf.append(settings.getTargetDelimiter());
                lastLineFeed += settings.getTargetDelimiter().length();<commit_msg>#5287 Morph delimited list: exclude spaces from tokens<commit_after>/*
 * DBeaver - Universal Database Manager
 * Copyright (C) 2010-2019 <NAME> (<EMAIL>)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.jkiss.dbeaver.ui.editors.text.handlers;

import org.eclipse.core.commands.ExecutionEvent;
import org.eclipse.core.commands.ExecutionException;
import org.eclipse.jface.dialogs.Dialog;
import org.eclipse.jface.dialogs.IDialogConstants;
import org.eclipse.jface.dialogs.IDialogSettings;
import org.eclipse.jface.text.BadLocationException;
import org.eclipse.jface.text.IDocument;
import org.eclipse.jface.text.ITextSelection;
import org.eclipse.jface.viewers.ISelection;
import org.eclipse.jface.viewers.ISelectionProvider;
import org.eclipse.swt.SWT;
import org.eclipse.swt.layout.GridData;
import org.eclipse.swt.layout.GridLayout;
import org.eclipse.swt.widgets.*;
import org.eclipse.ui.handlers.HandlerUtil;
import org.jkiss.dbeaver.runtime.DBWorkbench;
import org.jkiss.dbeaver.ui.UIUtils;
import org.jkiss.dbeaver.ui.editors.text.BaseTextEditor;
import org.jkiss.utils.CommonUtils;

import java.util.ArrayList;
import java.util.List;
import java.util.StringTokenizer;

public final class MorphDelimitedListHandler extends AbstractTextHandler {

    @Override
    public Object execute(ExecutionEvent executionEvent) throws ExecutionException {
        Shell activeShell = HandlerUtil.getActiveShell(executionEvent);
        BaseTextEditor textEditor = BaseTextEditor.getTextEditor(HandlerUtil.getActiveEditor(executionEvent));
        if (textEditor != null) {
            IDocument document = textEditor.getDocumentProvider().getDocument(textEditor.getEditorInput());
            ISelectionProvider provider = textEditor.getSelectionProvider();
            if (provider != null) {
                ISelection selection = provider.getSelection();
                if (selection instanceof ITextSelection) {
                    ITextSelection textSelection = (ITextSelection) selection;
                    if (textSelection.getLength() <= 0) {
                        UIUtils.showMessageBox(activeShell, "Morph text", "Text selection is empty. You need to select some text to morph", SWT.ICON_INFORMATION);
                        return null;
                    }
                    String formattedText = morphText(activeShell, textSelection.getText());
                    if (formattedText != null) {
                        try {
                            document.replace(textSelection.getOffset(), textSelection.getLength(), formattedText);
                        } catch (BadLocationException e) {
                            DBWorkbench.getPlatformUI().showError("Morph text", "Error replacing text", e);
                        }
                    }
                }
            }
        }

        return null;
    }

    private String morphText(Shell activeShell, String text) {
        ConfigDialog configDialog = new ConfigDialog(activeShell);
        if (configDialog.open() != IDialogConstants.OK_ID) {
            return null;
        }

        List<String> tokens = new ArrayList<>();
        MorphDelimitedListSettings settings = configDialog.morphSettings;
        String sourceDelimiter = settings.getSourceDelimiter();

        // Fix line feed
        if (sourceDelimiter.contains("\n") && !sourceDelimiter.contains("\r")) {
            sourceDelimiter += "\r";
        }

        StringTokenizer st = new StringTokenizer(text, sourceDelimiter);
        while (st.hasMoreTokens()) {
            tokens.add(st.nextToken());
        }
        StringBuilder buf = new StringBuilder();
        if (!CommonUtils.isEmpty(settings.getLeadingText())) {
            buf.append(settings.getLeadingText());
        }
        int lastLineFeed = 0;
        for (int i = 0; i < tokens.size(); i++) {
            String token = tokens.get(i);
            int leadingSpaces = 0, trailingSpaces = 0;
            {
                boolean inIdentifier = false;
                for (int k = 0; k < token.length(); k++) {
                    char ch = token.charAt(k);
                    if (Character.isWhitespace(ch)) {
                        if (inIdentifier) {
                            trailingSpaces++;
                        } else {
                            buf.append(ch);
                            leadingSpaces++;
                        }
                    } else {
                        inIdentifier = true;
                    }
                }
            }
            if (leadingSpaces > 0 || trailingSpaces > 0) {
                token = token.substring(leadingSpaces, token.length() - trailingSpaces);
            }
            if (!CommonUtils.isEmpty(settings.getQuoteString())) {
                buf.append(settings.getQuoteString()).append(token).append(settings.getQuoteString());
                lastLineFeed += settings.getQuoteString().length() * 2 + token.length();
            } else {
                buf.append(token);
                lastLineFeed += token.length();
            }
            if (i < tokens.size() - 1) {
                buf.append(settings.getTargetDelimiter());
                lastLineFeed += settings.getTargetDelimiter().length();
{'fraction_non_alphanumeric': 0.0998684704998121, 'fraction_numerical': 0.004039834648628335, 'mean_word_length': 2.359103818239192, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6653809', 'n_tokens_mistral': 2772, 'n_tokens_neox': 2667, 'n_words': 640}