<commit_before>/*
 * Copyright 2000-2009 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.intellij.codeInsight.editorActions.wordSelection;

import com.intellij.codeInsight.editorActions.ExtendWordSelectionHandlerBase;
import com.intellij.openapi.editor.Editor;
import com.intellij.openapi.editor.SelectionModel;
import com.intellij.openapi.util.TextRange;
import com.intellij.psi.PsiComment;
import com.intellij.psi.PsiElement;
import com.intellij.psi.PsiPlainText;
import com.intellij.util.containers.CollectionFactory;

import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Set;

public class NaturalLanguageTextSelectioner extends ExtendWordSelectionHandlerBase {
  private static final Set<Character> NATURAL = CollectionFactory.troveSet('(', ')', '.', ',', ':', ';', '!', '?', '$', '@', '%', '\"', '\'');
  private static final Set<Character> SENTENCE_END = CollectionFactory.troveSet('.', '!', '?');

  public boolean canSelect(PsiElement e) {
    return e instanceof PsiPlainText || e instanceof PsiComment;
  }

  public List<TextRange> select(PsiElement e, CharSequence editorText, int cursorOffset, Editor editor) {
    final SelectionModel selectionModel = editor.getSelectionModel();
    if (!selectionModel.hasSelection()) {
      return Collections.emptyList();
    }

    int sentenceStart = selectionModel.getSelectionStart();
    final int elementStart = e.getTextRange().getStartOffset();
    if (sentenceStart <= elementStart) return Collections.emptyList();

    int paragraphStart = editorText.subSequence(elementStart, sentenceStart).toString().lastIndexOf("\n\n");
    if (paragraphStart < 0) paragraphStart = elementStart;
    else paragraphStart += 2 + elementStart;
    boolean isParagraph = paragraphStart == sentenceStart;

    while (sentenceStart > paragraphStart) {
      final char c = editorText.charAt(sentenceStart - 1);
      if (!isNatural(c)) {
        return Collections.emptyList();
      }

      if (SENTENCE_END.contains(c)) {
        break;
      }
      sentenceStart--;
    }
    while (Character.isWhitespace(editorText.charAt(sentenceStart))) {
      sentenceStart++;
    }

    int sentenceEnd = selectionModel.getSelectionEnd();
    final int elementEnd = e.getTextRange().getEndOffset();
    if (sentenceEnd > elementEnd) {
      return Collections.emptyList();
    }
    int paragraphEnd = editorText.subSequence(sentenceEnd, elementEnd).toString().indexOf("\n\n");
    if (paragraphEnd < 0) paragraphEnd = elementEnd;
    else paragraphEnd += sentenceEnd;
    isParagraph &= paragraphEnd == sentenceEnd;

    if (isParagraph) {
      return Collections.emptyList(); //whole text
    }

    if (sentenceEnd > elementStart) sentenceEnd--;
    while (sentenceEnd < paragraphEnd) {
      final char c = editorText.charAt(sentenceEnd);
      if (!isNatural(c)) {
        return Collections.emptyList();
      }

      sentenceEnd++;

      if (SENTENCE_END.contains(c)) {
        break;
      }

    }

    if (sentenceStart == selectionModel.getSelectionStart() && sentenceEnd == selectionModel.getSelectionEnd()) {
      return Arrays.asList(new TextRange(paragraphStart, paragraphEnd));
    }

    return Arrays.asList(new TextRange(sentenceStart, sentenceEnd));
  }

  private static boolean isNatural(char c) {
    return Character.isWhitespace(c) || Character.isLetterOrDigit(c) || NATURAL.contains(c);
  }
}<commit_msg>select word: quotes and parentheses in plain text<commit_after>/*
 * Copyright 2000-2009 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.intellij.codeInsight.editorActions.wordSelection;

import com.intellij.codeInsight.editorActions.ExtendWordSelectionHandlerBase;
import com.intellij.openapi.editor.Editor;
import com.intellij.openapi.editor.SelectionModel;
import com.intellij.openapi.util.TextRange;
import com.intellij.psi.PsiComment;
import com.intellij.psi.PsiElement;
import com.intellij.psi.PsiPlainText;
import com.intellij.util.containers.CollectionFactory;
import org.jetbrains.annotations.Nullable;

import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Set;

public class NaturalLanguageTextSelectioner extends ExtendWordSelectionHandlerBase {
  private static final Set<Character> NATURAL = CollectionFactory.troveSet('(', ')', '.', ',', ':', ';', '!', '?', '$', '@', '%', '\"', '\'');
  private static final Set<Character> SENTENCE_END = CollectionFactory.troveSet('.', '!', '?');

  public boolean canSelect(PsiElement e) {
    return e instanceof PsiPlainText || e instanceof PsiComment;
  }

  private static TextRange findParagraphRange(String text, int start, int end) {
    int paragraphStart = text.lastIndexOf("\n\n", start);
    int paragraphEnd = text.indexOf("\n\n", end);
    return new TextRange(Math.max(0, paragraphStart), paragraphEnd < 0 ? text.length() : paragraphEnd);
  }

  @Nullable
  private static TextRange findCustomRange(String text, int start, int end, char startChar, char endChar) {
    int prev = text.lastIndexOf(startChar, start);
    int next = text.indexOf(endChar, end);
    if (prev < 0 || next < 0) {
      return null;
    }
    if (prev + 1 < start || next > end) {
      return new TextRange(prev + 1, next);
    }

    return new TextRange(prev, next + 1);
  }

  private static TextRange findSentenceRange(String editorText, int start, int end) {
    int sentenceStart = start;

    while (sentenceStart > 0) {
      if (SENTENCE_END.contains(editorText.charAt(sentenceStart - 1)) || !isNatural(editorText.charAt(sentenceStart - 1))) {
        break;
      }
      sentenceStart--;
    }
    while (Character.isWhitespace(editorText.charAt(sentenceStart))) {
      sentenceStart++;
    }

    int sentenceEnd = Math.max(0, end - 1);

    while (sentenceEnd < editorText.length()) {
      sentenceEnd++;
      if (SENTENCE_END.contains(editorText.charAt(sentenceEnd - 1))) {
        break;
      }
    }
    return new TextRange(sentenceStart, sentenceEnd);
  }

  private static TextRange findNaturalRange(String editorText, int start, int end) {
    while (start > 0) {
      if (!isNatural(editorText.charAt(start - 1))) {
        break;
      }
      start--;
    }

    while (end < editorText.length()) {
      final char c = editorText.charAt(end);
      if (!isNatural(c)) {
        break;
      }
      end++;
    }
    return new TextRange(start, end);
  }

  public List<TextRange> select(PsiElement e, CharSequence editorText, int cursorOffset, Editor editor) {
    final SelectionModel selectionModel = editor.getSelectionModel();
    if (!selectionModel.hasSelection()) {
      return Collections.emptyList();
    }

    TextRange range = e.getTextRange();
    int shift = range.getStartOffset();
    int selStart = selectionModel.getSelectionStart();
    int selEnd = selectionModel.getSelectionEnd();
    if (selStart <= shift || selEnd >= range.getEndOffset()) {
      return Collections.emptyList();
    }

    String elementText = editorText.subSequence(shift, range.getEndOffset()).toString();
    int start = selStart - shift;
    int end = selEnd - shift;

    TextRange best = findSentenceRange(elementText, start, end);
    best = narrowRange(best, best, findCustomRange(elementText, start, end, '\"', '\"'));
    best = narrowRange(best, best, findCustomRange(elementText, start, end, '(', ')'));

    TextRange natural = findNaturalRange(elementText, start, end);
    if (!natural.contains(best)) {
      return Collections.emptyList();
    }

    TextRange paragraph = findParagraphRange(elementText, start, end);
    if (best.getStartOffset() == start && best.getEndOffset() == end) {
      return Arrays.asList(paragraph.shiftRight(shift));
    }


    return Arrays.asList(best.shiftRight(shift));
 }

  private static TextRange narrowRange(TextRange sentence, TextRange best, TextRange candidate) {
    return candidate != null && best.contains(candidate) ? candidate : best;
  }

  private static boolean isNatural(char c) {
    return Character.isWhitespace(c) || Character.isLetterOrDigit(c) || NATURAL.contains(c);
  }
}
{'fraction_non_alphanumeric': 0.11935730680948738, 'fraction_numerical': 0.004699967209531097, 'mean_word_length': 3.867021276595745, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5533664', 'n_tokens_mistral': 2709, 'n_tokens_neox': 2589, 'n_words': 680}