<commit_before>package com.jetbrains.python.documentation.doctest;

import com.intellij.openapi.module.Module;
import com.intellij.openapi.module.ModuleUtilCore;
import com.intellij.openapi.util.TextRange;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.psi.InjectedLanguagePlaces;
import com.intellij.psi.LanguageInjector;
import com.intellij.psi.PsiLanguageInjectionHost;
import com.intellij.psi.util.PsiTreeUtil;
import com.jetbrains.python.documentation.PyDocumentationSettings;
import com.jetbrains.python.psi.PyDocStringOwner;
import com.jetbrains.python.psi.PyStringLiteralExpression;
import org.jetbrains.annotations.NotNull;

import java.util.List;

/**
 * User: ktisha
 */
public class PyDocstringLanguageInjector implements LanguageInjector {
  @Override
  public void getLanguagesToInject(@NotNull final PsiLanguageInjectionHost host, @NotNull final InjectedLanguagePlaces injectionPlacesRegistrar) {
    if (!(host instanceof PyStringLiteralExpression)) {
      return;
    }
    final Module module = ModuleUtilCore.findModuleForPsiElement(host);
    if (module == null || !PyDocumentationSettings.getInstance(module).analyzeDoctest) return;

    final PyDocStringOwner
      docStringOwner = PsiTreeUtil.getParentOfType(host, PyDocStringOwner.class);
    if (docStringOwner != null && host.equals(docStringOwner.getDocStringExpression())) {
      int start = 0;
      int end = host.getTextLength() - 1;
      final String text = host.getText();
      final List<String> strings = StringUtil.split(text, "\n", false);

      boolean gotExample = false;

      int currentPosition = 0;
      int maxPosition = text.length();
      boolean endsWithSlash = false;
      for (String string : strings) {
        final String trimmedString = string.trim();
        if (!trimmedString.startsWith(">>>") && !trimmedString.startsWith("...") && gotExample && start < end) {
          gotExample = false;
          if (!endsWithSlash)
            injectionPlacesRegistrar.addPlace(PyDocstringLanguageDialect.getInstance(), TextRange.create(start, end),  null, null);
        }
        if (endsWithSlash) {
          endsWithSlash = false;
          injectionPlacesRegistrar.addPlace(PyDocstringLanguageDialect.getInstance(),
                                            TextRange.create(start, getEndOffset(currentPosition, string, maxPosition)),  null, null);
        }

        if (trimmedString.startsWith(">>>")) {
          if (trimmedString.endsWith("\\"))
            endsWithSlash = true;

          if (!gotExample)
            start = currentPosition;

          gotExample = true;
          end = getEndOffset(currentPosition, string, maxPosition);
        }<commit_msg>Docstrings highlighting: Do not stop on the first line after backslash<commit_after>package com.jetbrains.python.documentation.doctest;

import com.intellij.openapi.module.Module;
import com.intellij.openapi.module.ModuleUtilCore;
import com.intellij.openapi.util.TextRange;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.psi.InjectedLanguagePlaces;
import com.intellij.psi.LanguageInjector;
import com.intellij.psi.PsiLanguageInjectionHost;
import com.intellij.psi.util.PsiTreeUtil;
import com.jetbrains.python.documentation.PyDocumentationSettings;
import com.jetbrains.python.psi.PyDocStringOwner;
import com.jetbrains.python.psi.PyStringLiteralExpression;
import org.jetbrains.annotations.NotNull;

import java.util.List;

/**
 * User: ktisha
 */
public class PyDocstringLanguageInjector implements LanguageInjector {
  @Override
  public void getLanguagesToInject(@NotNull final PsiLanguageInjectionHost host, @NotNull final InjectedLanguagePlaces injectionPlacesRegistrar) {
    if (!(host instanceof PyStringLiteralExpression)) {
      return;
    }
    final Module module = ModuleUtilCore.findModuleForPsiElement(host);
    if (module == null || !PyDocumentationSettings.getInstance(module).analyzeDoctest) return;

    final PyDocStringOwner
      docStringOwner = PsiTreeUtil.getParentOfType(host, PyDocStringOwner.class);
    if (docStringOwner != null && host.equals(docStringOwner.getDocStringExpression())) {
      int start = 0;
      int end = host.getTextLength() - 1;
      final String text = host.getText();
      final List<String> strings = StringUtil.split(text, "\n", false);

      boolean gotExample = false;

      int currentPosition = 0;
      int maxPosition = text.length();
      boolean endsWithSlash = false;
      for (String string : strings) {
        final String trimmedString = string.trim();
        if (!trimmedString.startsWith(">>>") && !trimmedString.startsWith("...") && gotExample && start < end) {
          gotExample = false;
          if (!endsWithSlash)
            injectionPlacesRegistrar.addPlace(PyDocstringLanguageDialect.getInstance(), TextRange.create(start, end),  null, null);
        }
        if (endsWithSlash && !trimmedString.endsWith("\\")) {
          endsWithSlash = false;
          injectionPlacesRegistrar.addPlace(PyDocstringLanguageDialect.getInstance(),
                                            TextRange.create(start, getEndOffset(currentPosition, string, maxPosition)),  null, null);
        }

        if (trimmedString.startsWith(">>>")) {
          if (trimmedString.endsWith("\\"))
            endsWithSlash = true;

          if (!gotExample)
            start = currentPosition;

          gotExample = true;
          end = getEndOffset(currentPosition, string, maxPosition);
        }
{'fraction_non_alphanumeric': 0.10414394765539804, 'fraction_numerical': 0.0010905125408942203, 'mean_word_length': 3.570598006644518, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4234449', 'n_tokens_mistral': 1551, 'n_tokens_neox': 1528, 'n_words': 307}