<commit_before>package com.intellij.codeInsight.daemon.impl;

import com.intellij.codeInsight.daemon.impl.analysis.HighlightInfoHolder;
import com.intellij.codeInsight.daemon.impl.analysis.HighlightUtil;
import com.intellij.lang.annotation.Annotation;
import com.intellij.lang.annotation.ExternalAnnotator;
import com.intellij.lang.annotation.HighlightSeverity;
import com.intellij.openapi.editor.Editor;
import com.intellij.openapi.progress.ProgressIndicator;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.application.ApplicationManager;
import com.intellij.openapi.application.ModalityState;
import com.intellij.openapi.util.Computable;
import com.intellij.psi.PsiFile;

import javax.swing.*;
import java.util.List;
import java.util.ArrayList;
import java.lang.reflect.InvocationTargetException;

/**
 * @author ven
 */
public class ExternalToolPass extends TextEditorHighlightingPass {
  private final PsiFile myFile;
  private AnnotationHolderImpl myAnnotationHolder;
  HighlightInfoHolder myHolder;
  private Project myProject;

  public ExternalToolPass(PsiFile file,
                          Editor editor) {
    super(editor.getDocument());
    myFile = file;
    myProject = file.getProject();
    myAnnotationHolder = new AnnotationHolderImpl();
  }

  public void doCollectInformation(ProgressIndicator progress) {
    if (!HighlightUtil.isRootHighlighted(myFile)) return;
    final ExternalAnnotator externalAnnotator = myFile.getLanguage().getExternalAnnotator();
    
    if (externalAnnotator != null) {
      final HighlightInfo[] errors = DaemonCodeAnalyzerImpl.getHighlights(myDocument, HighlightSeverity.ERROR, myProject);
      
      if ( errors.length > 0) {
        return;
      }
      externalAnnotator.annotate(myFile, myAnnotationHolder);
    }
  }

  public void doApplyInformationToEditor() {
    if (myAnnotationHolder.hasAnnotations()) {
      List<HighlightInfo> infos = getHighlights();

      UpdateHighlightersUtil.setHighlightersToEditor(myProject, myDocument, 0, myFile.getTextLength(),
                                                     infos, UpdateHighlightersUtil.EXTERNAL_TOOLS_HIGHLIGHTERS_GROUP);
    }
  }

  public List<HighlightInfo> getHighlights() {
    Annotation[] annotations = myAnnotationHolder.getResult();
    List<HighlightInfo> infos = new ArrayList<HighlightInfo>();
    for (Annotation annotation : annotations) {
      infos.add(HighlightUtil.convertToHighlightInfo(annotation));
    }
    return infos;
  }

  public int getPassId() {
    return Pass.EXTERNAL_TOOLS;
  }
}
<commit_msg>external tool annotator works iff root is inspected only<commit_after>package com.intellij.codeInsight.daemon.impl;

import com.intellij.codeInsight.daemon.impl.analysis.HighlightInfoHolder;
import com.intellij.codeInsight.daemon.impl.analysis.HighlightUtil;
import com.intellij.lang.annotation.Annotation;
import com.intellij.lang.annotation.ExternalAnnotator;
import com.intellij.lang.annotation.HighlightSeverity;
import com.intellij.openapi.editor.Editor;
import com.intellij.openapi.progress.ProgressIndicator;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.application.ApplicationManager;
import com.intellij.openapi.application.ModalityState;
import com.intellij.openapi.util.Computable;
import com.intellij.psi.PsiFile;

import javax.swing.*;
import java.util.List;
import java.util.ArrayList;
import java.lang.reflect.InvocationTargetException;

/**
 * @author ven
 */
public class ExternalToolPass extends TextEditorHighlightingPass {
  private final PsiFile myFile;
  private AnnotationHolderImpl myAnnotationHolder;
  HighlightInfoHolder myHolder;
  private Project myProject;

  public ExternalToolPass(PsiFile file,
                          Editor editor) {
    super(editor.getDocument());
    myFile = file;
    myProject = file.getProject();
    myAnnotationHolder = new AnnotationHolderImpl();
  }

  public void doCollectInformation(ProgressIndicator progress) {
    if (!HighlightUtil.isRootInspected(myFile)) return;
    final ExternalAnnotator externalAnnotator = myFile.getLanguage().getExternalAnnotator();
    
    if (externalAnnotator != null) {
      final HighlightInfo[] errors = DaemonCodeAnalyzerImpl.getHighlights(myDocument, HighlightSeverity.ERROR, myProject);
      
      if ( errors.length > 0) {
        return;
      }
      externalAnnotator.annotate(myFile, myAnnotationHolder);
    }
  }

  public void doApplyInformationToEditor() {
    if (myAnnotationHolder.hasAnnotations()) {
      List<HighlightInfo> infos = getHighlights();

      UpdateHighlightersUtil.setHighlightersToEditor(myProject, myDocument, 0, myFile.getTextLength(),
                                                     infos, UpdateHighlightersUtil.EXTERNAL_TOOLS_HIGHLIGHTERS_GROUP);
    }
  }

  public List<HighlightInfo> getHighlights() {
    Annotation[] annotations = myAnnotationHolder.getResult();
    List<HighlightInfo> infos = new ArrayList<HighlightInfo>();
    for (Annotation annotation : annotations) {
      infos.add(HighlightUtil.convertToHighlightInfo(annotation));
    }
    return infos;
  }

  public int getPassId() {
    return Pass.EXTERNAL_TOOLS;
  }
}

{'fraction_non_alphanumeric': 0.09120646526842409, 'fraction_numerical': 0.0007696748123917645, 'mean_word_length': 4.255813953488372, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4939871', 'n_tokens_mistral': 1530, 'n_tokens_neox': 1477, 'n_words': 264}