<commit_before>package org.jetbrains.plugins.scala.lang.psi.impl

import com.intellij.ide.highlighter.JavaFileType
import com.intellij.openapi.project.ProjectUtil
import com.intellij.openapi.vfs.VirtualFile
import com.intellij.psi.impl.PsiTreeChangeEventImpl
import com.intellij.psi.impl.source.tree.LeafPsiElement
import com.intellij.psi.tree.TokenSet
import com.intellij.psi.util.PsiTreeUtil
import com.intellij.psi.{PsiComment, PsiElement, PsiTreeChangeEvent, TokenType}
import org.jetbrains.plugins.scala.ScalaFileType
import org.jetbrains.plugins.scala.extensions.ObjectExt
import org.jetbrains.plugins.scala.lang.lexer.{ScalaTokenType, ScalaTokenTypes}
import org.jetbrains.plugins.scala.lang.psi.impl.source.ScalaCodeFragment

private sealed trait ScalaPsiEventFilter {
  def shouldSkip(event: PsiTreeChangeEvent): Boolean = false
  def shouldSkip(element: PsiElement): Boolean = false
}

private object ScalaPsiEventFilter {

  def defaultFilters: Seq[ScalaPsiEventFilter] =
    Seq(InternalEventsFilter, NonSignificantParentsFilter, IgnoreLeafElementsFilter)

  private object InternalEventsFilter extends ScalaPsiEventFilter {
    override def shouldSkip(event: PsiTreeChangeEvent): Boolean =
      isGenericChange(event) || isFromIdeaInternalFile(event)

    private def isFromIdeaInternalFile(event: PsiTreeChangeEvent) = {
      val vFile = event.getFile match {
        case null => event.getOldValue.asOptionOf[VirtualFile]
        case file =>
          val fileType = file.getFileType
          if (fileType == ScalaFileType.INSTANCE || fileType == JavaFileType.INSTANCE) None
          else Option(file.getVirtualFile)
      }
      vFile.exists(ProjectUtil.isProjectOrWorkspaceFile)
    }

    private def isGenericChange(event: PsiTreeChangeEvent) = event match {
      case impl: PsiTreeChangeEventImpl => impl.isGenericChange
      case _ => false
    }
  }

  private object NonSignificantParentsFilter extends ScalaPsiEventFilter {
    override def shouldSkip(element: PsiElement): Boolean = {
      // do not update on changes in dummy file or comments
      PsiTreeUtil.getParentOfType(element, classOf[ScalaCodeFragment], classOf[PsiComment]) != null
    }
  }

  private object IgnoreLeafElementsFilter extends ScalaPsiEventFilter {

    import ScalaTokenTypes._
    import TokenType._

    private val significantTokens: TokenSet = {
      TokenSet.orSet(IDENTIFIER_TOKEN_SET, KEYWORDS, LITERALS)
    }

    override def shouldSkip(element: PsiElement): Boolean = {
      element match {
        case leaf: LeafPsiElement =>
          leaf.getElementType match {
            case ERROR_ELEMENT | BAD_CHARACTER | WHITE_SPACE => true
            case scalaToken: ScalaTokenType                  => !significantTokens.contains(scalaToken)
            case _                                           => false
          }
        case _ => false
      }
    }
  }
}
<commit_msg>PsiErrorElement has no children, but is not a LeafElement<commit_after>package org.jetbrains.plugins.scala.lang.psi.impl

import com.intellij.ide.highlighter.JavaFileType
import com.intellij.openapi.project.ProjectUtil
import com.intellij.openapi.vfs.VirtualFile
import com.intellij.psi.impl.PsiTreeChangeEventImpl
import com.intellij.psi.impl.source.tree.LeafPsiElement
import com.intellij.psi.tree.TokenSet
import com.intellij.psi.util.PsiTreeUtil
import com.intellij.psi.{PsiComment, PsiElement, PsiErrorElement, PsiTreeChangeEvent, TokenType}
import org.jetbrains.plugins.scala.ScalaFileType
import org.jetbrains.plugins.scala.extensions.ObjectExt
import org.jetbrains.plugins.scala.lang.lexer.{ScalaTokenType, ScalaTokenTypes}
import org.jetbrains.plugins.scala.lang.psi.impl.source.ScalaCodeFragment

private sealed trait ScalaPsiEventFilter {
  def shouldSkip(event: PsiTreeChangeEvent): Boolean = false
  def shouldSkip(element: PsiElement): Boolean = false
}

private object ScalaPsiEventFilter {

  def defaultFilters: Seq[ScalaPsiEventFilter] =
    Seq(InternalEventsFilter, NonSignificantParentsFilter, IgnoreLeafElementsFilter)

  private object InternalEventsFilter extends ScalaPsiEventFilter {
    override def shouldSkip(event: PsiTreeChangeEvent): Boolean =
      isGenericChange(event) || isFromIdeaInternalFile(event)

    private def isFromIdeaInternalFile(event: PsiTreeChangeEvent) = {
      val vFile = event.getFile match {
        case null => event.getOldValue.asOptionOf[VirtualFile]
        case file =>
          val fileType = file.getFileType
          if (fileType == ScalaFileType.INSTANCE || fileType == JavaFileType.INSTANCE) None
          else Option(file.getVirtualFile)
      }
      vFile.exists(ProjectUtil.isProjectOrWorkspaceFile)
    }

    private def isGenericChange(event: PsiTreeChangeEvent) = event match {
      case impl: PsiTreeChangeEventImpl => impl.isGenericChange
      case _ => false
    }
  }

  private object NonSignificantParentsFilter extends ScalaPsiEventFilter {
    override def shouldSkip(element: PsiElement): Boolean = {
      // do not update on changes in dummy file or comments
      PsiTreeUtil.getParentOfType(element, classOf[ScalaCodeFragment], classOf[PsiComment]) != null
    }
  }

  private object IgnoreLeafElementsFilter extends ScalaPsiEventFilter {

    import ScalaTokenTypes._
    import TokenType._

    private val significantTokens: TokenSet = {
      TokenSet.orSet(IDENTIFIER_TOKEN_SET, KEYWORDS, LITERALS)
    }

    override def shouldSkip(element: PsiElement): Boolean = {
      element match {
        case leaf: LeafPsiElement =>
          leaf.getElementType match {
            case WHITE_SPACE | BAD_CHARACTER => true
            case scalaToken: ScalaTokenType  => !significantTokens.contains(scalaToken)
            case _                           => false
          }
        case _: PsiErrorElement              => true
        case _                               => false
      }
    }
  }
}

{'fraction_non_alphanumeric': 0.07590647238224331, 'fraction_numerical': 0.0, 'mean_word_length': 3.547765793528505, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7396083', 'n_tokens_mistral': 1727, 'n_tokens_neox': 1670, 'n_words': 345}