<commit_before>package org.jetbrains.plugins.scala
package lang
package parser
package util

import _root_.scala.collection.mutable._

import org.jetbrains.plugins.scala.lang.lexer.ScalaTokenTypes
import org.jetbrains.plugins.scala.lang.lexer.ScalaElementType
import org.jetbrains.plugins.scala.lang.parser.ScalaElementTypes
import org.jetbrains.plugins.scala.util.DebugPrint
import com.intellij.psi.tree.IElementType
import com.intellij.psi.tree.TokenSet
import com.intellij.lang.PsiBuilder
import parsing.builder.ScalaPsiBuilder
import com.intellij.lang.PsiBuilder.Marker
import annotation.tailrec


object ParserUtils extends ParserUtilsBase {

  def lookAheadSeq(n: Int)(builder: PsiBuilder) = (1 to n).map(i => {
    val token = if (!builder.eof) builder.getTokenType else null
    builder.advanceLexer
    token
  })

  //Write element node
  def eatElement(builder: PsiBuilder, elem: IElementType): Unit = {
    if (!builder.eof()) {
      builder.advanceLexer // Ate something
    }
    ()

  }

  def parseTillLast(builder: PsiBuilder, lastSet: TokenSet): Unit = {
    while (!builder.eof() && !lastSet.contains(builder.getTokenType)) {
      builder.advanceLexer
      DebugPrint println "an error"
    }

    if (builder.eof()) /*builder error "unexpected end of file"; */ return

    if (lastSet.contains(builder.getTokenType)) builder advanceLexer;
    return
  }

  def eatSeqWildcardNext(builder: PsiBuilder): Boolean = {
    val marker = builder.mark
    if (builder.getTokenType == ScalaTokenTypes.tUNDER) {
      builder.advanceLexer
      if (builder.getTokenType == ScalaTokenTypes.tIDENTIFIER &&
              builder.getTokenText == "*") {
        builder.advanceLexer
        marker.done(ScalaElementTypes.SEQ_WILDCARD)
        true
      } else {
        marker.rollbackTo
        false
      }
    } else {
      marker.drop
      false
    }
  }


  def build(t : IElementType, builder : PsiBuilder)  (inner : => Boolean) : Boolean = {
    val marker = builder.mark
    val parsed = inner
    if (parsed) marker.done(t) else marker.rollbackTo
    parsed
  }

  def isAssignmentOperator(id: String) = id.charAt(id.length - 1) match {
    case '=' if id != "<=" && id != ">=" && id != "!=" => true
    case _ => false
  }

  //Defines priority
  def priority(id: String, assignments: Boolean = false): Int = {
    if (assignments && isAssignmentOperator(id)) {
      return 4
    }
    id.charAt(0) match {
      case '~' | '#' | '@' | '$' | '?' | '\\' => 0
      case '*' | '/' | '%' => 1
      case '+' | '-' => 2
      case ':' => 3
      case '=' | '!' => 4
      case '<' | '>' => 5
      case '&' => 6
      case '^' => 7
      case '|' => 8
      case _ => 9
    }
  }

  def caseLookAheadFunction(builder: ScalaPsiBuilder): IElementType = {
    val marker: Marker = builder.mark
    builder.advanceLexer
    val res = builder.getTokenType
    marker.rollbackTo
    res
  }

  @tailrec
  def parseLoopUntilRBrace(builder: ScalaPsiBuilder, fun: () => Unit, braceReported: Boolean = false) {
    var br = braceReported
    fun()
    builder.getTokenType match {
      case ScalaTokenTypes.tRBRACE =>
        builder.advanceLexer
        return
      case _ =>
        if (!braceReported) {
          builder error ErrMsg("rbrace.expected")
          br = true
        }
        builder.advanceLexer
        if (builder.eof) {
          return
        }
    }
    parseLoopUntilRBrace(builder, fun, br)
  }

  def elementCanStartStatement(element: IElementType, builder: ScalaPsiBuilder): Boolean = {
    element match {
      case ScalaTokenTypes.kCATCH => false
      case ScalaTokenTypes.kELSE => false
      case ScalaTokenTypes.kEXTENDS => false
      case ScalaTokenTypes.kFINALLY => false
      case ScalaTokenTypes.kMATCH => false
      case ScalaTokenTypes.kWITH => false
      case ScalaTokenTypes.kYIELD => false
      case ScalaTokenTypes.tCOMMA => false
      case ScalaTokenTypes.tDOT => false<commit_msg>Fixed exception from parser during reparse LazyParseableElement.<commit_after>package org.jetbrains.plugins.scala
package lang
package parser
package util

import _root_.scala.collection.mutable._

import org.jetbrains.plugins.scala.lang.lexer.ScalaTokenTypes
import org.jetbrains.plugins.scala.lang.lexer.ScalaElementType
import org.jetbrains.plugins.scala.lang.parser.ScalaElementTypes
import org.jetbrains.plugins.scala.util.DebugPrint
import com.intellij.psi.tree.IElementType
import com.intellij.psi.tree.TokenSet
import com.intellij.lang.PsiBuilder
import parsing.builder.ScalaPsiBuilder
import com.intellij.lang.PsiBuilder.Marker
import annotation.tailrec


object ParserUtils extends ParserUtilsBase {

  def lookAheadSeq(n: Int)(builder: PsiBuilder) = (1 to n).map(i => {
    val token = if (!builder.eof) builder.getTokenType else null
    builder.advanceLexer
    token
  })

  //Write element node
  def eatElement(builder: PsiBuilder, elem: IElementType): Unit = {
    if (!builder.eof()) {
      builder.advanceLexer // Ate something
    }
    ()

  }

  def parseTillLast(builder: PsiBuilder, lastSet: TokenSet): Unit = {
    while (!builder.eof() && !lastSet.contains(builder.getTokenType)) {
      builder.advanceLexer
      DebugPrint println "an error"
    }

    if (builder.eof()) /*builder error "unexpected end of file"; */ return

    if (lastSet.contains(builder.getTokenType)) builder advanceLexer;
    return
  }

  def eatSeqWildcardNext(builder: PsiBuilder): Boolean = {
    val marker = builder.mark
    if (builder.getTokenType == ScalaTokenTypes.tUNDER) {
      builder.advanceLexer
      if (builder.getTokenType == ScalaTokenTypes.tIDENTIFIER &&
              builder.getTokenText == "*") {
        builder.advanceLexer
        marker.done(ScalaElementTypes.SEQ_WILDCARD)
        true
      } else {
        marker.rollbackTo
        false
      }
    } else {
      marker.drop
      false
    }
  }


  def build(t : IElementType, builder : PsiBuilder)  (inner : => Boolean) : Boolean = {
    val marker = builder.mark
    val parsed = inner
    if (parsed) marker.done(t) else marker.rollbackTo
    parsed
  }

  def isAssignmentOperator(id: String) = id.charAt(id.length - 1) match {
    case '=' if id != "<=" && id != ">=" && id != "!=" => true
    case _ => false
  }

  //Defines priority
  def priority(id: String, assignments: Boolean = false): Int = {
    if (assignments && isAssignmentOperator(id)) {
      return 4
    }
    id.charAt(0) match {
      case '~' | '#' | '@' | '$' | '?' | '\\' => 0
      case '*' | '/' | '%' => 1
      case '+' | '-' => 2
      case ':' => 3
      case '=' | '!' => 4
      case '<' | '>' => 5
      case '&' => 6
      case '^' => 7
      case '|' => 8
      case _ => 9
    }
  }

  def caseLookAheadFunction(builder: ScalaPsiBuilder): IElementType = {
    val marker: Marker = builder.mark
    builder.advanceLexer
    val res = builder.getTokenType
    marker.rollbackTo
    res
  }

  @tailrec
  def parseLoopUntilRBrace(builder: ScalaPsiBuilder, fun: () => Unit, braceReported: Boolean = false) {
    var br = braceReported
    fun()
    builder.getTokenType match {
      case ScalaTokenTypes.tRBRACE =>
        builder.advanceLexer
        return
      case ScalaTokenTypes.tLBRACE => //to avoid missing '{'
        if (!braceReported) {
          builder error ErrMsg("rbrace.expected")
          br = true
        }
        var balance = 1
        builder.advanceLexer
        while (balance != 0 && !builder.eof) {
          builder.getTokenType match {
            case ScalaTokenTypes.tRBRACE => balance -= 1
            case ScalaTokenTypes.tLBRACE => balance += 1
            case _ =>
          }
          builder.advanceLexer
        }
        if (builder.eof)
          return
      case _ =>
        if (!braceReported) {
          builder error ErrMsg("rbrace.expected")
          br = true
        }
        builder.advanceLexer
        if (builder.eof) {
          return
        }
    }
    parseLoopUntilRBrace(builder, fun, br)
  }

  def elementCanStartStatement(element: IElementType, builder: ScalaPsiBuilder): Boolean = {
    element match {
      case ScalaTokenTypes.kCATCH => false
      case ScalaTokenTypes.kELSE => false
      case ScalaTokenTypes.kEXTENDS => false
      case ScalaTokenTypes.kFINALLY => false
      case ScalaTokenTypes.kMATCH => false
      case ScalaTokenTypes.kWITH => false
      case ScalaTokenTypes.kYIELD => false
      case ScalaTokenTypes.tCOMMA => false
      case ScalaTokenTypes.tDOT => false
{'fraction_non_alphanumeric': 0.11536648597690313, 'fraction_numerical': 0.0037709168041480086, 'mean_word_length': 2.619189765458422, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6889488', 'n_tokens_mistral': 2755, 'n_tokens_neox': 2647, 'n_words': 578}