<commit_before>package org.jetbrains.plugins.scala.lang.parser.util

import org.jetbrains.plugins.scala.lang.lexer.ScalaTokenTypes
import org.jetbrains.plugins.scala.lang.lexer.ScalaElementType
import org.jetbrains.plugins.scala.lang.parser.ScalaElementTypes
import org.jetbrains.plugins.scala.lang.parser.parsing.ConstrItem
import org.jetbrains.plugins.scala.lang.parser.parsing.Constr
import com.intellij.psi.tree.IElementType

import com.intellij.lang.PsiBuilder

object ParserUtils {

  def rollForward(builder: PsiBuilder) : Unit = {
    var flag = true
    while ( !builder.eof() && flag){
       builder.getTokenType match{
         case ScalaTokenTypes.tLINE_TERMINATOR => builder.advanceLexer
         case _ => flag = false
       }
    }
  }

  //Write element node
  def eatElement(builder: PsiBuilder, elem: IElementType): Unit = {
    val marker = builder.mark()
    builder.advanceLexer // Ate something
    marker.done(elem)
  }

  def listOfSmth(builder: PsiBuilder, itemType : ConstrItem, delimiter : IElementType, listType : IElementType) : Unit = {
    val listMarker = builder.mark()

    if (itemType.first.contains(builder.getTokenType)) {
      itemType parse builder
    } else {
      builder error "wrong item"
      listMarker.drop()
      return
    }

    while (builder.getTokenType.equals(delimiter)) {
      eatElement(builder, delimiter);

      if (itemType.first.contains(builder.getTokenType)) {
        itemType parse builder
      } else {
        builder error "expected next item"
        listMarker.drop()
        return
      }
    }

    listMarker.done(listType)
  }

  /*
   *   Parse block in breaces
   *
   *   smthInBraces parse block between braces : expected pair '{ }', '[ ]', '{ }'
   *   @param builder - PsiBuilder of parser
   *   @param constr - The construction in block
   *   @param leftBrace - The left brace of block
   *   @param rightBrace - The right brace of block
   *   @param blockType - The type of block
   *
   *   @return parsed block
   */

  def smthInBraces(builder: PsiBuilder, constr : Constr, leftBrace : IElementType, rightBrace : IElementType, blockType : IElementType) : IElementType= {
    val blockMarker = builder.mark()

    if (leftBrace.equals(builder.getTokenType)) {
      eatElement(builder, leftBrace)

      constr parse builder

      if (rightBrace.equals(builder.getTokenType)) {
        eatElement(builder, rightBrace)
      } else {
        builder error "expected right brace"
        return ScalaElementTypes.WRONGWAY
      }

    } else {
      builder error "expected left brace"
      return ScalaElementTypes.WRONGWAY
    }

    blockMarker.done(blockType)
    blockType
  }

  def eatConstr(builder : PsiBuilder, constr: Constr, element : IElementType) : IElementType = {
    val marker = builder.mark()
    //Console.println("before parsing " + constr + " : " + builder.getTokenType())
    constr.parse(builder)
    //Console.println("after parsing " + constr + " : " + builder.getTokenType())
    marker.done(element)
    element
  }
  

  //Write element node
  def errorToken(builder: PsiBuilder,
                 marker: PsiBuilder.Marker ,
                 msg: String,
                 elem: ScalaElementType): ScalaElementType = {
    builder.error(msg)
    //marker.done(elem)
    marker.rollbackTo()
    ScalaElementTypes.WRONGWAY
  }

}<commit_msg>minor cleanup<commit_after>package org.jetbrains.plugins.scala.lang.parser.util

import org.jetbrains.plugins.scala.lang.lexer.ScalaTokenTypes
import org.jetbrains.plugins.scala.lang.lexer.ScalaElementType
import org.jetbrains.plugins.scala.lang.parser.ScalaElementTypes
import org.jetbrains.plugins.scala.lang.parser.parsing.ConstrItem
import org.jetbrains.plugins.scala.lang.parser.parsing.Constr
import com.intellij.psi.tree.IElementType

import com.intellij.lang.PsiBuilder

object ParserUtils {

  def rollForward(builder: PsiBuilder) : Unit = {
    while (!builder.eof()){
       builder.getTokenType match{
         case ScalaTokenTypes.tLINE_TERMINATOR => builder.advanceLexer
         case _ => return
       }
    }
  }

  //Write element node
  def eatElement(builder: PsiBuilder, elem: IElementType): Unit = {
    val marker = builder.mark()
    builder.advanceLexer // Ate something
    marker.done(elem)
  }

  def listOfSmth(builder: PsiBuilder, itemType : ConstrItem, delimiter : IElementType, listType : IElementType) : Unit = {
    val listMarker = builder.mark()

    if (itemType.first.contains(builder.getTokenType)) {
      itemType parse builder
    } else {
      builder error "wrong item"
      listMarker.drop()
      return
    }

    while (builder.getTokenType.equals(delimiter)) {
      eatElement(builder, delimiter);

      if (itemType.first.contains(builder.getTokenType)) {
        itemType parse builder
      } else {
        builder error "expected next item"
        listMarker.drop()
        return
      }
    }

    listMarker.done(listType)
  }

  /*
   *   Parse block in breaces
   *
   *   smthInBraces parse block between braces : expected pair '{ }', '[ ]', '{ }'
   *   @param builder - PsiBuilder of parser
   *   @param constr - The construction in block
   *   @param leftBrace - The left brace of block
   *   @param rightBrace - The right brace of block
   *   @param blockType - The type of block
   *
   *   @return parsed block
   */

  def smthInBraces(builder: PsiBuilder, constr : Constr, leftBrace : IElementType, rightBrace : IElementType, blockType : IElementType) : IElementType= {
    val blockMarker = builder.mark()

    if (leftBrace.equals(builder.getTokenType)) {
      eatElement(builder, leftBrace)

      constr parse builder

      if (rightBrace.equals(builder.getTokenType)) {
        eatElement(builder, rightBrace)
      } else {
        builder error "expected right brace"
        return ScalaElementTypes.WRONGWAY
      }

    } else {
      builder error "expected left brace"
      return ScalaElementTypes.WRONGWAY
    }

    blockMarker.done(blockType)
    blockType
  }

  def eatConstr(builder : PsiBuilder, constr: Constr, element : IElementType) : IElementType = {
    val marker = builder.mark()
    //Console.println("before parsing " + constr + " : " + builder.getTokenType())
    constr.parse(builder)
    //Console.println("after parsing " + constr + " : " + builder.getTokenType())
    marker.done(element)
    element
  }
  

  //Write element node
  def errorToken(builder: PsiBuilder,
                 marker: PsiBuilder.Marker ,
                 msg: String,
                 elem: ScalaElementType): ScalaElementType = {
    builder.error(msg)
    //marker.done(elem)
    marker.rollbackTo()
    ScalaElementTypes.WRONGWAY
  }

}
{'fraction_non_alphanumeric': 0.09708176827506501, 'fraction_numerical': 0.0, 'mean_word_length': 2.5213631739572735, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6272407', 'n_tokens_mistral': 2187, 'n_tokens_neox': 2033, 'n_words': 442}