<commit_before>package org.jetbrains.plugins.scala.lang.parser.parsing.top.template {

import com.intellij.lang.PsiBuilder
import com.intellij.psi.tree.IElementType
import com.intellij.psi.tree.TokenSet

import org.jetbrains.plugins.scala.lang.lexer.ScalaTokenTypes
import org.jetbrains.plugins.scala.lang.parser.ScalaElementTypes
import org.jetbrains.plugins.scala.lang.parser.parsing.base.Constructor
import org.jetbrains.plugins.scala.lang.parser.parsing.base.Import
import org.jetbrains.plugins.scala.lang.parser.parsing.base.AttributeClause
import org.jetbrains.plugins.scala.lang.parser.parsing.base.Modifiers
import org.jetbrains.plugins.scala.lang.parser.parsing.base.Ids
import org.jetbrains.plugins.scala.lang.parser.util.ParserUtils
import org.jetbrains.plugins.scala.util.DebugPrint
import org.jetbrains.plugins.scala.lang.parser.parsing.types.Type
import org.jetbrains.plugins.scala.lang.parser.parsing.types.SimpleType
import org.jetbrains.plugins.scala.lang.parser.bnf.BNF
import org.jetbrains.plugins.scala.lang.parser.parsing.expressions.Expr
import org.jetbrains.plugins.scala.lang.parser.parsing.base.StatementSeparator

/**
 * User: Dmitry.Krasilschikov
 * Date: 30.10.2006
 * Time: 15:04:19
 */

/*
 *  Template ::= TemplateParents [TemplateBody]
 */
 
object Template extends Constr{
  override def getElementType = ScalaElementTypes.TEMPLATE

  override def parseBody (builder : PsiBuilder) : Unit = {
    if (BNF.firstTemplateParents.contains(builder.getTokenType)){
      new TemplateParents parse builder
    } else builder error "template parents expected"

    if (BNF.firstTemplateBody.contains(builder.getTokenType)){
      TemplateBody parse builder
    }
  }
} 

/*
 *  TemplateParents ::= Constr {with SimpleType}
 */

  class TemplateParents extends ConstrItem {
    override def getElementType = ScalaElementTypes.TEMPLATE_PARENTS

    override def first = BNF.firstTemplateParents
    
    override def parseBody(builder : PsiBuilder) : Unit = {
      if (BNF.firstTemplateParents.contains(builder.getTokenType)) {
        Constructor.parse(builder)
      } else builder.error("identifier expected")

      while (ScalaTokenTypes.kWITH.equals(builder.getTokenType)) {
        ParserUtils.eatElement(builder, ScalaTokenTypes.kWITH)

        if (BNF.firstSimpleType.contains(builder.getTokenType)) {
          SimpleType.parse(builder)
        }
      }
    }
  }

/*
 *  TemplateBody ::= { TemplateStatSeq }
 */  

  object TemplateBody extends Constr {
    override def getElementType = ScalaElementTypes.TEMPLATE_BODY

    override def parseBody(builder : PsiBuilder) : Unit = {
      if (ScalaTokenTypes.tLBRACE.equals(builder.getTokenType)) {
        ParserUtils.eatElement(builder, ScalaTokenTypes.tLBRACE)
      } else {
        builder error "'{' expected"
        return
      }

      if (BNF.firstTemplateStatSeq.contains(builder.getTokenType)) {
        TemplateStatSeq parse builder
      }

      if (ScalaTokenTypes.tRBRACE.equals(builder.getTokenType)) {
        ParserUtils.eatElement(builder, ScalaTokenTypes.tRBRACE)
      } else {
        builder error "'}' expected"
        return
      }
    }
}

/*
 *  TemplateStatSeq ::= [TemplateStat] {StatementSeparator [TemplateStat}]
 */

  object TemplateStatSeq extends ConstrWithoutNode {
    override def parseBody (builder: PsiBuilder): Unit = {

      var isLocalError = false;
      var isError = false;
      var isEnd = false;

      while (!builder.eof && !isEnd) {

        isLocalError = false

        while (BNF.firstStatementSeparator.contains(builder.getTokenType)) {
          StatementSeparator parse builder
        }

        if (BNF.firstTemplateStat.contains(builder.getTokenType)) {
          TemplateStat.parse(builder)
        }

        if (ScalaTokenTypes.tRBRACE.equals(builder.getTokenType) || builder.eof) {
          isEnd = true;
          return
        }

        if (/*!isEnd && */!BNF.firstStatementSeparator.contains(builder.getTokenType)) {
          isLocalError = true;
          builder error "template statement declaration error"

           builder.getTokenType match {
              case ScalaTokenTypes.tRBRACE |
                   ScalaTokenTypes.tRSQBRACKET |
                   ScalaTokenTypes.tRPARENTHESIS  => return

              case _ => {}
            }


          if (!BNF.firstTemplateStat.contains(builder.getTokenType)) {
            tryParseSmth(builder)
          }
        }
       isError = isError || isLocalError
      }
    }

    def tryParseSmth (builder : PsiBuilder) : Unit = {
      var isAfterBlock = false;
      var unstructuredTrashMarker : PsiBuilder.Marker = builder.mark;

      while (!builder.eof){

        if (BNF.firstTemplateStat.contains(builder.getTokenType)) {
          TemplateStat parse builder
        } else {

          builder.getTokenType match {
            case ScalaTokenTypes.tLBRACE => unstructuredTrashMarker.done(ScalaElementTypes.TRASH); parseTemplateStatSeqInBlock(builder); return;//unstructuredTrashMarker = builder.mark
            case _ => {builder.advanceLexer}
          }
        }
      }

      unstructuredTrashMarker.drop
    }

    def parseTemplateStatSeqInBlock (builder : PsiBuilder) : Unit = {
      val trashBlockMarker = builder.mark

      builder.getTokenType match {
        case ScalaTokenTypes.tLBRACE |
             ScalaTokenTypes.tLSQBRACKET |
             ScalaTokenTypes.tLPARENTHESIS => builder.advanceLexer

        case _ => {builder error "open brace expected"; trashBlockMarker.drop; return}
      }<commit_msg>parser cleanup<commit_after>package org.jetbrains.plugins.scala.lang.parser.parsing.top.template {

import com.intellij.lang.PsiBuilder
import com.intellij.psi.tree.IElementType
import com.intellij.psi.tree.TokenSet

import org.jetbrains.plugins.scala.lang.lexer.ScalaTokenTypes
import org.jetbrains.plugins.scala.lang.parser.ScalaElementTypes
import org.jetbrains.plugins.scala.lang.parser.parsing.base.Constructor
import org.jetbrains.plugins.scala.lang.parser.parsing.base.Import
import org.jetbrains.plugins.scala.lang.parser.parsing.base.AttributeClause
import org.jetbrains.plugins.scala.lang.parser.parsing.base.Modifiers
import org.jetbrains.plugins.scala.lang.parser.parsing.base.Ids
import org.jetbrains.plugins.scala.lang.parser.util.ParserUtils
import org.jetbrains.plugins.scala.util.DebugPrint
import org.jetbrains.plugins.scala.lang.parser.parsing.types.Type
import org.jetbrains.plugins.scala.lang.parser.parsing.types.SimpleType
import org.jetbrains.plugins.scala.lang.parser.bnf.BNF
import org.jetbrains.plugins.scala.lang.parser.parsing.expressions.Expr
import org.jetbrains.plugins.scala.lang.parser.parsing.base.StatementSeparator

/**
 * User: Dmitry.Krasilschikov
 * Date: 30.10.2006
 * Time: 15:04:19
 */

/*
 *  Template ::= TemplateParents [TemplateBody]
 */
 
object Template extends Constr{
  override def getElementType = ScalaElementTypes.TEMPLATE

  override def parseBody (builder : PsiBuilder) : Unit = {
    if (BNF.firstTemplateParents.contains(builder.getTokenType)){
      new TemplateParents parse builder
    } else builder error "template parents expected"

    if (BNF.firstTemplateBody.contains(builder.getTokenType)){
      TemplateBody parse builder
    }
  }
} 

/*
 *  TemplateParents ::= Constr {with SimpleType}
 */

  class TemplateParents extends ConstrItem {
    override def getElementType = ScalaElementTypes.TEMPLATE_PARENTS

    override def first = BNF.firstTemplateParents
    
    override def parseBody(builder : PsiBuilder) : Unit = {
      if (BNF.firstTemplateParents.contains(builder.getTokenType)) {
        Constructor.parse(builder)
      } else builder.error("identifier expected")

      while (ScalaTokenTypes.kWITH.equals(builder.getTokenType)) {
        ParserUtils.eatElement(builder, ScalaTokenTypes.kWITH)

        if (BNF.firstSimpleType.contains(builder.getTokenType)) {
          SimpleType.parse(builder)
        }
      }
    }
  }

/*
 *  TemplateBody ::= { TemplateStatSeq }
 */  

  object TemplateBody extends Constr {
    override def getElementType = ScalaElementTypes.TEMPLATE_BODY

    override def parseBody(builder : PsiBuilder) : Unit = {
      if (ScalaTokenTypes.tLBRACE.equals(builder.getTokenType)) {
        ParserUtils.eatElement(builder, ScalaTokenTypes.tLBRACE)
      } else {
        builder error "'{' expected"
        return
      }

      if (BNF.firstTemplateStatSeq.contains(builder.getTokenType)) {
        TemplateStatSeq parse builder
      }

      if (ScalaTokenTypes.tRBRACE.equals(builder.getTokenType)) {
        ParserUtils.eatElement(builder, ScalaTokenTypes.tRBRACE)
      } else {
        builder error "'}' expected"
        return
      }
    }
}

/*
 *  TemplateStatSeq ::= [TemplateStat] {StatementSeparator [TemplateStat}]
 */

  object TemplateStatSeq extends ConstrWithoutNode {
    override def parseBody (builder: PsiBuilder): Unit = {

      var isLocalError = false;
      var isError = false;
      var isEnd = false;

      while (!builder.eof && !isEnd) {

        isLocalError = false

        while (BNF.firstStatementSeparator.contains(builder.getTokenType)) {
          StatementSeparator parse builder
        }

        if (BNF.firstTemplateStat.contains(builder.getTokenType)) {
          TemplateStat.parse(builder)
        }

        if (ScalaTokenTypes.tRBRACE.equals(builder.getTokenType) || builder.eof) {
          isEnd = true;
          return
        }

        if (/*!isEnd && */!BNF.firstStatementSeparator.contains(builder.getTokenType)) {
          isLocalError = true;
          builder error "template statement declaration error"

           builder.getTokenType match {
              case ScalaTokenTypes.tRBRACE |
                   ScalaTokenTypes.tRSQBRACKET |
                   ScalaTokenTypes.tRPARENTHESIS  => return

              case _ => {}
            }


          if (!BNF.firstTemplateStat.contains(builder.getTokenType)) {
            tryParseSmth(builder)
          }
        }
       isError = isError || isLocalError
      }
    }

    def tryParseSmth (builder : PsiBuilder) : Unit = {
      var isAfterBlock = false;
      var unstructuredTrashMarker : PsiBuilder.Marker = builder.mark;

      while (!builder.eof){

        if (BNF.firstTemplateStat.contains(builder.getTokenType)) {
          TemplateStat parse builder
        } else {

          builder.getTokenType match {
            case ScalaTokenTypes.tLBRACE => unstructuredTrashMarker.done(ScalaElementTypes.TRASH); parseTemplateStatSeqInBlock(builder); return;//unstructuredTrashMarker = builder.mark
            case _ => {builder.advanceLexer}
          }
        }
      }

      unstructuredTrashMarker.drop
    }

    def parseTemplateStatSeqInBlock (builder : PsiBuilder) : Unit = {
      val trashBlockMarker = builder.mark

      builder.getTokenType match {
        case ScalaTokenTypes.tLBRACE |
             ScalaTokenTypes.tLSQBRACKET |
             ScalaTokenTypes.tLPARENTHESIS => builder.advanceLexer

        case _ => {builder error "'{' or '[' or '(' expected"; trashBlockMarker.drop; return}
      }
{'fraction_non_alphanumeric': 0.09589411129119395, 'fraction_numerical': 0.0025211597334773997, 'mean_word_length': 2.9540761836952654, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1566511', 'n_tokens_mistral': 3389, 'n_tokens_neox': 3219, 'n_words': 539}