<commit_before>// This class is a runtime engine for parsers generated by Grammar-Kit.
// Do not modify or refactor without complete investigation and/or review.
package com.intellij.lang.parser;

import com.intellij.lang.*;
import com.intellij.lang.impl.PsiBuilderAdapter;
import com.intellij.lang.impl.PsiBuilderImpl;
import com.intellij.lexer.Lexer;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.Comparing;
import com.intellij.openapi.util.Key;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.text.StringHash;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.psi.PsiFile;
import com.intellij.psi.PsiReference;
import com.intellij.psi.TokenType;
import com.intellij.psi.impl.source.resolve.FileContextUtil;
import com.intellij.psi.impl.source.tree.CompositePsiElement;
import com.intellij.psi.tree.ICompositeElementType;
import com.intellij.psi.tree.IElementType;
import com.intellij.psi.tree.TokenSet;
import com.intellij.util.Function;
import com.intellij.util.PairProcessor;
import com.intellij.util.containers.LimitedPool;
import gnu.trove.THashSet;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.LinkedList;

/**
 * @author gregsh
 */
public class GeneratedParserUtilBase {

  private static final Logger LOG = Logger.getInstance("org.intellij.grammar.parser.GeneratedParserUtilBase");

  private static final int MAX_RECURSION_LEVEL = 1000;
  private static final int MAX_VARIANTS_SIZE = 10000;
  private static final int MAX_VARIANTS_TO_DISPLAY = 50;

  private static final int INITIAL_VARIANTS_SIZE = 1000;
  private static final int VARIANTS_POOL_SIZE = 10000;
  private static final int FRAMES_POOL_SIZE = 500;

  public static final IElementType DUMMY_BLOCK = new DummyBlockElementType();

  public interface Parser {
    boolean parse(PsiBuilder builder, int level);
  }

  public static final Parser TOKEN_ADVANCER = new Parser() {
    @Override
    public boolean parse(PsiBuilder builder, int level) {
      if (builder.eof()) return false;
      builder.advanceLexer();
      return true;
    }
  };

  public static final Parser TRUE_CONDITION = new Parser() {
    @Override
    public boolean parse(PsiBuilder builder, int level) {
      return true;
    }
  };

  public static boolean eof(PsiBuilder builder_, int level_) {
    return builder_.eof();
  }

  public static int current_position_(PsiBuilder builder_) {
    return builder_.rawTokenIndex();
  }

  public static boolean recursion_guard_(PsiBuilder builder_, int level_, String funcName_) {
    if (level_ > MAX_RECURSION_LEVEL) {
      builder_.error("Maximum recursion level (" + MAX_RECURSION_LEVEL + ") reached in '" + funcName_ + "'");
      return false;
    }
    return true;
  }

  @Deprecated
  public static void empty_element_parsed_guard_(PsiBuilder builder_, int offset_, String funcName_) {
    empty_element_parsed_guard_(builder_, funcName_);
  }

  public static void empty_element_parsed_guard_(PsiBuilder builder_, String funcName_) {
    builder_.error("Empty element parsed in '" + funcName_ +"' at offset " + builder_.getCurrentOffset());
  }

  public static boolean invalid_left_marker_guard_(PsiBuilder builder_, PsiBuilder.Marker marker_, String funcName_) {
    //builder_.error("Invalid left marker encountered in " + funcName_ +" at offset " + builder_.getCurrentOffset());
    boolean goodMarker = marker_ != null; // && ((LighterASTNode)marker_).getTokenType() != TokenType.ERROR_ELEMENT;
    if (!goodMarker) return false;
    ErrorState state = ErrorState.get(builder_);

    Frame frame = state.frameStack.peekLast();
    return frame == null || frame.errorReportedAt <= builder_.getCurrentOffset();
  }

  public static TokenSet create_token_set_(IElementType... tokenTypes_) {
    return TokenSet.create(tokenTypes_);
  }<commit_msg>introduce boolean empty_element_parsed_guard_()<commit_after>// This class is a runtime engine for parsers generated by Grammar-Kit.
// Do not modify or refactor without complete investigation and/or review.
package com.intellij.lang.parser;

import com.intellij.lang.*;
import com.intellij.lang.impl.PsiBuilderAdapter;
import com.intellij.lang.impl.PsiBuilderImpl;
import com.intellij.lexer.Lexer;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.Comparing;
import com.intellij.openapi.util.Key;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.text.StringHash;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.psi.PsiFile;
import com.intellij.psi.PsiReference;
import com.intellij.psi.TokenType;
import com.intellij.psi.impl.source.resolve.FileContextUtil;
import com.intellij.psi.impl.source.tree.CompositePsiElement;
import com.intellij.psi.tree.ICompositeElementType;
import com.intellij.psi.tree.IElementType;
import com.intellij.psi.tree.TokenSet;
import com.intellij.util.Function;
import com.intellij.util.PairProcessor;
import com.intellij.util.containers.LimitedPool;
import gnu.trove.THashSet;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.LinkedList;

/**
 * @author gregsh
 */
public class GeneratedParserUtilBase {

  private static final Logger LOG = Logger.getInstance("org.intellij.grammar.parser.GeneratedParserUtilBase");

  private static final int MAX_RECURSION_LEVEL = 1000;
  private static final int MAX_VARIANTS_SIZE = 10000;
  private static final int MAX_VARIANTS_TO_DISPLAY = 50;

  private static final int INITIAL_VARIANTS_SIZE = 1000;
  private static final int VARIANTS_POOL_SIZE = 10000;
  private static final int FRAMES_POOL_SIZE = 500;

  public static final IElementType DUMMY_BLOCK = new DummyBlockElementType();

  public interface Parser {
    boolean parse(PsiBuilder builder, int level);
  }

  public static final Parser TOKEN_ADVANCER = new Parser() {
    @Override
    public boolean parse(PsiBuilder builder, int level) {
      if (builder.eof()) return false;
      builder.advanceLexer();
      return true;
    }
  };

  public static final Parser TRUE_CONDITION = new Parser() {
    @Override
    public boolean parse(PsiBuilder builder, int level) {
      return true;
    }
  };

  public static boolean eof(PsiBuilder builder_, int level_) {
    return builder_.eof();
  }

  public static int current_position_(PsiBuilder builder_) {
    return builder_.rawTokenIndex();
  }

  public static boolean recursion_guard_(PsiBuilder builder_, int level_, String funcName_) {
    if (level_ > MAX_RECURSION_LEVEL) {
      builder_.error("Maximum recursion level (" + MAX_RECURSION_LEVEL + ") reached in '" + funcName_ + "'");
      return false;
    }
    return true;
  }

  @Deprecated
  public static void empty_element_parsed_guard_(PsiBuilder builder_, int offset_, String funcName_) {
    builder_.error("Empty element parsed in '" + funcName_ + "' at offset " + offset_);
  }

  public static boolean empty_element_parsed_guard_(PsiBuilder builder_, String funcName_, int prev_position_) {
    if (prev_position_ == current_position_(builder_)) {
      builder_.error("Empty element parsed in '" + funcName_ + "' at offset " + builder_.getCurrentOffset());
      return false;
    }
    return true;
  }

  public static boolean invalid_left_marker_guard_(PsiBuilder builder_, PsiBuilder.Marker marker_, String funcName_) {
    //builder_.error("Invalid left marker encountered in " + funcName_ +" at offset " + builder_.getCurrentOffset());
    boolean goodMarker = marker_ != null; // && ((LighterASTNode)marker_).getTokenType() != TokenType.ERROR_ELEMENT;
    if (!goodMarker) return false;
    ErrorState state = ErrorState.get(builder_);

    Frame frame = state.frameStack.peekLast();
    return frame == null || frame.errorReportedAt <= builder_.getCurrentOffset();
  }

  public static TokenSet create_token_set_(IElementType... tokenTypes_) {
    return TokenSet.create(tokenTypes_);
  }
{'fraction_non_alphanumeric': 0.09584664536741214, 'fraction_numerical': 0.005652494470385844, 'mean_word_length': 4.616977225672878, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3875249', 'n_tokens_mistral': 2521, 'n_tokens_neox': 2431, 'n_words': 599}