<commit_before>package net.hillsdon.reviki.wiki.renderer.creole;

import java.util.ArrayList;
import java.util.List;

import org.antlr.v4.runtime.ANTLRErrorStrategy;
import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.DefaultErrorStrategy;
import org.antlr.v4.runtime.atn.PredictionMode;
import org.antlr.v4.runtime.tree.ParseTree;

import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;

import net.hillsdon.reviki.vc.PageInfo;
import net.hillsdon.reviki.vc.PageStore;
import net.hillsdon.reviki.web.urls.URLOutputFilter;
import net.hillsdon.reviki.wiki.renderer.creole.ast.ASTNode;
import net.hillsdon.reviki.wiki.renderer.creole.links.LinkPartsHandler;
import net.hillsdon.reviki.wiki.renderer.macro.Macro;

/**
 * Renderer for the Reviki dialect of WikiCreole.
 *
 * @author msw
 */
public class CreoleRenderer {
  /**
   * How deep macros will be expanded
   */
  public static final int MACRO_DEPTH_LIMIT = 100;
  
  /**
   * Macro expansion depth limit. This gets reset when rendering a page, but not
   * when rendering a partial page. If it hits zero, no macros are expanded.
   */
  private static int _expansionLimit;

  /**
   * Try to run a parser, resetting the input on failure.
   *
   * @param tokens The token stream. Consumed by the parser, and reset on
   *          failure.
   * @param parser The parser. Reset on failure.
   * @param errors The error recovery strategy.
   * @param pmode The prediction mode.
   * @return A parse tree.
   */
  private static ParseTree tryParse(final CommonTokenStream tokens, final Creole parser, final ANTLRErrorStrategy errors, final PredictionMode pmode) {
    parser.setErrorHandler(errors);
    parser.getInterpreter().setPredictionMode(pmode);
    try {
      return parser.creole();
    }
    catch (RuntimeException ex) {
      tokens.reset();
      parser.reset();
      throw ex;
    }
  }

  /**
   * Helper for tryParse using the default error strategy. See
   * {@link #tryParse(CommonTokenStream, Creole, ANTLRErrorStrategy, PredictionMode)}
   */
  private static ParseTree tryParse(final CommonTokenStream tokens, final Creole parser, final PredictionMode pmode) {
    return tryParse(tokens, parser, new DefaultErrorStrategy(), pmode);
  }

  /**
   * Render a stream of text.
   *
   * @param in The input stream to render.
   * @param visitor The visitor to do the rendering.
   * @param macros List of macros to apply.
   * @param reset Whether to reset the expansion limit or not.
   * @return The AST of the page, after macro expansion.
   */
  private static ASTNode renderInternal(final ANTLRInputStream in, final CreoleASTBuilder visitor, final Supplier<List<Macro>> macros, final boolean reset) {
    CreoleTokens lexer = new CreoleTokens(in);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    Creole parser = new Creole(tokens);

    // First try parsing in SLL mode. This is really fast for pages with no
    // parse errors.
    ParseTree tree;

    try {
      tree = tryParse(tokens, parser, PredictionMode.SLL);
    }
    catch (Exception e1) {
      tree = tryParse(tokens, parser, PredictionMode.LL);
    }

    ASTNode rendered = visitor.visit(tree);

    // Expand macros
    if (reset) {
      _expansionLimit = MACRO_DEPTH_LIMIT;
    }

    ASTNode expanded = rendered;
    _expansionLimit--;

    if (_expansionLimit >= 0) {
      expanded = rendered.expandMacros(macros);
    }<commit_msg>Remove exceptions from tryParse<commit_after>package net.hillsdon.reviki.wiki.renderer.creole;

import java.util.ArrayList;
import java.util.List;

import org.antlr.v4.runtime.ANTLRErrorStrategy;
import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.DefaultErrorStrategy;
import org.antlr.v4.runtime.atn.PredictionMode;
import org.antlr.v4.runtime.tree.ParseTree;

import com.google.common.base.Optional;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;

import net.hillsdon.reviki.vc.PageInfo;
import net.hillsdon.reviki.vc.PageStore;
import net.hillsdon.reviki.web.urls.URLOutputFilter;
import net.hillsdon.reviki.wiki.renderer.creole.ast.ASTNode;
import net.hillsdon.reviki.wiki.renderer.creole.links.LinkPartsHandler;
import net.hillsdon.reviki.wiki.renderer.macro.Macro;

/**
 * Renderer for the Reviki dialect of WikiCreole.
 *
 * @author msw
 */
public class CreoleRenderer {
  /**
   * How deep macros will be expanded
   */
  public static final int MACRO_DEPTH_LIMIT = 100;

  /**
   * Macro expansion depth limit. This gets reset when rendering a page, but not
   * when rendering a partial page. If it hits zero, no macros are expanded.
   */
  private static int _expansionLimit;

  /**
   * Try to run a parser, resetting the input on failure.
   *
   * @param tokens The token stream. Consumed by the parser, and reset on
   *          failure.
   * @param parser The parser. Reset on failure.
   * @param errors The error recovery strategy.
   * @param pmode The prediction mode.
   * @return A parse tree.
   */
  private static Optional<ParseTree> tryParse(final CommonTokenStream tokens, final Creole parser, final ANTLRErrorStrategy errors, final PredictionMode pmode) {
    parser.setErrorHandler(errors);
    parser.getInterpreter().setPredictionMode(pmode);

    try {
      return Optional.of((ParseTree) parser.creole());
    }
    catch (Exception e) {
      tokens.reset();
      parser.reset();

      return Optional.<ParseTree> absent();
    }
  }

  /**
   * Helper for tryParse using the default error strategy. See
   * {@link #tryParse(CommonTokenStream, Creole, ANTLRErrorStrategy, PredictionMode)}
   */
  private static Optional<ParseTree> tryParse(final CommonTokenStream tokens, final Creole parser, final PredictionMode pmode) {
    return tryParse(tokens, parser, new DefaultErrorStrategy(), pmode);
  }

  /**
   * Render a stream of text.
   *
   * @param in The input stream to render.
   * @param visitor The visitor to do the rendering.
   * @param macros List of macros to apply.
   * @param reset Whether to reset the expansion limit or not.
   * @return The AST of the page, after macro expansion.
   */
  private static ASTNode renderInternal(final ANTLRInputStream in, final CreoleASTBuilder visitor, final Supplier<List<Macro>> macros, final boolean reset) {
    CreoleTokens lexer = new CreoleTokens(in);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    Creole parser = new Creole(tokens);

    // First try parsing in SLL mode. This is really fast for pages with no
    // parse errors.
    Optional<ParseTree> tree = tryParse(tokens, parser, PredictionMode.SLL);

    if (!tree.isPresent()) {
      tree = tryParse(tokens, parser, PredictionMode.LL);
    }

    ASTNode rendered = visitor.visit(tree.get());

    // Expand macros
    if (reset) {
      _expansionLimit = MACRO_DEPTH_LIMIT;
    }

    ASTNode expanded = rendered;
    _expansionLimit--;

    if (_expansionLimit >= 0) {
      expanded = rendered.expandMacros(macros);
    }
{'fraction_non_alphanumeric': 0.09528501055594651, 'fraction_numerical': 0.002955665024630542, 'mean_word_length': 3.54638515674984, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3019606', 'n_tokens_mistral': 2196, 'n_tokens_neox': 2143, 'n_words': 630}