<commit_before>/**********************************************************************
 * Copyright (c) 2014 HubSpot Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 **********************************************************************/

package com.hubspot.jinjava.interpret;

import static com.hubspot.jinjava.util.Logging.ENGINE_LOG;

import com.google.common.base.Strings;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import com.hubspot.jinjava.Jinjava;
import com.hubspot.jinjava.JinjavaConfig;
import com.hubspot.jinjava.el.ExpressionResolver;
import com.hubspot.jinjava.el.ext.DeferredParsingException;
import com.hubspot.jinjava.el.ext.ExtendedParser;
import com.hubspot.jinjava.interpret.TemplateError.ErrorItem;
import com.hubspot.jinjava.interpret.TemplateError.ErrorReason;
import com.hubspot.jinjava.interpret.TemplateError.ErrorType;
import com.hubspot.jinjava.interpret.errorcategory.BasicTemplateErrorCategory;
import com.hubspot.jinjava.lib.tag.ExtendsTag;
import com.hubspot.jinjava.lib.tag.eager.EagerGenericTag;
import com.hubspot.jinjava.objects.serialization.PyishObjectMapper;
import com.hubspot.jinjava.objects.serialization.PyishSerializable;
import com.hubspot.jinjava.random.ConstantZeroRandomNumberGenerator;
import com.hubspot.jinjava.random.DeferredRandomNumberGenerator;
import com.hubspot.jinjava.tree.Node;
import com.hubspot.jinjava.tree.TagNode;
import com.hubspot.jinjava.tree.TreeParser;
import com.hubspot.jinjava.tree.output.BlockInfo;
import com.hubspot.jinjava.tree.output.BlockPlaceholderOutputNode;
import com.hubspot.jinjava.tree.output.OutputList;
import com.hubspot.jinjava.tree.output.OutputNode;
import com.hubspot.jinjava.tree.output.RenderedOutputNode;
import com.hubspot.jinjava.util.Variable;
import com.hubspot.jinjava.util.WhitespaceUtils;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Random;
import java.util.Set;
import java.util.Stack;
import java.util.concurrent.ThreadLocalRandom;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;

public class JinjavaInterpreter implements PyishSerializable {
  private final Multimap<String, BlockInfo> blocks = ArrayListMultimap.create();
  private final LinkedList<Node> extendParentRoots = new LinkedList<>();

  private Context context;
  private final JinjavaConfig config;

  private final ExpressionResolver expressionResolver;
  private final Jinjava application;
  private final Random random;

  private int lineNumber = -1;
  private int position = 0;
  private int scopeDepth = 1;
  private final List<TemplateError> errors = new LinkedList<>();
  private static final int MAX_ERROR_SIZE = 100;

  public JinjavaInterpreter(
    Jinjava application,
    Context context,
    JinjavaConfig renderConfig
  ) {
    this.context = context;
    this.config = renderConfig;
    this.application = application;

    this.config.getExecutionMode().prepareContext(this.context);

    switch (config.getRandomNumberGeneratorStrategy()) {
      case THREAD_LOCAL:
        random = ThreadLocalRandom.current();
        break;
      case CONSTANT_ZERO:
        random = new ConstantZeroRandomNumberGenerator();
        break;
      case DEFERRED:
        random = new DeferredRandomNumberGenerator();
        break;
      default:
        throw new IllegalStateException(
          "No random number generator with strategy " +
          config.getRandomNumberGeneratorStrategy()
        );
    }

    this.expressionResolver = new ExpressionResolver(this, application);
  }

  public JinjavaInterpreter(JinjavaInterpreter orig) {
    this(orig.application, new Context(orig.context), orig.config);
    scopeDepth = orig.getScopeDepth() + 1;
  }

  /**
   * @deprecated use {{@link #getConfig()}}
   */
  @Deprecated
  public JinjavaConfig getConfiguration() {
    return config;
  }

  public void addExtendParentRoot(Node root) {
    extendParentRoots.add(root);
  }

  public void addBlock(String name, BlockInfo blockInfo) {
    blocks.put(name, blockInfo);
  }

  /**
   * Creates a new variable scope, extending from the current scope. Allows you to create a nested
   * contextual scope which can override variables from higher levels.
   *
   * Should be used in a try/finally context, similar to lock-use patterns:
   *
   * <code>
   * interpreter.enterScope();
   * try (interpreter.enterScope()) {
   *   // ...
   * }
   * </code>
   */
  public InterpreterScopeClosable enterScope() {
    return enterScope(null);
  }

  public InterpreterScopeClosable enterScope(Map<Context.Library, Set<String>> disabled) {
    context = new Context(context, null, disabled);
    scopeDepth++;
    return new InterpreterScopeClosable();
  }

  public InterpreterScopeClosable enterNonStackingScope() {
    context = new Context(context, null, null, false);
    scopeDepth++;
    return new InterpreterScopeClosable();
  }

  public void leaveScope() {
    Context parent = context.getParent();
    scopeDepth--;
    if (parent != null) {
      parent.addDependencies(context.getDependencies());
      context = parent;
    }
  }

  public Random getRandom() {
    return random;
  }

  public boolean isValidationMode() {
    return config.isValidationMode();
  }

  public class InterpreterScopeClosable implements AutoCloseable {

    @Override
    public void close() {
      leaveScope();
    }
  }

  public Node parse(String template) {
    return new TreeParser(this, template).buildTree();
  }

  /**
   * Parse the given string into a root Node, and then render it without processing any extend parents.
   * This method should be used when the template is known to not have any extends or block tags.
   *
   * @param template
   *          string to parse
   * @return rendered result
   */
  public String renderFlat(String template) {
    int depth = context.getRenderDepth();

    try {
      if (depth > config.getMaxRenderDepth()) {
        ENGINE_LOG.warn("Max render depth exceeded: {}", Integer.toString(depth));
        return template;
      } else {
        context.setRenderDepth(depth + 1);
        return render(parse(template), false);
      }
    } finally {
      context.setRenderDepth(depth);
    }
  }

  /**
   * Parse the given string into a root Node, and then renders it processing extend parents.
   *
   * @param template
   *          string to parse
   * @return rendered result
   */
  public String render(String template) {
    return render(parse(template), true);
  }

  /**
   * Render the given root node, processing extend parents. Equivalent to render(root, true)
   *
   * @param root
   *          node to render
   * @return rendered result
   */
  public String render(Node root) {
    return render(root, true);
  }

  /**
   * Render the given root node using this interpreter's current context
   *
   * @param root
   *          node to render
   * @param processExtendRoots
   *          if true, also render all extend parents
   * @return rendered result
   */
  public String render(Node root, boolean processExtendRoots) {
    OutputList output = new OutputList(config.getMaxOutputSize());

    for (Node node : root.getChildren()) {
      lineNumber = node.getLineNumber();
      position = node.getStartPosition();
      String renderStr = node.getMaster().getImage();
      try {
        if (context.doesRenderStackContain(renderStr)) {
          // This is a circular rendering. Stop rendering it here.
          addError(
            new TemplateError(
              ErrorType.WARNING,
              ErrorReason.EXCEPTION,
              ErrorItem.TAG,
              "Rendering cycle detected: '" + renderStr + "'",
              null,
              getLineNumber(),
              node.getStartPosition(),
              null,
              BasicTemplateErrorCategory.IMPORT_CYCLE_DETECTED,
              ImmutableMap.of("string", renderStr)
            )
          );
          output.addNode(new RenderedOutputNode(renderStr));
        } else {
          OutputNode out;
          context.pushRenderStack(renderStr);
          try {
            out = node.render(this);
          } catch (DeferredValueException e) {
            context.handleDeferredNode(node);
            out = new RenderedOutputNode(node.getMaster().getImage());
          }
          context.popRenderStack();
          output.addNode(out);
        }
      } catch (OutputTooBigException e) {
        addError(TemplateError.fromOutputTooBigException(e));
        return output.getValue();
      } catch (CollectionTooBigException e) {
        addError(
          new TemplateError(
            ErrorType.FATAL,
            ErrorReason.COLLECTION_TOO_BIG,
            ErrorItem.OTHER,
            ExceptionUtils.getMessage(e),
            null,
            -1,
            -1,
            e,
            BasicTemplateErrorCategory.UNKNOWN,
            ImmutableMap.of()
          )
        );
        return output.getValue();
      }
    }

    // render all extend parents, keeping the last as the root output
    if (processExtendRoots) {
      Optional<String> extendPath = context.getExtendPathStack().peek();
      while (!extendParentRoots.isEmpty()) {<commit_msg>Break from extending if the same path is encountered multiple times<commit_after>/**********************************************************************
 * Copyright (c) 2014 HubSpot Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 **********************************************************************/

package com.hubspot.jinjava.interpret;

import static com.hubspot.jinjava.util.Logging.ENGINE_LOG;

import com.google.common.base.Strings;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import com.hubspot.jinjava.Jinjava;
import com.hubspot.jinjava.JinjavaConfig;
import com.hubspot.jinjava.el.ExpressionResolver;
import com.hubspot.jinjava.el.ext.DeferredParsingException;
import com.hubspot.jinjava.el.ext.ExtendedParser;
import com.hubspot.jinjava.interpret.TemplateError.ErrorItem;
import com.hubspot.jinjava.interpret.TemplateError.ErrorReason;
import com.hubspot.jinjava.interpret.TemplateError.ErrorType;
import com.hubspot.jinjava.interpret.errorcategory.BasicTemplateErrorCategory;
import com.hubspot.jinjava.lib.tag.ExtendsTag;
import com.hubspot.jinjava.lib.tag.eager.EagerGenericTag;
import com.hubspot.jinjava.objects.serialization.PyishObjectMapper;
import com.hubspot.jinjava.objects.serialization.PyishSerializable;
import com.hubspot.jinjava.random.ConstantZeroRandomNumberGenerator;
import com.hubspot.jinjava.random.DeferredRandomNumberGenerator;
import com.hubspot.jinjava.tree.Node;
import com.hubspot.jinjava.tree.TagNode;
import com.hubspot.jinjava.tree.TreeParser;
import com.hubspot.jinjava.tree.output.BlockInfo;
import com.hubspot.jinjava.tree.output.BlockPlaceholderOutputNode;
import com.hubspot.jinjava.tree.output.OutputList;
import com.hubspot.jinjava.tree.output.OutputNode;
import com.hubspot.jinjava.tree.output.RenderedOutputNode;
import com.hubspot.jinjava.util.Variable;
import com.hubspot.jinjava.util.WhitespaceUtils;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Random;
import java.util.Set;
import java.util.Stack;
import java.util.concurrent.ThreadLocalRandom;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;

public class JinjavaInterpreter implements PyishSerializable {
  private final Multimap<String, BlockInfo> blocks = ArrayListMultimap.create();
  private final LinkedList<Node> extendParentRoots = new LinkedList<>();

  private Context context;
  private final JinjavaConfig config;

  private final ExpressionResolver expressionResolver;
  private final Jinjava application;
  private final Random random;

  private int lineNumber = -1;
  private int position = 0;
  private int scopeDepth = 1;
  private final List<TemplateError> errors = new LinkedList<>();
  private static final int MAX_ERROR_SIZE = 100;

  public JinjavaInterpreter(
    Jinjava application,
    Context context,
    JinjavaConfig renderConfig
  ) {
    this.context = context;
    this.config = renderConfig;
    this.application = application;

    this.config.getExecutionMode().prepareContext(this.context);

    switch (config.getRandomNumberGeneratorStrategy()) {
      case THREAD_LOCAL:
        random = ThreadLocalRandom.current();
        break;
      case CONSTANT_ZERO:
        random = new ConstantZeroRandomNumberGenerator();
        break;
      case DEFERRED:
        random = new DeferredRandomNumberGenerator();
        break;
      default:
        throw new IllegalStateException(
          "No random number generator with strategy " +
          config.getRandomNumberGeneratorStrategy()
        );
    }

    this.expressionResolver = new ExpressionResolver(this, application);
  }

  public JinjavaInterpreter(JinjavaInterpreter orig) {
    this(orig.application, new Context(orig.context), orig.config);
    scopeDepth = orig.getScopeDepth() + 1;
  }

  /**
   * @deprecated use {{@link #getConfig()}}
   */
  @Deprecated
  public JinjavaConfig getConfiguration() {
    return config;
  }

  public void addExtendParentRoot(Node root) {
    extendParentRoots.add(root);
  }

  public void addBlock(String name, BlockInfo blockInfo) {
    blocks.put(name, blockInfo);
  }

  /**
   * Creates a new variable scope, extending from the current scope. Allows you to create a nested
   * contextual scope which can override variables from higher levels.
   *
   * Should be used in a try/finally context, similar to lock-use patterns:
   *
   * <code>
   * interpreter.enterScope();
   * try (interpreter.enterScope()) {
   *   // ...
   * }
   * </code>
   */
  public InterpreterScopeClosable enterScope() {
    return enterScope(null);
  }

  public InterpreterScopeClosable enterScope(Map<Context.Library, Set<String>> disabled) {
    context = new Context(context, null, disabled);
    scopeDepth++;
    return new InterpreterScopeClosable();
  }

  public InterpreterScopeClosable enterNonStackingScope() {
    context = new Context(context, null, null, false);
    scopeDepth++;
    return new InterpreterScopeClosable();
  }

  public void leaveScope() {
    Context parent = context.getParent();
    scopeDepth--;
    if (parent != null) {
      parent.addDependencies(context.getDependencies());
      context = parent;
    }
  }

  public Random getRandom() {
    return random;
  }

  public boolean isValidationMode() {
    return config.isValidationMode();
  }

  public class InterpreterScopeClosable implements AutoCloseable {

    @Override
    public void close() {
      leaveScope();
    }
  }

  public Node parse(String template) {
    return new TreeParser(this, template).buildTree();
  }

  /**
   * Parse the given string into a root Node, and then render it without processing any extend parents.
   * This method should be used when the template is known to not have any extends or block tags.
   *
   * @param template
   *          string to parse
   * @return rendered result
   */
  public String renderFlat(String template) {
    int depth = context.getRenderDepth();

    try {
      if (depth > config.getMaxRenderDepth()) {
        ENGINE_LOG.warn("Max render depth exceeded: {}", Integer.toString(depth));
        return template;
      } else {
        context.setRenderDepth(depth + 1);
        return render(parse(template), false);
      }
    } finally {
      context.setRenderDepth(depth);
    }
  }

  /**
   * Parse the given string into a root Node, and then renders it processing extend parents.
   *
   * @param template
   *          string to parse
   * @return rendered result
   */
  public String render(String template) {
    return render(parse(template), true);
  }

  /**
   * Render the given root node, processing extend parents. Equivalent to render(root, true)
   *
   * @param root
   *          node to render
   * @return rendered result
   */
  public String render(Node root) {
    return render(root, true);
  }

  /**
   * Render the given root node using this interpreter's current context
   *
   * @param root
   *          node to render
   * @param processExtendRoots
   *          if true, also render all extend parents
   * @return rendered result
   */
  public String render(Node root, boolean processExtendRoots) {
    OutputList output = new OutputList(config.getMaxOutputSize());

    for (Node node : root.getChildren()) {
      lineNumber = node.getLineNumber();
      position = node.getStartPosition();
      String renderStr = node.getMaster().getImage();
      try {
        if (context.doesRenderStackContain(renderStr)) {
          // This is a circular rendering. Stop rendering it here.
          addError(
            new TemplateError(
              ErrorType.WARNING,
              ErrorReason.EXCEPTION,
              ErrorItem.TAG,
              "Rendering cycle detected: '" + renderStr + "'",
              null,
              getLineNumber(),
              node.getStartPosition(),
              null,
              BasicTemplateErrorCategory.IMPORT_CYCLE_DETECTED,
              ImmutableMap.of("string", renderStr)
            )
          );
          output.addNode(new RenderedOutputNode(renderStr));
        } else {
          OutputNode out;
          context.pushRenderStack(renderStr);
          try {
            out = node.render(this);
          } catch (DeferredValueException e) {
            context.handleDeferredNode(node);
            out = new RenderedOutputNode(node.getMaster().getImage());
          }
          context.popRenderStack();
          output.addNode(out);
        }
      } catch (OutputTooBigException e) {
        addError(TemplateError.fromOutputTooBigException(e));
        return output.getValue();
      } catch (CollectionTooBigException e) {
        addError(
          new TemplateError(
            ErrorType.FATAL,
            ErrorReason.COLLECTION_TOO_BIG,
            ErrorItem.OTHER,
            ExceptionUtils.getMessage(e),
            null,
            -1,
            -1,
            e,
            BasicTemplateErrorCategory.UNKNOWN,
            ImmutableMap.of()
          )
        );
        return output.getValue();
      }
    }

    // render all extend parents, keeping the last as the root output
    if (processExtendRoots) {
      Set<String> extendPaths = new HashSet<>();
      Optional<String> extendPath = context.getExtendPathStack().peek();
      while (!extendParentRoots.isEmpty()) {
        if (extendPaths.contains(extendPath.orElse(""))) {
          addError(
            TemplateError.fromException(
              new ExtendsTagCycleException(
                extendPath.orElse(""),
                context.getExtendPathStack().getTopLineNumber(),
                context.getExtendPathStack().getTopStartPosition()
              )
            )
          );
          break;
        }
        extendPaths.add(extendPath.orElse(""));
{'fraction_non_alphanumeric': 0.10612480739599384, 'fraction_numerical': 0.001926040061633282, 'mean_word_length': 3.1126732673267328, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '418637', 'n_tokens_mistral': 5763, 'n_tokens_neox': 5622, 'n_words': 1427}