<commit_before>// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.intellij.lexer;

import com.intellij.openapi.util.Comparing;
import com.intellij.psi.tree.IElementType;
import com.intellij.psi.tree.TokenSet;
import com.intellij.util.ArrayUtil;
import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

/**
 * This class represents the result of lexing: text and the tokens produced from it by some lexer.
 * It allows clients to inspect all tokens at once and easily move back and forward to implement some simple lexer-based checks.
 */
@ApiStatus.Experimental
public interface TokenList {

  /**
   * @return the number of tokens inside
   */
  int getTokenCount();

  /**
   * @return the full text that was split into the tokens represented here
   */
  @NotNull
  CharSequence getTokenizedText();

  /**
   * @return the start offset of the token with the given index
   */
  int getTokenStart(int index);

  /**
   * @return the end offset of the token with the given index
   */
  int getTokenEnd(int index);

  /**
   * @return the type of the token with the given index, or null if the index is negative or exceeds token count
   */
  IElementType getTokenType(int index);

  /**
   * @return the text of the token with the given index, or null if the index is negative or exceeds token count
   */
  default CharSequence getTokenText(int index) {
    if (index < 0 || index >= getTokenCount()) return null;
    return getTokenizedText().subSequence(getTokenStart(index), getTokenEnd(index));
  }

  /**
   * @return whether {@link #getTokenType}(index) would return the given type
   */
  default boolean hasType(int index, @NotNull IElementType type) {
    return getTokenType(index) == type;
  }

  /**
   * @return whether {@link #getTokenType}(index) would return any of the given types (null acceptable, indicating start or end of the text)<commit_msg>freemarker/velocity: support dummy insertions into JSON template data text (IDEA-224925)<commit_after>// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.intellij.lexer;

import com.intellij.openapi.util.Comparing;
import com.intellij.openapi.util.TextRange;
import com.intellij.psi.tree.IElementType;
import com.intellij.psi.tree.TokenSet;
import com.intellij.util.ArrayUtil;
import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

/**
 * This class represents the result of lexing: text and the tokens produced from it by some lexer.
 * It allows clients to inspect all tokens at once and easily move back and forward to implement some simple lexer-based checks.
 */
@ApiStatus.Experimental
public interface TokenList {

  /**
   * @return the number of tokens inside
   */
  int getTokenCount();

  /**
   * @return the full text that was split into the tokens represented here
   */
  @NotNull
  CharSequence getTokenizedText();

  /**
   * @return the start offset of the token with the given index
   */
  int getTokenStart(int index);

  /**
   * @return the end offset of the token with the given index
   */
  int getTokenEnd(int index);

  /**
   * @return the range of the token with the given index
   */
  @NotNull
  default TextRange getTokenRange(int index) {
    return new TextRange(getTokenStart(index), getTokenEnd(index));
  }

  /**
   * @return the type of the token with the given index, or null if the index is negative or exceeds token count
   */
  IElementType getTokenType(int index);

  /**
   * @return the text of the token with the given index, or null if the index is negative or exceeds token count
   */
  default CharSequence getTokenText(int index) {
    if (index < 0 || index >= getTokenCount()) return null;
    return getTokenizedText().subSequence(getTokenStart(index), getTokenEnd(index));
  }

  /**
   * @return whether {@link #getTokenType}(index) would return the given type
   */
  default boolean hasType(int index, @NotNull IElementType type) {
    return getTokenType(index) == type;
  }

  /**
   * @return whether {@link #getTokenType}(index) would return any of the given types (null acceptable, indicating start or end of the text)
{'fraction_non_alphanumeric': 0.0894844424256189, 'fraction_numerical': 0.006359300476947536, 'mean_word_length': 3.781758957654723, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4040003', 'n_tokens_mistral': 1287, 'n_tokens_neox': 1225, 'n_words': 493}