<commit_before>package org.scalafmt.internal

import scala.annotation.tailrec
import scala.meta.Tree
import scala.meta.tokens.Token
import scala.meta.tokens.Tokens

import org.scalafmt.config.ScalafmtConfig
import org.scalafmt.rewrite.FormatTokensRewrite
import org.scalafmt.util.StyleMap
import org.scalafmt.util.TokenOps
import org.scalafmt.util.TreeOps
import org.scalafmt.util.Whitespace

class FormatTokens(leftTok2tok: Map[TokenOps.TokenHash, Int])(
    val arr: Array[FormatToken]
) extends IndexedSeq[FormatToken] {

  private def this(arr: Array[FormatToken]) = this {
    val result = Map.newBuilder[TokenOps.TokenHash, Int]
    result.sizeHint(arr.length)
    arr.foreach(t => result += FormatTokens.thash(t.left) -> t.meta.idx)
    result += FormatTokens.thash(arr.last.right) -> arr.last.meta.idx
    result.result()
  }(arr)

  private lazy val matchingParentheses: Map[TokenOps.TokenHash, Token] =
    TreeOps.getMatchingParentheses(arr.view.map(_.right))

  override def length: Int = arr.length
  override def apply(idx: Int): FormatToken = arr(idx)

  private def get(tok: Token, isBefore: Boolean): FormatToken = {
    val idx = leftTok2tok.getOrElse(
      FormatTokens.thash(tok),
      throw new NoSuchElementException(
        s"Missing token index [${tok.start}:${tok.end}]: `$tok`"
      )
    )
    if (idx >= arr.length) arr.last
    else {
      val ft = arr(idx)
      if (isBefore) {
        if (ft.left.start <= tok.start) ft else at(idx - 1)
      } else {
        if (ft.left.start >= tok.start) ft else at(idx + 1)
      }
    }
  }

  def at(off: Int): FormatToken =
    if (off < 0) arr.head else if (off < arr.length) arr(off) else arr.last

  // get token; if rewritten, the one before
  @inline def before(tok: Token): FormatToken = get(tok, true)
  // get token; if rewritten, the one after
  @inline def after(tok: Token): FormatToken = get(tok, false)
  @inline def apply(tok: Token): FormatToken = before(tok)

  /** If the token is missing:
    *   - to go backward, start from the next token
    *   - to go forward, start from the previous token This ensures that the
    *     next token in the direction of search is counted.
    */
  def apply(tok: Token, off: Int): FormatToken =
    apply(if (off < 0) after(tok) else before(tok), off)
  def apply(ft: FormatToken, off: Int): FormatToken = at(ft.meta.idx + off)

  @inline def hasNext(ft: FormatToken): Boolean = ft.meta.idx < (arr.length - 1)
  @inline def hasPrev(ft: FormatToken): Boolean = ft.meta.idx > 0

  @inline def prev(ft: FormatToken): FormatToken = apply(ft, -1)
  @inline def next(ft: FormatToken): FormatToken = apply(ft, 1)

  @inline def matching(token: Token): Token =
    matchingParentheses.getOrElse(
      TokenOps.hash(token),
      throw new NoSuchElementException(
        s"Missing matching token index [${token.start}:${token.end}]: `$token`"
      )
    )
  @inline def matchingOpt(token: Token): Option[Token] =
    matchingParentheses.get(TokenOps.hash(token))
  @inline def hasMatching(token: Token): Boolean =
    matchingParentheses.contains(TokenOps.hash(token))
  @inline def areMatching(t1: Token)(t2: Token): Boolean =
    matchingOpt(t1).contains(t2)

  def getClosingIfEnclosedInMatching(tree: Tree): Option[FormatToken] =
    getHeadOpt(tree).flatMap { head =>
      matchingOpt(head.left).flatMap { last =>
        val ft = getLastNonTrivial(tree)
        if (ft.left eq last) Some(ft) else None
      }
    }

  def isEnclosedInMatching(tree: Tree): Boolean =
    getClosingIfEnclosedInMatching(tree).isDefined

  def isCloseMatchingHead(close: Token)(tree: => Tree): Boolean =
    matchingOpt(close).exists(_ eq getHead(tree).left)

  @tailrec
  final def findTokenWith[A](
      ft: FormatToken,
      iter: FormatToken => FormatToken
  )(f: FormatToken => Option[A]): Either[FormatToken, A] =
    f(ft) match {
      case Some(a) => Right(a)
      case _ =>
        val nextFt = iter(ft)
        if (nextFt eq ft) Left(ft)
        else findTokenWith(nextFt, iter)(f)
    }

  final def findToken(
      ft: FormatToken,
      iter: FormatToken => FormatToken
  )(f: FormatToken => Boolean): Either[FormatToken, FormatToken] =
    findTokenWith(ft, iter)(Some(_).filter(f))

  final def nextNonCommentSameLine(curr: FormatToken): FormatToken =
    findToken(curr, next)(ft => ft.hasBreak || !ft.right.is[Token.Comment])
      .fold(identity, identity)

  final def nextNonComment(curr: FormatToken): FormatToken =
    findToken(curr, next)(!_.right.is[Token.Comment]).fold(identity, identity)

  final def nextNonComment(curr: FormatToken.Meta): FormatToken =
    nextNonComment(arr(curr.idx))

  final def prevNonCommentSameLine(curr: FormatToken): FormatToken =
    findToken(curr, prev)(ft => ft.hasBreak || !ft.left.is[Token.Comment])
      .fold(identity, identity)

  final def prevNonComment(curr: FormatToken): FormatToken =
    findToken(curr, prev)(!_.left.is[Token.Comment]).fold(identity, identity)

  @inline
  final def prevNonCommentBefore(curr: FormatToken): FormatToken =
    prevNonComment(prev(curr))

  def getHead(tree: Tree): FormatToken =
    after(tree.tokens.head)
  def getHeadOpt(tree: Tree): Option[FormatToken] =
    tree.tokens.headOption.map(after)

  def getLast(tree: Tree): FormatToken =
    apply(TokenOps.findLastVisibleToken(tree.tokens))
  def getLastOpt(tree: Tree): Option[FormatToken] =
    TokenOps.findLastVisibleTokenOpt(tree.tokens).map(apply)

  def getLastNonTrivial(tree: Tree): FormatToken =
    apply(TokenOps.findLastNonTrivialToken(tree.tokens))
  def getLastNonTrivialOpt(tree: Tree): Option[FormatToken] =
    TokenOps.findLastNonTrivialTokenOpt(tree.tokens).map(apply)

  /* the following methods return the first format token such that
   * its `right` is after the parameter and is not a comment */
  @inline
  def tokenAfter(token: Token): FormatToken = nextNonComment(before(token))
  @inline
  def tokenAfter(tree: Tree): FormatToken = nextNonComment(getLast(tree))
  @inline
  def tokenAfter(trees: Seq[Tree]): FormatToken = tokenAfter(trees.last)

  def tokenAfterOpt(tree: Tree): Option[FormatToken] =
    getLastOpt(tree).map(nextNonComment)
  def tokenAfterOpt(trees: Seq[Tree]): Option[FormatToken] =
    trees.lastOption.flatMap(tokenAfterOpt)

  /* the following methods return the last format token such that
   * its `left` is before the parameter */
  @inline
  def justBefore(token: Token): FormatToken = apply(token, -1)
  @inline<commit_msg>FormatTokens: define getHead/getLast for Tokens<commit_after>package org.scalafmt.internal

import scala.annotation.tailrec
import scala.meta.Tree
import scala.meta.tokens.Token
import scala.meta.tokens.Tokens

import org.scalafmt.config.ScalafmtConfig
import org.scalafmt.rewrite.FormatTokensRewrite
import org.scalafmt.util.StyleMap
import org.scalafmt.util.TokenOps
import org.scalafmt.util.TreeOps
import org.scalafmt.util.Whitespace

class FormatTokens(leftTok2tok: Map[TokenOps.TokenHash, Int])(
    val arr: Array[FormatToken]
) extends IndexedSeq[FormatToken] {

  private def this(arr: Array[FormatToken]) = this {
    val result = Map.newBuilder[TokenOps.TokenHash, Int]
    result.sizeHint(arr.length)
    arr.foreach(t => result += FormatTokens.thash(t.left) -> t.meta.idx)
    result += FormatTokens.thash(arr.last.right) -> arr.last.meta.idx
    result.result()
  }(arr)

  private lazy val matchingParentheses: Map[TokenOps.TokenHash, Token] =
    TreeOps.getMatchingParentheses(arr.view.map(_.right))

  override def length: Int = arr.length
  override def apply(idx: Int): FormatToken = arr(idx)

  private def get(tok: Token, isBefore: Boolean): FormatToken = {
    val idx = leftTok2tok.getOrElse(
      FormatTokens.thash(tok),
      throw new NoSuchElementException(
        s"Missing token index [${tok.start}:${tok.end}]: `$tok`"
      )
    )
    if (idx >= arr.length) arr.last
    else {
      val ft = arr(idx)
      if (isBefore) {
        if (ft.left.start <= tok.start) ft else at(idx - 1)
      } else {
        if (ft.left.start >= tok.start) ft else at(idx + 1)
      }
    }
  }

  def at(off: Int): FormatToken =
    if (off < 0) arr.head else if (off < arr.length) arr(off) else arr.last

  // get token; if rewritten, the one before
  @inline def before(tok: Token): FormatToken = get(tok, true)
  // get token; if rewritten, the one after
  @inline def after(tok: Token): FormatToken = get(tok, false)
  @inline def apply(tok: Token): FormatToken = before(tok)

  /** If the token is missing:
    *   - to go backward, start from the next token
    *   - to go forward, start from the previous token This ensures that the
    *     next token in the direction of search is counted.
    */
  def apply(tok: Token, off: Int): FormatToken =
    apply(if (off < 0) after(tok) else before(tok), off)
  def apply(ft: FormatToken, off: Int): FormatToken = at(ft.meta.idx + off)

  @inline def hasNext(ft: FormatToken): Boolean = ft.meta.idx < (arr.length - 1)
  @inline def hasPrev(ft: FormatToken): Boolean = ft.meta.idx > 0

  @inline def prev(ft: FormatToken): FormatToken = apply(ft, -1)
  @inline def next(ft: FormatToken): FormatToken = apply(ft, 1)

  @inline def matching(token: Token): Token =
    matchingParentheses.getOrElse(
      TokenOps.hash(token),
      throw new NoSuchElementException(
        s"Missing matching token index [${token.start}:${token.end}]: `$token`"
      )
    )
  @inline def matchingOpt(token: Token): Option[Token] =
    matchingParentheses.get(TokenOps.hash(token))
  @inline def hasMatching(token: Token): Boolean =
    matchingParentheses.contains(TokenOps.hash(token))
  @inline def areMatching(t1: Token)(t2: Token): Boolean =
    matchingOpt(t1).contains(t2)

  def getClosingIfEnclosedInMatching(tokens: Tokens): Option[FormatToken] =
    getHeadOpt(tokens).flatMap { head =>
      matchingOpt(head.left).flatMap { last =>
        val ft = getLastNonTrivial(tokens)
        if (ft.left eq last) Some(ft) else None
      }
    }

  def isEnclosedInMatching(tokens: Tokens): Boolean =
    getClosingIfEnclosedInMatching(tokens).isDefined
  def isEnclosedInMatching(tree: Tree): Boolean =
    isEnclosedInMatching(tree.tokens)

  def isCloseMatchingHead(close: Token)(tree: => Tree): Boolean =
    matchingOpt(close).exists(_ eq getHead(tree).left)

  @tailrec
  final def findTokenWith[A](
      ft: FormatToken,
      iter: FormatToken => FormatToken
  )(f: FormatToken => Option[A]): Either[FormatToken, A] =
    f(ft) match {
      case Some(a) => Right(a)
      case _ =>
        val nextFt = iter(ft)
        if (nextFt eq ft) Left(ft)
        else findTokenWith(nextFt, iter)(f)
    }

  final def findToken(
      ft: FormatToken,
      iter: FormatToken => FormatToken
  )(f: FormatToken => Boolean): Either[FormatToken, FormatToken] =
    findTokenWith(ft, iter)(Some(_).filter(f))

  final def nextNonCommentSameLine(curr: FormatToken): FormatToken =
    findToken(curr, next)(ft => ft.hasBreak || !ft.right.is[Token.Comment])
      .fold(identity, identity)

  final def nextNonComment(curr: FormatToken): FormatToken =
    findToken(curr, next)(!_.right.is[Token.Comment]).fold(identity, identity)

  final def nextNonComment(curr: FormatToken.Meta): FormatToken =
    nextNonComment(arr(curr.idx))

  final def prevNonCommentSameLine(curr: FormatToken): FormatToken =
    findToken(curr, prev)(ft => ft.hasBreak || !ft.left.is[Token.Comment])
      .fold(identity, identity)

  final def prevNonComment(curr: FormatToken): FormatToken =
    findToken(curr, prev)(!_.left.is[Token.Comment]).fold(identity, identity)

  @inline
  final def prevNonCommentBefore(curr: FormatToken): FormatToken =
    prevNonComment(prev(curr))

  def getHead(tokens: Tokens): FormatToken =
    after(tokens.head)
  def getHead(tree: Tree): FormatToken =
    getHead(tree.tokens)
  def getHeadOpt(tokens: Tokens): Option[FormatToken] =
    tokens.headOption.map(after)
  def getHeadOpt(tree: Tree): Option[FormatToken] =
    getHeadOpt(tree.tokens)

  def getLast(tokens: Tokens): FormatToken =
    apply(TokenOps.findLastVisibleToken(tokens))
  def getLast(tree: Tree): FormatToken =
    getLast(tree.tokens)
  def getLastOpt(tokens: Tokens): Option[FormatToken] =
    TokenOps.findLastVisibleTokenOpt(tokens).map(apply)
  def getLastOpt(tree: Tree): Option[FormatToken] =
    getLastOpt(tree.tokens)

  def getLastNonTrivial(tokens: Tokens): FormatToken =
    apply(TokenOps.findLastNonTrivialToken(tokens))
  def getLastNonTrivial(tree: Tree): FormatToken =
    getLastNonTrivial(tree.tokens)
  def getLastNonTrivialOpt(tokens: Tokens): Option[FormatToken] =
    TokenOps.findLastNonTrivialTokenOpt(tokens).map(apply)
  def getLastNonTrivialOpt(tree: Tree): Option[FormatToken] =
    getLastNonTrivialOpt(tree.tokens)

  /* the following methods return the first format token such that
   * its `right` is after the parameter and is not a comment */
  @inline
  def tokenAfter(token: Token): FormatToken = nextNonComment(before(token))
  @inline
  def tokenAfter(tree: Tree): FormatToken = nextNonComment(getLast(tree))
  @inline
  def tokenAfter(trees: Seq[Tree]): FormatToken = tokenAfter(trees.last)

  def tokenAfterOpt(tree: Tree): Option[FormatToken] =
    getLastOpt(tree).map(nextNonComment)
  def tokenAfterOpt(trees: Seq[Tree]): Option[FormatToken] =
    trees.lastOption.flatMap(tokenAfterOpt)

  /* the following methods return the last format token such that
   * its `left` is before the parameter */
  @inline
  def justBefore(token: Token): FormatToken = apply(token, -1)
  @inline
{'fraction_non_alphanumeric': 0.1215257244234181, 'fraction_numerical': 0.0022176227084565346, 'mean_word_length': 4.020037105751391, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6037529', 'n_tokens_mistral': 4368, 'n_tokens_neox': 4272, 'n_words': 1060}