<commit_before>package dotty.tools
package dotc
package printing

import parsing.Tokens._
import scala.annotation.switch
import scala.collection.mutable.StringBuilder
import util.Chars

/** This object provides functions for syntax highlighting in the REPL */
object SyntaxHighlighting {

  // Keep in sync with SyntaxHighlightingTests
  val NoColor         = Console.RESET
  val CommentColor    = Console.BLUE
  val KeywordColor    = Console.YELLOW
  val ValDefColor     = Console.CYAN
  val LiteralColor    = Console.RED
  val StringColor     = Console.GREEN
  val TypeColor       = Console.MAGENTA
  val AnnotationColor = Console.MAGENTA

  private def none(str: String) = str
  private def keyword(str: String) = KeywordColor + str + NoColor
  private def typeDef(str: String) = TypeColor + str + NoColor
  private def literal(str: String) = LiteralColor + str + NoColor
  private def valDef(str: String) = ValDefColor + str + NoColor
  private def operator(str: String) = TypeColor + str + NoColor
  private def annotation(str: String) =
    if (str.trim == "@") str else { AnnotationColor + str + NoColor }
  private val tripleQs = Console.RED_B + "???" + NoColor

  private val keywords: Seq[String] = for {
    index <- IF to ENUM // All alpha keywords
  } yield tokenString(index)

  private val interpolationPrefixes =
    'A' :: 'B' :: 'C' :: 'D' :: 'E' :: 'F' :: 'G' :: 'H' :: 'I' :: 'J' :: 'K' ::
    'L' :: 'M' :: 'N' :: 'O' :: 'P' :: 'Q' :: 'R' :: 'S' :: 'T' :: 'U' :: 'V' ::
    'W' :: 'X' :: 'Y' :: 'Z' :: '$' :: '_' :: 'a' :: 'b' :: 'c' :: 'd' :: 'e' ::
    'f' :: 'g' :: 'h' :: 'i' :: 'j' :: 'k' :: 'l' :: 'm' :: 'n' :: 'o' :: 'p' ::
    'q' :: 'r' :: 's' :: 't' :: 'u' :: 'v' :: 'w' :: 'x' :: 'y' :: 'z' :: Nil

  private val typeEnders =
   '{'  :: '}' :: ')' :: '(' :: '[' :: ']' :: '=' :: ' ' :: ',' :: '.' :: '|' ::
   '&' :: '\n' :: Nil

  def apply(chars: Iterable[Char]): Iterable[Char] = {
    var prev: Char = 0
    var remaining  = chars.toStream
    val newBuf     = new StringBuilder
    var lastValDefToken  = ""

    @inline def keywordStart =
      prev == 0    || prev == ' ' || prev == '{' || prev == '(' ||
      prev == '\n' || prev == '[' || prev == ',' || prev == ':' ||
      prev == '|'  || prev == '&'

    @inline def numberStart(c: Char) =
      c.isDigit && (!prev.isLetter || prev == '.' || prev == ' ' || prev == '(' || prev == '\u0000')

    def takeChar(): Char = takeChars(1).head
    def takeChars(x: Int): Seq[Char] = {
      val taken = remaining.take(x)
      remaining = remaining.drop(x)
      taken
    }

    while (remaining.nonEmpty) {
      val n = takeChar()
      if (interpolationPrefixes.contains(n)) {
        // Interpolation prefixes are a superset of the keyword start chars
        val next = remaining.take(3).mkString
        if (next.startsWith("\"")) {
          newBuf += n
          prev = n
          if (remaining.nonEmpty) takeChar() // drop 1 for appendLiteral
          appendString('"', next == "\"\"\"")
        } else {
          if (n.isUpper && (keywordStart || prev == '.')) {
            appendWhile(n, !typeEnders.contains(_), typeDef)
          } else if (keywordStart) {
            append(n, keywords.contains(_), { kw =>
              if (kw == "new") typeDef(kw) else keyword(kw)
            })
          } else {
            newBuf += n
            prev = n
          }
        }
      } else {
        (n: @switch) match {
          case '/'  =>
            if (remaining.nonEmpty) {
              remaining.head match {
                case '/' =>
                  takeChar()
                  eolComment()
                case '*' =>
                  takeChar()
                  blockComment()
                case x =>
                  newBuf += '/'
              }
            } else newBuf += '/'
          case '='  =>
            append('=', _ == "=>", operator)
          case '<'  =>
            append('<', { x => x == "<-" || x == "<:" || x == "<%" }, operator)
          case '>'  =>
            append('>', { x => x == ">:" }, operator)
          case '#'  =>
            if (prev != ' ' && prev != '.') newBuf append operator("#")
            else newBuf += n
            prev = '#'
          case '@'  =>
            appendWhile('@', !typeEnders.contains(_), annotation)
          case '\"' =>
            appendString('\"', multiline = remaining.take(2).mkString == "\"\"")
          case '\'' =>
            appendSingleQuote('\'')
          case '`'  =>
            appendTo('`', _ == '`', none)
          case _    => {
            if (n == '?' && remaining.take(2).mkString == "??") {
              takeChars(2)
              newBuf append tripleQs
              prev = '?'
            }
            else if (n.isUpper && keywordStart)
              appendWhile(n, !typeEnders.contains(_), typeDef)
            else if (numberStart(n)) {
              def isNumber(c: Char): Boolean =
                c.isDigit || c == '\u0000' || (c == '.' && remaining.nonEmpty && remaining.head.isDigit)
              appendWhile(n, isNumber , literal)
            } else
              newBuf += n; prev = n
          }
        }
      }
    }
<commit_msg>Style fix<commit_after>package dotty.tools
package dotc
package printing

import parsing.Tokens._
import scala.annotation.switch
import scala.collection.mutable.StringBuilder
import util.Chars

/** This object provides functions for syntax highlighting in the REPL */
object SyntaxHighlighting {

  // Keep in sync with SyntaxHighlightingTests
  val NoColor         = Console.RESET
  val CommentColor    = Console.BLUE
  val KeywordColor    = Console.YELLOW
  val ValDefColor     = Console.CYAN
  val LiteralColor    = Console.RED
  val StringColor     = Console.GREEN
  val TypeColor       = Console.MAGENTA
  val AnnotationColor = Console.MAGENTA

  private def none(str: String) = str
  private def keyword(str: String) = KeywordColor + str + NoColor
  private def typeDef(str: String) = TypeColor + str + NoColor
  private def literal(str: String) = LiteralColor + str + NoColor
  private def valDef(str: String) = ValDefColor + str + NoColor
  private def operator(str: String) = TypeColor + str + NoColor
  private def annotation(str: String) =
    if (str.trim == "@") str else { AnnotationColor + str + NoColor }
  private val tripleQs = Console.RED_B + "???" + NoColor

  private val keywords: Seq[String] = for {
    index <- IF to ENUM // All alpha keywords
  } yield tokenString(index)

  private val interpolationPrefixes =
    'A' :: 'B' :: 'C' :: 'D' :: 'E' :: 'F' :: 'G' :: 'H' :: 'I' :: 'J' :: 'K' ::
    'L' :: 'M' :: 'N' :: 'O' :: 'P' :: 'Q' :: 'R' :: 'S' :: 'T' :: 'U' :: 'V' ::
    'W' :: 'X' :: 'Y' :: 'Z' :: '$' :: '_' :: 'a' :: 'b' :: 'c' :: 'd' :: 'e' ::
    'f' :: 'g' :: 'h' :: 'i' :: 'j' :: 'k' :: 'l' :: 'm' :: 'n' :: 'o' :: 'p' ::
    'q' :: 'r' :: 's' :: 't' :: 'u' :: 'v' :: 'w' :: 'x' :: 'y' :: 'z' :: Nil

  private val typeEnders =
   '{'  :: '}' :: ')' :: '(' :: '[' :: ']' :: '=' :: ' ' :: ',' :: '.' :: '|' ::
   '&' :: '\n' :: Nil

  def apply(chars: Iterable[Char]): Iterable[Char] = {
    var prev: Char = 0
    var remaining  = chars.toStream
    val newBuf     = new StringBuilder
    var lastValDefToken  = ""

    @inline def keywordStart =
      prev == 0    || prev == ' ' || prev == '{' || prev == '(' ||
      prev == '\n' || prev == '[' || prev == ',' || prev == ':' ||
      prev == '|'  || prev == '&'

    @inline def numberStart(c: Char) =
      c.isDigit && (!prev.isLetter || prev == '.' || prev == ' ' || prev == '(' || prev == '\u0000')

    def takeChar(): Char = takeChars(1).head
    def takeChars(x: Int): Seq[Char] = {
      val taken = remaining.take(x)
      remaining = remaining.drop(x)
      taken
    }

    while (remaining.nonEmpty) {
      val n = takeChar()
      if (interpolationPrefixes.contains(n)) {
        // Interpolation prefixes are a superset of the keyword start chars
        val next = remaining.take(3).mkString
        if (next.startsWith("\"")) {
          newBuf += n
          prev = n
          if (remaining.nonEmpty) takeChar() // drop 1 for appendLiteral
          appendString('"', next == "\"\"\"")
        } else {
          if (n.isUpper && (keywordStart || prev == '.')) {
            appendWhile(n, !typeEnders.contains(_), typeDef)
          } else if (keywordStart) {
            append(n, keywords.contains(_), { kw =>
              if (kw == "new") typeDef(kw) else keyword(kw)
            })
          } else {
            newBuf += n
            prev = n
          }
        }
      } else {
        (n: @switch) match {
          case '/'  =>
            if (remaining.nonEmpty) {
              remaining.head match {
                case '/' =>
                  takeChar()
                  eolComment()
                case '*' =>
                  takeChar()
                  blockComment()
                case x =>
                  newBuf += '/'
              }
            } else newBuf += '/'
          case '='  =>
            append('=', _ == "=>", operator)
          case '<'  =>
            append('<', { x => x == "<-" || x == "<:" || x == "<%" }, operator)
          case '>'  =>
            append('>', { x => x == ">:" }, operator)
          case '#'  =>
            if (prev != ' ' && prev != '.') newBuf append operator("#")
            else newBuf += n
            prev = '#'
          case '@'  =>
            appendWhile('@', !typeEnders.contains(_), annotation)
          case '\"' =>
            appendString('\"', multiline = remaining.take(2).mkString == "\"\"")
          case '\'' =>
            appendSingleQuote('\'')
          case '`'  =>
            appendTo('`', _ == '`', none)
          case _    => {
            if (n == '?' && remaining.take(2).mkString == "??") {
              takeChars(2)
              newBuf append tripleQs
              prev = '?'
            }
            else if (n.isUpper && keywordStart)
              appendWhile(n, !typeEnders.contains(_), typeDef)
            else if (numberStart(n)) {
              def isNumber(c: Char): Boolean =
                c.isDigit || c == '\u0000' || (c == '.' && remaining.nonEmpty && remaining.head.isDigit)
              appendWhile(n, isNumber, literal)
            } else
              newBuf += n; prev = n
          }
        }
      }
    }

{'fraction_non_alphanumeric': 0.18754238109076818, 'fraction_numerical': 0.003099874067616003, 'mean_word_length': 1.7910246012435793, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7320154', 'n_tokens_mistral': 3306, 'n_tokens_neox': 3217, 'n_words': 799}