<commit_before>package scala.meta
package internal
package tokenizers

import java.lang.ref.WeakReference
import java.util.{concurrent => juc}
import scala.collection.{immutable, mutable}
import org.scalameta._
import org.scalameta.invariants._
import Chars.{CR, LF, FF}
import LegacyToken._
import scala.annotation.tailrec
import scala.meta.inputs._
import scala.meta.tokens._
import scala.meta.tokenizers._
import scala.meta.internal.tokenizers.PlatformTokenizerCache._

class ScalametaTokenizer(input: Input, dialect: Dialect) {
  def tokenize(): Tokens = {
    import ScalametaTokenizer._
    val miniCache = {
      var result = megaCache.get(dialect)
      if (result == null) {
        val unsyncResult = newUnsyncResult
        val syncResult = putIfAbsent(dialect, unsyncResult)
        result = if (syncResult != null) syncResult else unsyncResult
      }
      result
    }
    val entry = miniCacheSyncRoot.synchronized { miniCache.get(input) }
    entry match {
      case Some(tokens) =>
        tokens
      case None =>
        val tokens = uncachedTokenize()
        miniCacheSyncRoot.synchronized { miniCache.update(input, tokens) }
        tokens
    }
  }

  private def uncachedTokenize(): Tokens = {
    def legacyTokenToToken(curr: LegacyTokenData): Token = {
      (curr.token: @scala.annotation.switch) match {
        case IDENTIFIER       => Token.Ident(input, dialect, curr.offset, curr.endOffset + 1, curr.name)
        case BACKQUOTED_IDENT => Token.Ident(input, dialect, curr.offset, curr.endOffset + 1, curr.name)

        case INTLIT          => Token.Constant.Int(input, dialect, curr.offset, curr.endOffset + 1, curr.intVal)
        case LONGLIT         => Token.Constant.Long(input, dialect, curr.offset, curr.endOffset + 1, curr.longVal)
        case FLOATLIT        => Token.Constant.Float(input, dialect, curr.offset, curr.endOffset + 1, curr.floatVal)
        case DOUBLELIT       => Token.Constant.Double(input, dialect, curr.offset, curr.endOffset + 1, curr.doubleVal)
        case CHARLIT         => Token.Constant.Char(input, dialect, curr.offset, curr.endOffset + 1, curr.charVal)
        case SYMBOLLIT       => Token.Constant.Symbol(input, dialect, curr.offset, curr.endOffset + 1, scala.Symbol(curr.strVal))
        case STRINGLIT       => Token.Constant.String(input, dialect, curr.offset, curr.endOffset + 1, curr.strVal)
        case TRUE            => Token.KwTrue(input, dialect, curr.offset)
        case FALSE           => Token.KwFalse(input, dialect, curr.offset)
        case NULL            => Token.KwNull(input, dialect, curr.offset)

        case INTERPOLATIONID => Token.Interpolation.Id(input, dialect, curr.offset, curr.endOffset + 1, curr.name)
        case XMLLIT          => Token.Xml.Start(input, dialect, curr.offset, curr.offset)

        case NEW   => Token.KwNew(input, dialect, curr.offset)
        case THIS  => Token.KwThis(input, dialect, curr.offset)
        case SUPER => Token.KwSuper(input, dialect, curr.offset)

        case IMPLICIT  => Token.KwImplicit(input, dialect, curr.offset)
        case OVERRIDE  => Token.KwOverride(input, dialect, curr.offset)
        case PROTECTED => Token.KwProtected(input, dialect, curr.offset)
        case PRIVATE   => Token.KwPrivate(input, dialect, curr.offset)
        case ABSTRACT  => Token.KwAbstract(input, dialect, curr.offset)
        case FINAL     => Token.KwFinal(input, dialect, curr.offset)<commit_msg>Add missing unreachable tokens to token conversion<commit_after>package scala.meta
package internal
package tokenizers

import java.lang.ref.WeakReference
import java.util.{concurrent => juc}
import scala.collection.{immutable, mutable}
import org.scalameta._
import org.scalameta.invariants._
import Chars.{CR, LF, FF}
import LegacyToken._
import scala.annotation.tailrec
import scala.meta.inputs._
import scala.meta.tokens._
import scala.meta.tokenizers._
import scala.meta.internal.tokenizers.PlatformTokenizerCache._

class ScalametaTokenizer(input: Input, dialect: Dialect) {
  def tokenize(): Tokens = {
    import ScalametaTokenizer._
    val miniCache = {
      var result = megaCache.get(dialect)
      if (result == null) {
        val unsyncResult = newUnsyncResult
        val syncResult = putIfAbsent(dialect, unsyncResult)
        result = if (syncResult != null) syncResult else unsyncResult
      }
      result
    }
    val entry = miniCacheSyncRoot.synchronized { miniCache.get(input) }
    entry match {
      case Some(tokens) =>
        tokens
      case None =>
        val tokens = uncachedTokenize()
        miniCacheSyncRoot.synchronized { miniCache.update(input, tokens) }
        tokens
    }
  }

  private def uncachedTokenize(): Tokens = {
    def legacyTokenToToken(curr: LegacyTokenData): Token = {
      (curr.token: @scala.annotation.switch) match {
        case IDENTIFIER       => Token.Ident(input, dialect, curr.offset, curr.endOffset + 1, curr.name)
        case BACKQUOTED_IDENT => Token.Ident(input, dialect, curr.offset, curr.endOffset + 1, curr.name)

        case INTLIT          => Token.Constant.Int(input, dialect, curr.offset, curr.endOffset + 1, curr.intVal)
        case LONGLIT         => Token.Constant.Long(input, dialect, curr.offset, curr.endOffset + 1, curr.longVal)
        case FLOATLIT        => Token.Constant.Float(input, dialect, curr.offset, curr.endOffset + 1, curr.floatVal)
        case DOUBLELIT       => Token.Constant.Double(input, dialect, curr.offset, curr.endOffset + 1, curr.doubleVal)
        case CHARLIT         => Token.Constant.Char(input, dialect, curr.offset, curr.endOffset + 1, curr.charVal)
        case SYMBOLLIT       => Token.Constant.Symbol(input, dialect, curr.offset, curr.endOffset + 1, scala.Symbol(curr.strVal))
        case STRINGLIT       => Token.Constant.String(input, dialect, curr.offset, curr.endOffset + 1, curr.strVal)
        case STRINGPART      => unreachable
        case TRUE            => Token.KwTrue(input, dialect, curr.offset)
        case FALSE           => Token.KwFalse(input, dialect, curr.offset)
        case NULL            => Token.KwNull(input, dialect, curr.offset)

        case INTERPOLATIONID => Token.Interpolation.Id(input, dialect, curr.offset, curr.endOffset + 1, curr.name)
        case XMLLIT          => Token.Xml.Start(input, dialect, curr.offset, curr.offset)
        case XMLLITEND       => unreachable

        case NEW   => Token.KwNew(input, dialect, curr.offset)
        case THIS  => Token.KwThis(input, dialect, curr.offset)
        case SUPER => Token.KwSuper(input, dialect, curr.offset)

        case IMPLICIT  => Token.KwImplicit(input, dialect, curr.offset)
        case OVERRIDE  => Token.KwOverride(input, dialect, curr.offset)
        case PROTECTED => Token.KwProtected(input, dialect, curr.offset)
        case PRIVATE   => Token.KwPrivate(input, dialect, curr.offset)
        case ABSTRACT  => Token.KwAbstract(input, dialect, curr.offset)
        case FINAL     => Token.KwFinal(input, dialect, curr.offset)
{'fraction_non_alphanumeric': 0.10721944245889921, 'fraction_numerical': 0.0028591851322373124, 'mean_word_length': 2.9977142857142858, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5457009', 'n_tokens_mistral': 2146, 'n_tokens_neox': 2179, 'n_words': 495}