<commit_before>package scala.tools.nsc

import scala.reflect.runtime.universe._
import collection.JavaConverters._
import com.fasterxml.jackson.annotation._
import com.fasterxml.jackson.core.util.DefaultPrettyPrinter
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.dataformat.yaml.{YAMLFactory, YAMLGenerator}
import com.fasterxml.jackson.module.scala.DefaultScalaModule

object ScalaCompilerOptionsExporter {

  case class Category(name: String, load: Int) extends Ordered[Category] {
    def compare(that: Category): Int = (this.load) compare (that.load)
  }
  val StandardSettings = Category("Standard Settings", 0)
  val JVMSettings = Category("JVM Settings", 1)
  val PluginSettings = Category("Plugin Settings", 2)
  val AdvancedSettings = Category("Advanced Settings", 3)
  val PrivateSettings = Category("Private Settings", 4)
  val WarningSettings = Category("Warning Settings", 5)

  trait JacksonWorkaround {
     val category: String
  }
  @JsonIgnoreProperties(Array("_category"))
  @JsonPropertyOrder(Array("category", "description", "options"))
  case class Section(_category: Category, description: Option[String], options: List[ScalacOption]) extends JacksonWorkaround{
    val category: String = _category.name
  }
  case class ScalacOption(
    option: String,
    schema: Schema,
    description: String,
    abbreviations: Seq[String] = Seq.empty,
    deprecated: Option[String] = None,
    note: Option[String] = None
  )
  case class Schema(
    @JsonProperty("type") _type: String,
    arg: Option[String] = None,
    multiple: Option[Boolean] = None,
    default: Option[Any] = None,
    choices: Seq[Choice] = Seq.empty,
    min: Option[Any] = None,
    max: Option[Any] = None
  )
  case class Choice(choice: String, description: Option[String] = None, deprecated: Option[String] = None)

  def main(args: Array[String]): Unit = {
    val runtimeMirror = scala.reflect.runtime.currentMirror

    val settings = new scala.tools.nsc.Settings(s => ())
    val instanceMirror = runtimeMirror.reflect(settings)
    val sortedInOrderOfAppearance = runtimeMirror.classSymbol(settings.getClass).toType.members.sorted
    val accessors = sortedInOrderOfAppearance.collect {
      case m: MethodSymbol if m.isGetter && m.isPublic => m
    }

    def mergeChoice(labels: Seq[String], descriptions: Seq[String]): Seq[Choice] = {
      for {
        (choice, d) <- (labels zipAll (descriptions, "", ""))
      } yield {
        Choice(
          choice,
          description = Option(d).filter(_.nonEmpty),
          deprecated = Some("EXPLAIN_ALTERNATIVE").filter(_ => d.toLowerCase.contains("deprecated"))
        )
      }
    }

    val extractedSettings : List[ScalacOption] = accessors.map(acc => instanceMirror.reflectMethod(acc).apply()).collect {
      case s: settings.Setting =>
        val schema = s match {
          case b: settings.BooleanSetting => new Schema(_type="Boolean")
          case i: settings.IntSetting => new Schema(_type="Int", default = Some(i.default), min = i.range.map(_._1), max = i.range.map(_._2))
          case c: settings.ChoiceSetting =>
            val choices = mergeChoice(c.choices, c.choicesHelp)
            new Schema(_type="Choice", arg = Some(c.helpArg), default = Option(c.default), choices = choices)
          case mc: settings.MultiChoiceSetting[_] =>
            val choices = mergeChoice(mc.choices, mc.descriptions)
            new Schema(_type="Choice", multiple = Some(true), arg = Some(mc.helpArg), choices = choices)
          case ps: settings.PhasesSetting => new Schema(_type="Phases", default = Option(ps.default))
          case px: settings.PrefixSetting => new Schema(_type="Prefix")
          case sv: settings.ScalaVersionSetting => new Schema(_type="ScalaVerion", arg = Some(sv.arg), default = Some(sv.initial.unparse))
          case pathStr: settings.PathSetting => new Schema(_type="Path", arg = Some(pathStr.arg), default = Some(pathStr.default))
          case str: settings.StringSetting => new Schema(_type="String", arg = Some(str.arg), default = Some(str.default))
          case ms: settings.MultiStringSetting => new Schema(_type="String", multiple = Some(true), arg = Some(ms.arg))
        }

        new ScalacOption(
          option = s.name,
          schema = schema,
          description = s.helpDescription,
          abbreviations = s.abbreviations,
          deprecated = Some("EXPLAIN_ALTERNATIVE").filter(_ => s.helpDescription.toLowerCase.contains("deprecated"))
        )
    }

<commit_msg>Rewrite `_' to `_` so that it can be highlighted in markdown<commit_after>package scala.tools.nsc

import scala.reflect.runtime.universe._
import collection.JavaConverters._
import com.fasterxml.jackson.annotation._
import com.fasterxml.jackson.core.util.DefaultPrettyPrinter
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.dataformat.yaml.{YAMLFactory, YAMLGenerator}
import com.fasterxml.jackson.module.scala.DefaultScalaModule

object ScalaCompilerOptionsExporter {

  case class Category(name: String, load: Int) extends Ordered[Category] {
    def compare(that: Category): Int = (this.load) compare (that.load)
  }
  val StandardSettings = Category("Standard Settings", 0)
  val JVMSettings = Category("JVM Settings", 1)
  val PluginSettings = Category("Plugin Settings", 2)
  val AdvancedSettings = Category("Advanced Settings", 3)
  val PrivateSettings = Category("Private Settings", 4)
  val WarningSettings = Category("Warning Settings", 5)

  trait JacksonWorkaround {
     val category: String
  }
  @JsonIgnoreProperties(Array("_category"))
  @JsonPropertyOrder(Array("category", "description", "options"))
  case class Section(_category: Category, description: Option[String], options: List[ScalacOption]) extends JacksonWorkaround{
    val category: String = _category.name
  }
  case class ScalacOption(
    option: String,
    schema: Schema,
    description: String,
    abbreviations: Seq[String] = Seq.empty,
    deprecated: Option[String] = None,
    note: Option[String] = None
  )
  case class Schema(
    @JsonProperty("type") _type: String,
    arg: Option[String] = None,
    multiple: Option[Boolean] = None,
    default: Option[Any] = None,
    choices: Seq[Choice] = Seq.empty,
    min: Option[Any] = None,
    max: Option[Any] = None
  )
  case class Choice(choice: String, description: Option[String] = None, deprecated: Option[String] = None)

  private val quoted = """`([^`']+)'""".r

  def markdownifyBackquote(string: String) : String = {
    quoted.replaceAllIn(string, "`$1`")
  }

  def main(args: Array[String]): Unit = {
    val runtimeMirror = scala.reflect.runtime.currentMirror

    val settings = new scala.tools.nsc.Settings(s => ())
    val instanceMirror = runtimeMirror.reflect(settings)
    val sortedInOrderOfAppearance = runtimeMirror.classSymbol(settings.getClass).toType.members.sorted
    val accessors = sortedInOrderOfAppearance.collect {
      case m: MethodSymbol if m.isGetter && m.isPublic => m
    }

    def mergeChoice(labels: Seq[String], descriptions: Seq[String]): Seq[Choice] = {
      for {
        (choice, d) <- (labels zipAll (descriptions, "", ""))
      } yield {
        Choice(
          choice,
          description = Option(d).map(markdownifyBackquote).filter(_.nonEmpty),
          deprecated = Some("EXPLAIN_ALTERNATIVE").filter(_ => d.toLowerCase.contains("deprecated"))
        )
      }
    }

    val extractedSettings : List[ScalacOption] = accessors.map(acc => instanceMirror.reflectMethod(acc).apply()).collect {
      case s: settings.Setting =>
        val schema = s match {
          case b: settings.BooleanSetting => new Schema(_type="Boolean")
          case i: settings.IntSetting => new Schema(_type="Int", default = Some(i.default), min = i.range.map(_._1), max = i.range.map(_._2))
          case c: settings.ChoiceSetting =>
            val choices = mergeChoice(c.choices, c.choicesHelp)
            new Schema(_type="Choice", arg = Some(c.helpArg), default = Option(c.default), choices = choices)
          case mc: settings.MultiChoiceSetting[_] =>
            val choices = mergeChoice(mc.choices, mc.descriptions)
            new Schema(_type="Choice", multiple = Some(true), arg = Some(mc.helpArg), choices = choices)
          case ps: settings.PhasesSetting => new Schema(_type="Phases", default = Option(ps.default))
          case px: settings.PrefixSetting => new Schema(_type="Prefix")
          case sv: settings.ScalaVersionSetting => new Schema(_type="ScalaVerion", arg = Some(sv.arg), default = Some(sv.initial.unparse))
          case pathStr: settings.PathSetting => new Schema(_type="Path", arg = Some(pathStr.arg), default = Some(pathStr.default))
          case str: settings.StringSetting => new Schema(_type="String", arg = Some(str.arg), default = Some(str.default))
          case ms: settings.MultiStringSetting => new Schema(_type="String", multiple = Some(true), arg = Some(ms.arg))
        }

        new ScalacOption(
          option = s.name,
          schema = schema,
          description = markdownifyBackquote(s.helpDescription),
          abbreviations = s.abbreviations,
          deprecated = Some("EXPLAIN_ALTERNATIVE").filter(_ => s.helpDescription.toLowerCase.contains("deprecated"))
        )
    }


{'fraction_non_alphanumeric': 0.12490602513156482, 'fraction_numerical': 0.0018257974438835786, 'mean_word_length': 3.6700100300902707, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2275575', 'n_tokens_mistral': 2716, 'n_tokens_neox': 2631, 'n_words': 629}