<commit_before>package sjsonnet

import java.io.StringWriter
import java.text.DecimalFormat

import ujson.Js

import scala.collection.mutable.ArrayBuffer

object Format{
  case class FormatSpec(label: Option[String],
                        alternate: Boolean,
                        zeroPadded: Boolean,
                        leftAdjusted: Boolean,
                        blankBeforePositive: Boolean,
                        signCharacter: Boolean,
                        width: Option[Int],
                        precision: Option[Int],
                        conversion: Char)
  import fastparse.all._
  val integer           = P( CharIn('1' to '9') ~ CharsWhileIn('0' to '9', min = 0) )

  val label = P( ("(" ~ CharsWhile(_ != ')').! ~ ")").? )
  val flags = P( CharsWhileIn("#0- +", min = 0).! ).log()
  val width = P( (integer | "*").!.? )
  val precision = P( ("." ~ integer.!).? )
  val conversion = P( CharIn("diouxXeEfFgGcrsa%").! )
  val formatSpec = P( label ~ flags ~ width ~ precision ~ CharIn("hlL").? ~ conversion ).map{
    case (label, flags, width, precision, conversion) =>
      pprint.log(flags)
      FormatSpec(
        label,
        flags.contains('#'),
        flags.contains('0'),
        flags.contains('-'),
        flags.contains(' '),
        flags.contains('+'),
        width.map(_.toInt),
        precision.map(_.toInt),
        conversion.charAt(0)
      )
  }


  val plain = P( CharsWhile(_ != '%', min = 0).! )
  val format = P( plain ~ (("%" ~ formatSpec) ~ plain).rep ~ End)

  def zeroPad(s: Double, r: String, precision: Option[Int]) = {
    precision match{
      case None => r -> (s > 0)
      case Some(p) =>
        val shortage = p - r.length
        pprint.log(shortage)
        (if (shortage > 0) "0" * shortage + r else r) -> (s > 0)
    }
  }

  def toOctalString(s: Int, alternate: Boolean) = {
    val r = math.abs(s).toOctalString
    (if (s < 0) "-" else "") + (if (!alternate || r(0) == '0') r else "0" + r)
  }
  def toHexString(s: Int, alternate: Boolean) = {
    val r = math.abs(s).toHexString
    (if (s < 0) "-" else "") + (if (!alternate || r(0) == '0') r else "0x" + r)
  }
  def format(s: String, values0: ujson.Js): String = synchronized{
    val values = values0 match{
      case x: ujson.Js.Arr => x.arr
      case x => ArrayBuffer(x)
    }
    val (leading, chunks) = format.parse(s).get.value
    val output = new StringBuilder
    output.append(leading)
    var i = 0
    for((formatted, literal) <- chunks){
      pprint.log(formatted)
      val (cooked0, numeric, signedConversion) = formatted.conversion match{
        case '%' => ("%", false, false)
        case _ =>
          val value = values(i)
          i += 1
          val (cooked, signedConversion) = value match{
            case Js.Str(s) => s -> false
            case Js.Num(s) =>
              formatted.conversion match{
                case 'd' | 'i' | 'u' => zeroPad(s, s.toInt.toString, formatted.precision)
                case 'o' => zeroPad(s, toOctalString(s.toInt, formatted.alternate), formatted.precision)
                case 'x' => zeroPad(s, toHexString(s.toInt, formatted.alternate), formatted.precision)
                case 'X' => zeroPad(s, toHexString(s.toInt, formatted.alternate), formatted.precision)

                case 'e' => new DecimalFormat("0.#####E0").format(s).toLowerCase() -> false
                case 'E' => new DecimalFormat("0.#####E0").format(s) -> false

                case 'f' | 'F' => s.toString -> false

//                case 'g' =>
//                case 'G' =>

                case 'c' => s.toChar.toString -> false
              }
            case Js.True => "true" -> false
            case Js.False => "false" -> false
            case v => v.toString -> false
          }
          (cooked, value.isInstanceOf[Js.Num], signedConversion)
      }

      val cooked =
        if(signedConversion && formatted.blankBeforePositive) " " + cooked0
        else if(signedConversion && formatted.signCharacter) "+" + cooked0
        else cooked0
      val missingWidth = formatted.width.getOrElse(-1) - cooked.length
      val widened =
        if (missingWidth <= 0) cooked
        else if (formatted.zeroPadded) {
          if (numeric) "0" * missingWidth + cooked
          else {
            if (formatted.leftAdjusted) cooked + " " * missingWidth
            else " " * missingWidth + cooked
          }
        }
        else if (formatted.leftAdjusted) cooked + " " * missingWidth
        else " " * missingWidth + cooked
      output.append(widened)
      output.append(literal)


    }

    output.toString()
  }
}<commit_msg>More formatting tests pass<commit_after>package sjsonnet

import java.io.StringWriter
import java.text.DecimalFormat

import ujson.Js

import scala.collection.mutable.ArrayBuffer

object Format{
  case class FormatSpec(label: Option[String],
                        alternate: Boolean,
                        zeroPadded: Boolean,
                        leftAdjusted: Boolean,
                        blankBeforePositive: Boolean,
                        signCharacter: Boolean,
                        width: Option[Int],
                        precision: Option[Int],
                        conversion: Char)
  import fastparse.all._
  val integer           = P( CharIn('1' to '9') ~ CharsWhileIn('0' to '9', min = 0) )

  val label = P( ("(" ~ CharsWhile(_ != ')').! ~ ")").? )
  val flags = P( CharsWhileIn("#0- +", min = 0).! ).log()
  val width = P( (integer | "*").!.? )
  val precision = P( ("." ~ integer.!).? )
  val conversion = P( CharIn("diouxXeEfFgGcrsa%").! )
  val formatSpec = P( label ~ flags ~ width ~ precision ~ CharIn("hlL").? ~ conversion ).map{
    case (label, flags, width, precision, conversion) =>
      pprint.log(flags)
      FormatSpec(
        label,
        flags.contains('#'),
        flags.contains('0'),
        flags.contains('-'),
        flags.contains(' '),
        flags.contains('+'),
        width.map(_.toInt),
        precision.map(_.toInt),
        conversion.charAt(0)
      )
  }


  val plain = P( CharsWhile(_ != '%', min = 0).! )
  val format = P( plain ~ (("%" ~ formatSpec) ~ plain).rep ~ End)

  def zeroPad(lhs: String, rhs: String, precision: Option[Int]) = {
    precision match{
      case None => rhs
      case Some(p) =>
        val shortage = p - rhs.length
        (if (shortage > 0) ("0" * shortage + rhs) else rhs)
    }
  }

  def toDecimalString(s: Int, alternate: Boolean) = {
    val r = math.abs(s).toString
    (if (s < 0) "-" else "", r)
  }
  def toOctalString(s: Int, alternate: Boolean) = {
    val r = math.abs(s).toOctalString
    (if (s < 0) "-" else "", r)
  }
  def toHexString(s: Int, alternate: Boolean) = {
    val r = math.abs(s).toHexString
    (if (s < 0) "-" else "", r)
  }
  def widen(formatted: FormatSpec,
            lhs: String,
            mhs: String,
            rhs: String,
            numeric: Boolean,
            signedConversion: Boolean) = {

    val lhs2 =
      if(signedConversion && formatted.blankBeforePositive) " " + lhs
      else if(signedConversion && formatted.signCharacter) "+" + lhs
      else lhs

    val missingWidth = formatted.width.getOrElse(-1) - lhs2.length - mhs.length - rhs.length

    if (missingWidth <= 0) lhs2 + mhs + rhs
    else if (formatted.zeroPadded) {
      if (numeric) lhs2 + mhs + "0" * missingWidth + rhs
      else {
        if (formatted.leftAdjusted) lhs2 + mhs + rhs + " " * missingWidth
        else " " * missingWidth + lhs2 + mhs + rhs
      }
    }
    else if (formatted.leftAdjusted) lhs2 + mhs + rhs + " " * missingWidth
    else " " * missingWidth + lhs2 + mhs + rhs
  }
  def format(s: String, values0: ujson.Js): String = synchronized{
    val values = values0 match{
      case x: ujson.Js.Arr => x.arr
      case x => ArrayBuffer(x)
    }
    val (leading, chunks) = format.parse(s).get.value
    val output = new StringBuilder
    output.append(leading)
    var i = 0
    for((formatted, literal) <- chunks){
      pprint.log(formatted)
      val cooked0 = formatted.conversion match{
        case '%' => widen(formatted, "", "", "%", false, false)
        case _ =>
          val value = values(i)
          i += 1
          value match{
            case Js.Str(s) => widen(formatted, "", "", s, false, false)
            case Js.Num(s) =>
              formatted.conversion match{
                case 'd' | 'i' | 'u' =>
                  val (lhs, rhs) = toDecimalString(s.toInt, formatted.alternate)
                  val rhs2 = zeroPad(lhs, rhs, formatted.precision)
                  widen(
                    formatted,
                    lhs, "", rhs2,
                    true, s > 0
                  )
                case 'o' =>
                  val (lhs, rhs) = toOctalString(s.toInt, formatted.alternate)
                  val rhs2 = zeroPad(lhs, rhs, formatted.precision)
                  widen(
                    formatted,
                    lhs, (if (!formatted.alternate || rhs2(0) == '0') "" else "0"), rhs2,
                    true, s > 0
                  )
                case 'x' =>
                  val (lhs, rhs) = toHexString(s.toInt, formatted.alternate)
                  val rhs2 = zeroPad(lhs, rhs, formatted.precision)
                  widen(
                    formatted,
                    lhs, (if (!formatted.alternate) "" else "0x"), rhs2,
                    true, s > 0
                  )
                case 'X' =>
                  val (lhs, rhs) = toHexString(s.toInt, formatted.alternate)
                  val rhs2 = zeroPad(lhs, rhs, formatted.precision)
                  widen(
                    formatted,
                    lhs, (if (!formatted.alternate) "" else "0x"), rhs2,
                    true, s > 0
                  ).toUpperCase()

                case 'e' => new DecimalFormat("0.#####E0").format(s).toLowerCase() -> false
                case 'E' => new DecimalFormat("0.#####E0").format(s) -> false

                case 'f' | 'F' => s.toString -> false

                case 'c' => widen(formatted, "", "", s.toChar.toString , false, false)
              }
            case Js.True => widen(formatted, "", "", "true", false, false)
            case Js.False => widen(formatted, "", "", "false", false, false)
            case v => widen(formatted, "", "", v.toString, false, false)
          }

      }

      output.append(cooked0)
      output.append(literal)


    }

    output.toString()
  }
}
{'fraction_non_alphanumeric': 0.14546148706690848, 'fraction_numerical': 0.008685692469218288, 'mean_word_length': 1.7815237589593842, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 66, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '228561', 'n_tokens_mistral': 3323, 'n_tokens_neox': 3163, 'n_words': 793}