<commit_before>package com.ephox.argonaut

import util.parsing.combinator._
import Json._

class JsonParser extends Parsers {
  type Elem = Char

  def jobject: Parser[Json] = '{' ~> repsep(pair, ',') <~ '}' ^^ jsonObject

  def jarray: Parser[Json] = '[' ~> repsep(jvalue, ',') <~ ']' ^^ jsonArray

  def jvalue = jstring | jboolean | jnull | jarray | jnumber | jobject

  def jstring = string ^^ jsonString

  def jnumber = number ^^ jsonNumber

  def jnull = acceptSeq("null") ^^^ jsonNull

  def jboolean = (f | t) ^^ jsonBool

  //---------------------------------------------------------------------------

  def pair: Parser[(String, Json)] = (string <~ ':') ~ jvalue ^^ { case k ~ v => (k, v)}

  def f = acceptSeq("false") ^^^ false

  def t = acceptSeq("true") ^^^ true

  def string = '"' ~> chars <~ '"'

  def chars = (char*) ^^ {_.mkString}

  // has top be a Parser[List[Char]] as unicode char may be more than one java char...? not really sure how this hangs together
  def char =
          ('\\' ~ '\"' ^^^ "\""
    	    |'\\' ~ '\\' ^^^ "\\"
    	    |'\\' ~ '/'  ^^^ "/"
    	    |'\\' ~ 'b'  ^^^ "\b"
    	    |'\\' ~ 'f'  ^^^ "\f"
    	    |'\\' ~ 'n'  ^^^ "\n"
    	    |'\\' ~ 'r'  ^^^ "\r"
    	    |'\\' ~ 't'  ^^^ "\t"
    	    |'\\' ~> 'u' ~> unicode)
<commit_msg>Story: Json Parser [id=17924] Improved unicode and char support.<commit_after>package com.ephox.argonaut

import util.parsing.combinator._
import Json._

class JsonParser extends Parsers {
  type Elem = Char

  def jobject: Parser[Json] = '{' ~> repsep(pair, ',') <~ '}' ^^ jsonObject

  def jarray: Parser[Json] = '[' ~> repsep(jvalue, ',') <~ ']' ^^ jsonArray

  def jvalue = jstring | jboolean | jnull | jarray | jnumber | jobject

  def jstring = string ^^ jsonString

  def jnumber = number ^^ jsonNumber

  def jnull = acceptSeq("null") ^^^ jsonNull

  def jboolean = (f | t) ^^ jsonBool

  //---------------------------------------------------------------------------

  def pair: Parser[(String, Json)] = (string <~ ':') ~ jvalue ^^ { case k ~ v => (k, v)}

  def f = acceptSeq("false") ^^^ false

  def t = acceptSeq("true") ^^^ true

  def string = '"' ~> chars <~ '"'

  def chars = (char*) ^^ {_.mkString}

  // has top be a Parser[List[Char]] as unicode char may be more than one java char...? not really sure how/if this hangs together
  def char =
          (escape ~ '\"' ^^^ "\""
    	    |escape ~ '\\' ^^^ "\\"
    	    |escape ~ '/'  ^^^ "/"
    	    |escape ~ 'b'  ^^^ "\b"
    	    |escape ~ 'f'  ^^^ "\f"
    	    |escape ~ 'n'  ^^^ "\n"
    	    |escape ~ 'r'  ^^^ "\r"
    	    |escape ~ 't'  ^^^ "\t"
    	    |escape ~> 'u' ~> unicode)

  def escape = '\\'

{'fraction_non_alphanumeric': 0.2605607476635514, 'fraction_numerical': 0.001869158878504673, 'mean_word_length': 1.9536423841059603, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5622804', 'n_tokens_mistral': 1004, 'n_tokens_neox': 951, 'n_words': 239}