<commit_before>#
#
#            Nim's Runtime Library
#        (c) Copyright 2012 <NAME>
#
#    See the file "copying.txt", included in this
#    distribution, for details about the copyright.
#

# Nim support for C/C++'s `wide strings`:idx:. This is part of the system
# module! Do not import it directly!

#when not declared(ThisIsSystem):
#  {.error: "You must not import this module explicitly".}

type
  Utf16Char* = distinct int16

when defined(nimv2):

  type
    WideCString* = ptr UncheckedArray[Utf16Char]

    WideCStringObj* = object
      bytes: int
      data: WideCString

  proc `=destroy`(a: var WideCStringObj) =
    if a.data != nil:
      deallocShared(a.data)
      a.data = nil

  proc `=`(a: var WideCStringObj; b: WideCStringObj) {.error.}

  proc `=sink`(a: var WideCStringObj; b: WideCStringObj) =
    a.bytes = b.bytes
    a.data = b.data

  proc createWide(a: var WideCStringObj; bytes: int) =
    a.bytes = bytes
    a.data = cast[typeof(a.data)](allocShared0(bytes))

  template `[]`*(a: WideCStringObj; idx: int): Utf16Char = a.data[idx]
  template `[]=`*(a: WideCStringObj; idx: int; val: Utf16Char) = a.data[idx] = val

  template nullWide(): untyped = WideCStringObj(bytes: 0, data: nil)

  converter toWideCString*(x: WideCStringObj): WideCString {.inline.} =
    result = x.data

else:
  template nullWide(): untyped = nil

  type
    WideCString* = ref UncheckedArray[Utf16Char]
    WideCStringObj* = WideCString

  template createWide(a; L) =
    unsafeNew(a, L * 4 + 2)

proc ord(arg: Utf16Char): int = int(cast[uint16](arg))

proc len*(w: WideCString): int =
  ## returns the length of a widestring. This traverses the whole string to
  ## find the binary zero end marker!
  result = 0
  while int16(w[result]) != 0'i16: inc result

const
  UNI_REPLACEMENT_CHAR = Utf16Char(0xFFFD'i16)
  UNI_MAX_BMP = 0x0000FFFF
  UNI_MAX_UTF16 = 0x0010FFFF
  UNI_MAX_UTF32 = 0x7FFFFFFF
  UNI_MAX_LEGAL_UTF32 = 0x0010FFFF

  halfShift = 10
  halfBase = 0x0010000
  halfMask = 0x3FF

  UNI_SUR_HIGH_START = 0xD800
  UNI_SUR_HIGH_END = 0xDBFF
  UNI_SUR_LOW_START = 0xDC00
  UNI_SUR_LOW_END = 0xDFFF
  UNI_REPL = 0xFFFD

template ones(n: untyped): untyped = ((1 shl n)-1)

template fastRuneAt(s: cstring, i, L: int, result: untyped, doInc = true) =
  ## Returns the unicode character ``s[i]`` in `result`. If ``doInc == true``
  ## `i` is incremented by the number of bytes that have been processed.
  bind ones

  if ord(s[i]) <= 127:
    result = ord(s[i])
    when doInc: inc(i)
  elif ord(s[i]) shr 5 == 0b110:
    #assert(ord(s[i+1]) shr 6 == 0b10)
    if i <= L - 2:
      result = (ord(s[i]) and (ones(5))) shl 6 or (ord(s[i+1]) and ones(6))
      when doInc: inc(i, 2)
    else:
      result = UNI_REPL
      when doInc: inc(i)
  elif ord(s[i]) shr 4 == 0b1110:
    if i <= L - 3:
      #assert(ord(s[i+1]) shr 6 == 0b10)
      #assert(ord(s[i+2]) shr 6 == 0b10)
      result = (ord(s[i]) and ones(4)) shl 12 or
               (ord(s[i+1]) and ones(6)) shl 6 or
               (ord(s[i+2]) and ones(6))
      when doInc: inc(i, 3)
    else:
      result = UNI_REPL
      when doInc: inc(i)
  elif ord(s[i]) shr 3 == 0b11110:
    if i <= L - 4:
      #assert(ord(s[i+1]) shr 6 == 0b10)
      #assert(ord(s[i+2]) shr 6 == 0b10)
      #assert(ord(s[i+3]) shr 6 == 0b10)
      result = (ord(s[i]) and ones(3)) shl 18 or
               (ord(s[i+1]) and ones(6)) shl 12 or
               (ord(s[i+2]) and ones(6)) shl 6 or
               (ord(s[i+3]) and ones(6))
      when doInc: inc(i, 4)
    else:
      result = UNI_REPL
      when doInc: inc(i)
  else:
    result = 0xFFFD
    when doInc: inc(i)

iterator runes(s: cstring, L: int): int =
  var
    i = 0
    result: int
  while i < L:
    fastRuneAt(s, i, L, result, true)
    yield result

proc newWideCString*(source: cstring, L: int): WideCStringObj =
  createWide(result, L * 4 + 2)
  #result = cast[wideCString](alloc(L * 4 + 2))
  var d = 0
  for ch in runes(source, L):

    if ch <= UNI_MAX_BMP:
      if ch >= UNI_SUR_HIGH_START and ch <= UNI_SUR_LOW_END:
        result[d] = UNI_REPLACEMENT_CHAR
      else:
        result[d] = cast[Utf16Char](uint16(ch))
    elif ch > UNI_MAX_UTF16:
      result[d] = UNI_REPLACEMENT_CHAR
    else:
      let ch = ch - halfBase
      result[d] = cast[Utf16Char](uint16((ch shr halfShift) + UNI_SUR_HIGH_START))
      inc d<commit_msg>fix #14750, don't allocate too much in newWideCString (#14773)<commit_after>#
#
#            Nim's Runtime Library
#        (c) Copyright 2012 <NAME>
#
#    See the file "copying.txt", included in this
#    distribution, for details about the copyright.
#

# Nim support for C/C++'s `wide strings`:idx:. This is part of the system
# module! Do not import it directly!

#when not declared(ThisIsSystem):
#  {.error: "You must not import this module explicitly".}

type
  Utf16Char* = distinct int16

when defined(nimv2):

  type
    WideCString* = ptr UncheckedArray[Utf16Char]

    WideCStringObj* = object
      bytes: int
      data: WideCString

  proc `=destroy`(a: var WideCStringObj) =
    if a.data != nil:
      deallocShared(a.data)
      a.data = nil

  proc `=`(a: var WideCStringObj; b: WideCStringObj) {.error.}

  proc `=sink`(a: var WideCStringObj; b: WideCStringObj) =
    a.bytes = b.bytes
    a.data = b.data

  proc createWide(a: var WideCStringObj; bytes: int) =
    a.bytes = bytes
    a.data = cast[typeof(a.data)](allocShared0(bytes))

  template `[]`*(a: WideCStringObj; idx: int): Utf16Char = a.data[idx]
  template `[]=`*(a: WideCStringObj; idx: int; val: Utf16Char) = a.data[idx] = val

  template nullWide(): untyped = WideCStringObj(bytes: 0, data: nil)

  converter toWideCString*(x: WideCStringObj): WideCString {.inline.} =
    result = x.data

else:
  template nullWide(): untyped = nil

  type
    WideCString* = ref UncheckedArray[Utf16Char]
    WideCStringObj* = WideCString

  template createWide(a; L) =
    unsafeNew(a, L)

proc ord(arg: Utf16Char): int = int(cast[uint16](arg))

proc len*(w: WideCString): int =
  ## returns the length of a widestring. This traverses the whole string to
  ## find the binary zero end marker!
  result = 0
  while int16(w[result]) != 0'i16: inc result

const
  UNI_REPLACEMENT_CHAR = Utf16Char(0xFFFD'i16)
  UNI_MAX_BMP = 0x0000FFFF
  UNI_MAX_UTF16 = 0x0010FFFF
  UNI_MAX_UTF32 = 0x7FFFFFFF
  UNI_MAX_LEGAL_UTF32 = 0x0010FFFF

  halfShift = 10
  halfBase = 0x0010000
  halfMask = 0x3FF

  UNI_SUR_HIGH_START = 0xD800
  UNI_SUR_HIGH_END = 0xDBFF
  UNI_SUR_LOW_START = 0xDC00
  UNI_SUR_LOW_END = 0xDFFF
  UNI_REPL = 0xFFFD

template ones(n: untyped): untyped = ((1 shl n)-1)

template fastRuneAt(s: cstring, i, L: int, result: untyped, doInc = true) =
  ## Returns the unicode character ``s[i]`` in `result`. If ``doInc == true``
  ## `i` is incremented by the number of bytes that have been processed.
  bind ones

  if ord(s[i]) <= 127:
    result = ord(s[i])
    when doInc: inc(i)
  elif ord(s[i]) shr 5 == 0b110:
    #assert(ord(s[i+1]) shr 6 == 0b10)
    if i <= L - 2:
      result = (ord(s[i]) and (ones(5))) shl 6 or (ord(s[i+1]) and ones(6))
      when doInc: inc(i, 2)
    else:
      result = UNI_REPL
      when doInc: inc(i)
  elif ord(s[i]) shr 4 == 0b1110:
    if i <= L - 3:
      #assert(ord(s[i+1]) shr 6 == 0b10)
      #assert(ord(s[i+2]) shr 6 == 0b10)
      result = (ord(s[i]) and ones(4)) shl 12 or
               (ord(s[i+1]) and ones(6)) shl 6 or
               (ord(s[i+2]) and ones(6))
      when doInc: inc(i, 3)
    else:
      result = UNI_REPL
      when doInc: inc(i)
  elif ord(s[i]) shr 3 == 0b11110:
    if i <= L - 4:
      #assert(ord(s[i+1]) shr 6 == 0b10)
      #assert(ord(s[i+2]) shr 6 == 0b10)
      #assert(ord(s[i+3]) shr 6 == 0b10)
      result = (ord(s[i]) and ones(3)) shl 18 or
               (ord(s[i+1]) and ones(6)) shl 12 or
               (ord(s[i+2]) and ones(6)) shl 6 or
               (ord(s[i+3]) and ones(6))
      when doInc: inc(i, 4)
    else:
      result = UNI_REPL
      when doInc: inc(i)
  else:
    result = 0xFFFD
    when doInc: inc(i)

iterator runes(s: cstring, L: int): int =
  var
    i = 0
    result: int
  while i < L:
    fastRuneAt(s, i, L, result, true)
    yield result

proc newWideCString*(source: cstring, L: int): WideCStringObj =
  createWide(result, L * 2 + 2)
  var d = 0
  for ch in runes(source, L):

    if ch <= UNI_MAX_BMP:
      if ch >= UNI_SUR_HIGH_START and ch <= UNI_SUR_LOW_END:
        result[d] = UNI_REPLACEMENT_CHAR
      else:
        result[d] = cast[Utf16Char](uint16(ch))
    elif ch > UNI_MAX_UTF16:
      result[d] = UNI_REPLACEMENT_CHAR
    else:
      let ch = ch - halfBase
      result[d] = cast[Utf16Char](uint16((ch shr halfShift) + UNI_SUR_HIGH_START))
      inc d
{'fraction_non_alphanumeric': 0.13614969578693606, 'fraction_numerical': 0.042245436804040866, 'mean_word_length': 2.9207920792079207, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2191036', 'n_tokens_mistral': 3781, 'n_tokens_neox': 3451, 'n_words': 941}