<commit_before>#
#
#            Nimrod's Runtime Library
#        (c) Copyright 2012 <NAME>
#
#    See the file "copying.txt", included in this
#    distribution, for details about the copyright.
#

## This module provides support to handle the Unicode UTF-8 encoding.

{.deadCodeElim: on.}

include "system/inclrtl"

type
  irune = int # underlying type of TRune
  TRune* = distinct irune   ## type that can hold any Unicode character
  TRune16* = distinct int16 ## 16 bit Unicode character
  
proc `<=%`*(a, b: TRune): bool = return int(a) <=% int(b)
proc `<%`*(a, b: TRune): bool = return int(a) <% int(b)
proc `==`*(a, b: TRune): bool = return int(a) == int(b)

template ones(n: expr): expr = ((1 shl n)-1)

proc runeLen*(s: string): int {.rtl, extern: "nuc$1".} =
  ## returns the number of Unicode characters of the string `s`.
  var i = 0
  while i < len(s):
    if ord(s[i]) <=% 127: inc(i)
    elif ord(s[i]) shr 5 == 0b110: inc(i, 2)
    elif ord(s[i]) shr 4 == 0b1110: inc(i, 3)
    elif ord(s[i]) shr 3 == 0b11110: inc(i, 4)
    elif ord(s[i]) shr 2 == 0b111110: inc(i, 5)
    elif ord(s[i]) shr 1 == 0b1111110: inc(i, 6)
    else: inc i
    inc(result)

proc runeLenAt*(s: string, i: int): int =
  ## returns the number of bytes the rune starting at ``s[i]`` takes.
  if ord(s[i]) <=% 127: result = 1
  elif ord(s[i]) shr 5 == 0b110: result = 2
  elif ord(s[i]) shr 4 == 0b1110: result = 3
  elif ord(s[i]) shr 3 == 0b11110: result = 4
  elif ord(s[i]) shr 2 == 0b111110: result = 5
  elif ord(s[i]) shr 1 == 0b1111110: result = 6
  else: result = 1

template fastRuneAt*(s: string, i: int, result: expr, doInc = true) =
  ## Returns the unicode character ``s[i]`` in `result`. If ``doInc == true``
  ## `i` is incremented by the number of bytes that have been processed.
  bind ones
  if ord(s[i]) <=% 127:
    result = TRune(ord(s[i]))
    when doInc: inc(i)
  elif ord(s[i]) shr 5 == 0b110:
    # assert(ord(s[i+1]) shr 6 == 0b10)
    result = TRune((ord(s[i]) and (ones(5))) shl 6 or 
                   (ord(s[i+1]) and ones(6)))
    when doInc: inc(i, 2)
  elif ord(s[i]) shr 4 == 0b1110:
    # assert(ord(s[i+1]) shr 6 == 0b10)
    # assert(ord(s[i+2]) shr 6 == 0b10)
    result = TRune((ord(s[i]) and ones(4)) shl 12 or
             (ord(s[i+1]) and ones(6)) shl 6 or
             (ord(s[i+2]) and ones(6)))
    when doInc: inc(i, 3)
  elif ord(s[i]) shr 3 == 0b11110:
    # assert(ord(s[i+1]) shr 6 == 0b10)
    # assert(ord(s[i+2]) shr 6 == 0b10)
    # assert(ord(s[i+3]) shr 6 == 0b10)
    result = TRune((ord(s[i]) and ones(3)) shl 18 or
             (ord(s[i+1]) and ones(6)) shl 12 or
             (ord(s[i+2]) and ones(6)) shl 6 or
             (ord(s[i+3]) and ones(6)))
    when doInc: inc(i, 4)
  elif ord(s[i]) shr 2 == 0b111110: 
    # assert(ord(s[i+1]) shr 6 == 0b10)
    # assert(ord(s[i+2]) shr 6 == 0b10)
    # assert(ord(s[i+3]) shr 6 == 0b10)
    # assert(ord(s[i+4]) shr 6 == 0b10)
    result = TRune((ord(s[i]) and ones(2)) shl 24 or
             (ord(s[i+1]) and ones(6)) shl 18 or
             (ord(s[i+2]) and ones(6)) shl 12 or
             (ord(s[i+3]) and ones(6)) shl 6 or
             (ord(s[i+4]) and ones(6)))
    when doInc: inc(i, 5)
  elif ord(s[i]) shr 1 == 0b1111110: 
    # assert(ord(s[i+1]) shr 6 == 0b10)
    # assert(ord(s[i+2]) shr 6 == 0b10)
    # assert(ord(s[i+3]) shr 6 == 0b10)
    # assert(ord(s[i+4]) shr 6 == 0b10)
    # assert(ord(s[i+5]) shr 6 == 0b10)
    result = TRune((ord(s[i]) and ones(1)) shl 30 or
             (ord(s[i+1]) and ones(6)) shl 24 or
             (ord(s[i+2]) and ones(6)) shl 18 or
             (ord(s[i+3]) and ones(6)) shl 12 or
             (ord(s[i+4]) and ones(6)) shl 6 or
             (ord(s[i+5]) and ones(6)))
    when doInc: inc(i, 6)
  else:
    result = TRune(ord(s[i]))
    when doInc: inc(i)

proc runeAt*(s: string, i: int): TRune =
  ## returns the unicode character in `s` at byte index `i`
  fastRuneAt(s, i, result, false)

proc toUTF8*(c: TRune): string {.rtl, extern: "nuc$1".} = 
  ## converts a rune into its UTF8 representation
  var i = irune(c)
  if i <=% 127:
    result = newString(1)
    result[0] = chr(i)
  elif i <=% 0x07FF:
    result = newString(2)
    result[0] = chr((i shr 6) or 0b110_00000)
    result[1] = chr((i and ones(6)) or 0b10_000000)
  elif i <=% 0xFFFF:
    result = newString(3)
    result[0] = chr(i shr 12 or 0b1110_0000)
    result[1] = chr(i shr 6 and ones(6) or 0b10_0000_00)
    result[2] = chr(i and ones(6) or 0b10_0000_00)
  elif i <=% 0x0010FFFF:
    result = newString(4)
    result[0] = chr(i shr 18 or 0b1111_0000)
    result[1] = chr(i shr 12 and ones(6) or 0b10_0000_00)
    result[2] = chr(i shr 6 and ones(6) or 0b10_0000_00)
    result[3] = chr(i and ones(6) or 0b10_0000_00)
  else:
    result = newString(1)
    result[0] = chr(i)

proc `$`*(rune: TRune): string = rune.toUTF8

proc `$`*(runes: seq[TRune]): string =
  ## converts a sequence of runes to a string
  result = ""
  for rune in runes: result.add(rune.toUTF8)

const
  alphaRanges = [
    0x00d8,  0x00f6,  #  - 
    0x00f8,  0x01f5,  #  -    
    0x0250,  0x02a8,  #  -    <commit_msg>Define $ operator for TRune<commit_after>#
#
#            Nimrod's Runtime Library
#        (c) Copyright 2012 <NAME>
#
#    See the file "copying.txt", included in this
#    distribution, for details about the copyright.
#

## This module provides support to handle the Unicode UTF-8 encoding.

{.deadCodeElim: on.}

include "system/inclrtl"

type
  irune = int # underlying type of TRune
  TRune* = distinct irune   ## type that can hold any Unicode character
  TRune16* = distinct int16 ## 16 bit Unicode character
  
proc `<=%`*(a, b: TRune): bool = return int(a) <=% int(b)
proc `<%`*(a, b: TRune): bool = return int(a) <% int(b)
proc `==`*(a, b: TRune): bool = return int(a) == int(b)

template ones(n: expr): expr = ((1 shl n)-1)

proc runeLen*(s: string): int {.rtl, extern: "nuc$1".} =
  ## returns the number of Unicode characters of the string `s`.
  var i = 0
  while i < len(s):
    if ord(s[i]) <=% 127: inc(i)
    elif ord(s[i]) shr 5 == 0b110: inc(i, 2)
    elif ord(s[i]) shr 4 == 0b1110: inc(i, 3)
    elif ord(s[i]) shr 3 == 0b11110: inc(i, 4)
    elif ord(s[i]) shr 2 == 0b111110: inc(i, 5)
    elif ord(s[i]) shr 1 == 0b1111110: inc(i, 6)
    else: inc i
    inc(result)

proc runeLenAt*(s: string, i: int): int =
  ## returns the number of bytes the rune starting at ``s[i]`` takes.
  if ord(s[i]) <=% 127: result = 1
  elif ord(s[i]) shr 5 == 0b110: result = 2
  elif ord(s[i]) shr 4 == 0b1110: result = 3
  elif ord(s[i]) shr 3 == 0b11110: result = 4
  elif ord(s[i]) shr 2 == 0b111110: result = 5
  elif ord(s[i]) shr 1 == 0b1111110: result = 6
  else: result = 1

template fastRuneAt*(s: string, i: int, result: expr, doInc = true) =
  ## Returns the unicode character ``s[i]`` in `result`. If ``doInc == true``
  ## `i` is incremented by the number of bytes that have been processed.
  bind ones
  if ord(s[i]) <=% 127:
    result = TRune(ord(s[i]))
    when doInc: inc(i)
  elif ord(s[i]) shr 5 == 0b110:
    # assert(ord(s[i+1]) shr 6 == 0b10)
    result = TRune((ord(s[i]) and (ones(5))) shl 6 or 
                   (ord(s[i+1]) and ones(6)))
    when doInc: inc(i, 2)
  elif ord(s[i]) shr 4 == 0b1110:
    # assert(ord(s[i+1]) shr 6 == 0b10)
    # assert(ord(s[i+2]) shr 6 == 0b10)
    result = TRune((ord(s[i]) and ones(4)) shl 12 or
             (ord(s[i+1]) and ones(6)) shl 6 or
             (ord(s[i+2]) and ones(6)))
    when doInc: inc(i, 3)
  elif ord(s[i]) shr 3 == 0b11110:
    # assert(ord(s[i+1]) shr 6 == 0b10)
    # assert(ord(s[i+2]) shr 6 == 0b10)
    # assert(ord(s[i+3]) shr 6 == 0b10)
    result = TRune((ord(s[i]) and ones(3)) shl 18 or
             (ord(s[i+1]) and ones(6)) shl 12 or
             (ord(s[i+2]) and ones(6)) shl 6 or
             (ord(s[i+3]) and ones(6)))
    when doInc: inc(i, 4)
  elif ord(s[i]) shr 2 == 0b111110: 
    # assert(ord(s[i+1]) shr 6 == 0b10)
    # assert(ord(s[i+2]) shr 6 == 0b10)
    # assert(ord(s[i+3]) shr 6 == 0b10)
    # assert(ord(s[i+4]) shr 6 == 0b10)
    result = TRune((ord(s[i]) and ones(2)) shl 24 or
             (ord(s[i+1]) and ones(6)) shl 18 or
             (ord(s[i+2]) and ones(6)) shl 12 or
             (ord(s[i+3]) and ones(6)) shl 6 or
             (ord(s[i+4]) and ones(6)))
    when doInc: inc(i, 5)
  elif ord(s[i]) shr 1 == 0b1111110: 
    # assert(ord(s[i+1]) shr 6 == 0b10)
    # assert(ord(s[i+2]) shr 6 == 0b10)
    # assert(ord(s[i+3]) shr 6 == 0b10)
    # assert(ord(s[i+4]) shr 6 == 0b10)
    # assert(ord(s[i+5]) shr 6 == 0b10)
    result = TRune((ord(s[i]) and ones(1)) shl 30 or
             (ord(s[i+1]) and ones(6)) shl 24 or
             (ord(s[i+2]) and ones(6)) shl 18 or
             (ord(s[i+3]) and ones(6)) shl 12 or
             (ord(s[i+4]) and ones(6)) shl 6 or
             (ord(s[i+5]) and ones(6)))
    when doInc: inc(i, 6)
  else:
    result = TRune(ord(s[i]))
    when doInc: inc(i)

proc runeAt*(s: string, i: int): TRune =
  ## returns the unicode character in `s` at byte index `i`
  fastRuneAt(s, i, result, false)

proc toUTF8*(c: TRune): string {.rtl, extern: "nuc$1".} = 
  ## converts a rune into its UTF8 representation
  var i = irune(c)
  if i <=% 127:
    result = newString(1)
    result[0] = chr(i)
  elif i <=% 0x07FF:
    result = newString(2)
    result[0] = chr((i shr 6) or 0b110_00000)
    result[1] = chr((i and ones(6)) or 0b10_000000)
  elif i <=% 0xFFFF:
    result = newString(3)
    result[0] = chr(i shr 12 or 0b1110_0000)
    result[1] = chr(i shr 6 and ones(6) or 0b10_0000_00)
    result[2] = chr(i and ones(6) or 0b10_0000_00)
  elif i <=% 0x0010FFFF:
    result = newString(4)
    result[0] = chr(i shr 18 or 0b1111_0000)
    result[1] = chr(i shr 12 and ones(6) or 0b10_0000_00)
    result[2] = chr(i shr 6 and ones(6) or 0b10_0000_00)
    result[3] = chr(i and ones(6) or 0b10_0000_00)
  else:
    result = newString(1)
    result[0] = chr(i)

proc `$`*(rune: TRune): string =
  ## converts a rune to a string
  rune.toUTF8

proc `$`*(runes: seq[TRune]): string =
  ## converts a sequence of runes to a string
  result = ""
  for rune in runes: result.add(rune.toUTF8)

const
  alphaRanges = [
    0x00d8,  0x00f6,  #  - 
    0x00f8,  0x01f5,  #  -    
    0x0250,  0x02a8,  #  -    
{'fraction_non_alphanumeric': 0.17726078068723838, 'fraction_numerical': 0.08468801713228852, 'mean_word_length': 2.7855563743551954, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3996611', 'n_tokens_mistral': 4884, 'n_tokens_neox': 4287, 'n_words': 1275}