<commit_before>class Spelling
  constructor: (props) ->
    @[key] = value for key, value of props
    @text ?= ''
    @type ?= 'normal'
    @syllables ?= [@text]
  toString: -> @text

class Calculation
  @parse: (formula) ->
    prefix = null
    a = formula.split ''
    sep = formula.search /[^a-z]/
    if sep == -1
      console.error "invalid calculation: missing separator"
      return null
    separator = formula.charAt(sep)
    operands = formula.split(separator)
    if formula.charAt(formula.length - 1) == separator
      operands.pop()  # trailing separator
    operator = operands.shift()
    unless operator of @factories
      console.error "unknown calculation: #{operator}"
      return null
    @factories[operator].parse operands

class Transliteration extends Calculation
  @parse: (args) ->
    return null if args.length != 2
    x = new Transliteration
    [x.left, x.right] = args[0..1]
    if x.left.length != x.right.length
      console.error "error parsing transliteration"
      return null
    return x
  calculate: (spelling) ->
    chars = spelling.text.split ''
    xlit_chars = ((if (index = @left.indexOf ch)  == -1 then ch \
      else @right.charAt(index)) for ch in chars)
    result = xlit_chars.join ''
    if result == spelling.text
      [spelling]
    else [
      new Spelling
        text: result
        type: spelling.type
        syllables: spelling.syllables.slice 0
        ancestor: spelling
        modifier: @
    ]

class Transformation extends Calculation
  @parse: (args, Klass = Transformation) ->
    return null if args.length != 2
    x = new Klass
    try
      x.left = new RegExp args[0], 'g'
      x.right = args[1]
    catch error
      console.error "error parsing transformation: #{error}"
      return null
    return x
  calculate: (spelling) ->
    result = spelling.text.replace @left, @right
    if result == spelling.text
      [spelling]
    else [
      new Spelling
        text: result<commit_msg>add support for Perl regexp \U, \L, \E<commit_after>class Spelling
  constructor: (props) ->
    @[key] = value for key, value of props
    @text ?= ''
    @type ?= 'normal'
    @syllables ?= [@text]
  toString: -> @text

# add support for Perl regexp \U, \L, \E
enhancedRegexpReplace = (str, left, right) ->
  str = str.replace left, right
  caseChangeExpr = /\\([UL]).*?(\\E|$)/
  if right.search(caseChangeExpr) != -1
    while str.search(caseChangeExpr) != -1
      str = str.replace caseChangeExpr, (text) ->
        if text.slice(0, 2) == '\\U'
          caseChange = String::toUpperCase
        else if text.slice(0, 2) == '\\L'
          caseChange = String::toLowerCase
        else
          return text  # what?
        end = if text.slice(-2) == '\\E' then -2 else text.length
        caseChange.apply text.slice 2, end
  str

class Calculation
  @parse: (formula) ->
    prefix = null
    a = formula.split ''
    sep = formula.search /[^a-z]/
    if sep == -1
      console.error "invalid calculation: missing separator"
      return null
    separator = formula.charAt(sep)
    operands = formula.split(separator)
    if formula.charAt(formula.length - 1) == separator
      operands.pop()  # trailing separator
    operator = operands.shift()
    unless operator of @factories
      console.error "unknown calculation: #{operator}"
      return null
    @factories[operator].parse operands

class Transliteration extends Calculation
  @parse: (args) ->
    return null if args.length != 2
    x = new Transliteration
    [x.left, x.right] = args[0..1]
    if x.left.length != x.right.length
      console.error "error parsing transliteration"
      return null
    return x
  calculate: (spelling) ->
    chars = spelling.text.split ''
    xlit_chars = ((if (index = @left.indexOf ch)  == -1 then ch \
      else @right.charAt(index)) for ch in chars)
    result = xlit_chars.join ''
    if result == spelling.text
      [spelling]
    else [
      new Spelling
        text: result
        type: spelling.type
        syllables: spelling.syllables.slice 0
        ancestor: spelling
        modifier: @
    ]

class Transformation extends Calculation
  @parse: (args, Klass = Transformation) ->
    return null if args.length != 2
    x = new Klass
    try
      x.left = new RegExp args[0], 'g'
      x.right = args[1]
    catch error
      console.error "error parsing transformation: #{error}"
      return null
    return x
  calculate: (spelling) ->
    result = enhancedRegexpReplace spelling.text, @left, @right
    if result == spelling.text
      [spelling]
    else [
      new Spelling
        text: result
{'fraction_non_alphanumeric': 0.11711125569290826, 'fraction_numerical': 0.006289308176100629, 'mean_word_length': 2.41882876204596, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7421514', 'n_tokens_mistral': 1462, 'n_tokens_neox': 1408, 'n_words': 423}