<commit_before>library = require './stdlib'
special = require './corelib'

class Context
  constructor: (scope, parent) ->
    @scope = scope
    @parent = parent
    return

  get: (ident) ->
    if ident of @scope
      @scope[ident]
    else if @parent isnt undefined
      @parent.get ident

  set: (ident, value) ->
    if @parnet is undefined
      @scope[ident] = value
    else
      @parent.set ident, value

class Var
  constructor: (type, value) ->
    @type = type
    @value = value
    return

tokenize = (input) ->
  o = input.split('"')
    .map((x, i) ->
      if i % 2 is 0 # not in string
        x.replace(/\(/g, ' ( ')
          .replace(/\)/g, ' ) ')
          .replace(/\{/g, ' { ')
          .replace(/\}/g, ' } ')
          .replace(/:/g, ' : ')
      else
        x.replace /\s/g, "!%!"
    )
    .join('"')
    .trim()
    .split(/\s+/)
    .map (x) -> x.replace(/!%!/g, " ")
  i = 1
  for t in o[1..]
    switch t
      when '('
        i++
        if i is 1
          o.unshift '('
          o.push ')'
          break
      when ')' then i--
  o

objectize = (input, object = {}, key = true) ->
  token = input.shift()
  if token is '}'
    categorize object
  else if token is ':'
    objectize input, object, false
  else if key
    object[categorize(token).value] = objectize input, object
    objectize input, object, true
  else
    if token is '('
      parenthesize token
    else
      categorize token


parenthesize = (input, list = []) ->
  token = input.shift()
  switch token
    when undefined
      list.pop()
    when '('
      list.push parenthesize input
      parenthesize input, list
    when ')'
      list
    when '{'
      list.push objectize input
      parenthesize input, list
    else
      parenthesize input, list.concat categorize token

categorize = (input) ->
  if not isNaN parseFloat input
    new Var 'literal', parseFloat input
  else if input[0] is '"' and input[-1..] is '"'
    new Var 'literal', input.slice(1, -1)
  else if input instanceof Object
    new Var 'dict', input
  else
    new Var 'identifier', input

parse = (input) ->
  parenthesize tokenize input

object_mapper = (obj, fn) -> fn v for v in Object.getOwnPropertyNames obj

interpret = (input, context=(new Context library)) ->
  if input instanceof Array
    interpret_list input, context
  else if input.type is 'identifier'
    context.get input.value
  else if input.type is 'dict'
    object_mapper input.value, (x) -> input.value[x] = interpret input.value[x]
    input.value
  else
    input.value

interpret_list = (input, context) ->
  if input[0].value of special
    special[input[0].value](input, context)
  else
    list = input.map (x) -> interpret(x, context)
    if list[0] instanceof Function
      list[0] list[1..]
    else
      list

exports.interpret = interpret
exports.parse = parse
exports.Context = Context
exports.Var = Var
<commit_msg>Fix for crash on empty list<commit_after>library = require './stdlib'
special = require './corelib'

class Context
  constructor: (scope, parent) ->
    @scope = scope
    @parent = parent
    return

  get: (ident) ->
    if ident of @scope
      @scope[ident]
    else if @parent isnt undefined
      @parent.get ident

  set: (ident, value) ->
    if @parnet is undefined
      @scope[ident] = value
    else
      @parent.set ident, value

class Var
  constructor: (type, value) ->
    @type = type
    @value = value
    return

tokenize = (input) ->
  o = input.split('"')
    .map((x, i) ->
      if i % 2 is 0 # not in string
        x.replace(/\(/g, ' ( ')
          .replace(/\)/g, ' ) ')
          .replace(/\{/g, ' { ')
          .replace(/\}/g, ' } ')
          .replace(/:/g, ' : ')
      else
        x.replace /\s/g, "!%!"
    )
    .join('"')
    .trim()
    .split(/\s+/)
    .map (x) -> x.replace(/!%!/g, " ")
  i = 1
  for t in o[1..]
    switch t
      when '('
        i++
        if i is 1
          o.unshift '('
          o.push ')'
          break
      when ')' then i--
  o

objectize = (input, object = {}, key = true) ->
  token = input.shift()
  if token is '}'
    categorize object
  else if token is ':'
    objectize input, object, false
  else if key
    object[categorize(token).value] = objectize input, object
    objectize input, object, true
  else
    if token is '('
      parenthesize token
    else
      categorize token


parenthesize = (input, list = []) ->
  token = input.shift()
  switch token
    when undefined
      list.pop()
    when '('
      list.push parenthesize input
      parenthesize input, list
    when ')'
      list
    when '{'
      list.push objectize input
      parenthesize input, list
    else
      parenthesize input, list.concat categorize token

categorize = (input) ->
  if not isNaN parseFloat input
    new Var 'literal', parseFloat input
  else if input[0] is '"' and input[-1..] is '"'
    new Var 'literal', input.slice(1, -1)
  else if input instanceof Object
    new Var 'dict', input
  else
    new Var 'identifier', input

parse = (input) ->
  parenthesize tokenize input

object_mapper = (obj, fn) -> fn v for v in Object.getOwnPropertyNames obj

interpret = (input, context=(new Context library)) ->
  if input instanceof Array
    interpret_list input, context
  else if input.type is 'identifier'
    context.get input.value
  else if input.type is 'dict'
    object_mapper input.value, (x) -> input.value[x] = interpret input.value[x]
    input.value
  else
    input.value

interpret_list = (input, context) ->
  return if input.length is 0
  if input[0].value of special
    special[input[0].value](input, context)
  else
    list = input.map (x) -> interpret(x, context)
    if list[0] instanceof Function
      list[0] list[1..]
    else
      list

exports.interpret = interpret
exports.parse = parse
exports.Context = Context
exports.Var = Var

{'fraction_non_alphanumeric': 0.13295687885010268, 'fraction_numerical': 0.004962354551676934, 'mean_word_length': 2.1835511982570806, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3897884', 'n_tokens_mistral': 1904, 'n_tokens_neox': 1892, 'n_words': 592}