<commit_before>export function q
  """
    '#{ "#it".replace /'/g "''" }'
  """

export function qq => match it
  | '*' => it
  | /^(.*?)\.(\d+)$/ => qq(that.1) + "[#{ 1 + parseInt that.2 }]"
  else => it.replace /^([^.]*)/ -> "\"#{ RegExp.$1.replace /"/g '""' }\""

export function walk(model, meta)
  return [] unless meta?[model]
  for col, spec of meta[model]
    [compile(model, spec), col]

export function compile(model, field)
  {$query, $from, $and, $} = field ? {}
  switch
  | $from? => let from-table = qq "#{$from}", model-table = qq "#{model}"
      """
      (SELECT COALESCE(ARRAY_TO_JSON(ARRAY_AGG(_)), '[]') FROM (SELECT * FROM #from-table
          WHERE #{ qq "_#model" } = #model-table."_id" AND #{
              switch
              | $query?                   => cond model, $query
              | _                         => true
          }
      ) AS _)
      """
  | $? => cond model, $
  | typeof field is \object => cond model, field
  | _ => field

export function cond(model, spec)
  switch typeof spec
  | \number => spec
  | \string => qq spec
  | \object =>
    if spec.$or
      [cond model, "#k": ex for k, ex of spec.$or].map(-> "(#it)") * " OR "
      # Implicit AND on all k,v
    else
      ([ test model, qq(k), v for k, v of spec ].reduce (++)) * " AND "
  | _ => it

export function test(model, key, expr)
  switch typeof expr
  | <[ number boolean ]> => ["(#key = #expr)"]
  | \string => ["(#key = #{ q expr })"]
  | \object =>
    unless expr?
      return ["(#key IS NULL)"]
    for op, ref of expr
      switch op
      | \$not => "(NOT #{test model, key, ref})"
      | \$lt
          res = evaluate model, ref
          "(#key < #res)"
      | \$gt
          res = evaluate model, ref
          "(#key > #res)"
      | \$contains
          ref = [ref] unless Array.isArray ref
          res = q "{#{ref.join \,}}"
          "(#key @> #res)"
      | \$ => let model-table = qq "#{model}s"
          "(#key = #model-table.#{ qq ref })"
      | \$literal => "(#key = #ref)"
      | _ => throw "Unknown operator: #op"
  | \undefined => [true]
<commit_msg>added the $matches and $matchesCase operators for regex matching<commit_after>export function q
  """
    '#{ "#it".replace /'/g "''" }'
  """

export function qq => match it
  | '*' => it
  | /^(.*?)\.(\d+)$/ => qq(that.1) + "[#{ 1 + parseInt that.2 }]"
  else => it.replace /^([^.]*)/ -> "\"#{ RegExp.$1.replace /"/g '""' }\""

export function walk(model, meta)
  return [] unless meta?[model]
  for col, spec of meta[model]
    [compile(model, spec), col]

export function compile(model, field)
  {$query, $from, $and, $} = field ? {}
  switch
  | $from? => let from-table = qq "#{$from}", model-table = qq "#{model}"
      """
      (SELECT COALESCE(ARRAY_TO_JSON(ARRAY_AGG(_)), '[]') FROM (SELECT * FROM #from-table
          WHERE #{ qq "_#model" } = #model-table."_id" AND #{
              switch
              | $query?                   => cond model, $query
              | _                         => true
          }
      ) AS _)
      """
  | $? => cond model, $
  | typeof field is \object => cond model, field
  | _ => field

export function cond(model, spec)
  switch typeof spec
  | \number => spec
  | \string => qq spec
  | \object =>
    if spec.$or
      [cond model, "#k": ex for k, ex of spec.$or].map(-> "(#it)") * " OR "
      # Implicit AND on all k,v
    else
      ([ test model, qq(k), v for k, v of spec ].reduce (++)) * " AND "
  | _ => it

export function test(model, key, expr)
  switch typeof expr
  | <[ number boolean ]> => ["(#key = #expr)"]
  | \string => ["(#key = #{ q expr })"]
  | \object =>
    unless expr?
      return ["(#key IS NULL)"]
    for op, ref of expr
      switch op
      | \$not => "(NOT #{test model, key, ref})"
      | \$lt
          res = evaluate model, ref
          "(#key < #res)"
      | \$gt
          res = evaluate model, ref
          "(#key > #res)"
      | \$matches
          res = evaluate model, ref
          "(#key ~* #res)"
      | \$matchesCase
          res = evaluate model, ref
          "(#key ~ #res)"
      | \$contains
          ref = [ref] unless Array.isArray ref
          res = q "{#{ref.join \,}}"
          "(#key @> #res)"
      | \$ => let model-table = qq "#{model}s"
          "(#key = #model-table.#{ qq ref })"
      | \$literal => "(#key = #ref)"
      | _ => throw "Unknown operator: #op"
  | \undefined => [true]

{'fraction_non_alphanumeric': 0.20484382073336352, 'fraction_numerical': 0.0018107741059302852, 'mean_word_length': 1.901510177281681, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3245336', 'n_tokens_mistral': 1520, 'n_tokens_neox': 1494, 'n_words': 426}