<commit_before>#lang rosette

(require json)
(require "interp-enumerate.rkt")

; array of mappings functions to expression operators, and their types
(define mappings (make-hash))

(hash-set! mappings 'or '("or" "BinaryExpression"))
(hash-set! mappings 'and '("and" "BinaryExpression"))
(hash-set! mappings 'not '("not" "UnaryExpression"))
(hash-set! mappings '= '("eq" "BinaryExpression"))
(hash-set! mappings '<= '("leq" "BinaryExpression"))
(hash-set! mappings '>= '("geq" "BinaryExpression"))
(hash-set! mappings '< '("lt" "BinaryExpression"))
(hash-set! mappings '> '("gt" "BinaryExpression"))
(hash-set! mappings 'concat '("concat" "BinaryExpression"))
(hash-set! mappings '+ '("plus" "BinaryExpression"))
(hash-set! mappings '- '("minus" "BinaryExpression"))
(hash-set! mappings '/ '("divide" "BinaryExpression"))
(hash-set! mappings '* '("multiply" "BinaryExpression"))
(hash-set! mappings 'quotient '("quotient" "BinaryExpression"))
(hash-set! mappings 'remainder '("modulo" "BinaryExpression"))
(hash-set! mappings 'abs '("absolute" "UnaryExpression"))
(hash-set! mappings 'ceiling '("ceiling" "UnaryExpression"))
(hash-set! mappings 'floor '("floor" "UnaryExpression"))
(hash-set! mappings 'truncate '("truncate" "UnaryExpression"))
(hash-set! mappings 'sign '("sign" "UnaryExpression"))
(hash-set! mappings 'if '("ifelse" "TernaryExpression"))
(hash-set! mappings 'substring '("substring" "BinaryExpression"))
(hash-set! mappings 'index-of '("index-of" "BinaryExpression"))
(hash-set! mappings 'length '("string-length" "UnaryExpression"))
(hash-set! mappings '== '("eq" "BinaryExpression"))
(hash-set! mappings 'in '("column" "Column"))
(hash-set! mappings 'is-null '("isnull" "UnaryExpression"))
(hash-set! mappings 'is-not-null '("is-not-null" "UnaryExpression"))



; define a set of mappings from functions to their pretty equivalents
(define pretty (make-hash))
(hash-set! pretty 'or "lhs or rhs")
(hash-set! pretty 'and "lhs and rhs")
(hash-set! pretty 'not "is not expression")
(hash-set! pretty '= "lhs = rhs")
(hash-set! pretty '<= "lhs <= rhs")
(hash-set! pretty '>= "lhs >= rhs")
(hash-set! pretty '< "(lhs < rhs)")
(hash-set! pretty '> "lhs > rhs")
(hash-set! pretty 'concat "lhs concatenated with rhs")
(hash-set! pretty '+ "lhs + rhs")
(hash-set! pretty '- "(lhs - rhs)")
(hash-set! pretty '/ "lhs / rhs")
(hash-set! pretty '* "lhs * rhs")
(hash-set! pretty 'quotient "'quotient when lhs / rhs" )
(hash-set! pretty 'remainder "<remainder when lhs / rhs")
(hash-set! pretty 'abs "absolute value of expression")
(hash-set! pretty 'ceiling "ceiling of expression")
(hash-set! pretty 'floor "floor of expression")
(hash-set! pretty 'truncate "truncated expression")
(hash-set! pretty 'sign "sign of expression")
(hash-set! pretty 'if "if one is true then two otherwise three")
(hash-set! pretty 'substring "a part of one from location two to location three")
(hash-set! pretty 'index-of "location of rhs in lhs")
(hash-set! pretty 'length "length of expression")
(hash-set! pretty '== "lhs = rhs")
(hash-set! pretty 'in "expression")
(hash-set! pretty 'is-null "expression is null")
(hash-set! pretty 'is-not-null "expression is not null")

(define (check-exp type)
  (if (hash-has-key? mappings type)
      (hash-ref mappings type)
      (raise "No expression found")))
<commit_msg>adding expressions for dates<commit_after>#lang rosette

(require json)
(require "interp-enumerate.rkt")

; array of mappings functions to expression operators, and their types
(define mappings (make-hash))

(hash-set! mappings 'or '("or" "BinaryExpression"))
(hash-set! mappings 'and '("and" "BinaryExpression"))
(hash-set! mappings 'not '("not" "UnaryExpression"))
(hash-set! mappings '= '("eq" "BinaryExpression"))
(hash-set! mappings '<= '("leq" "BinaryExpression"))
(hash-set! mappings '>= '("geq" "BinaryExpression"))
(hash-set! mappings '< '("lt" "BinaryExpression"))
(hash-set! mappings '> '("gt" "BinaryExpression"))
(hash-set! mappings 'concat '("concat" "BinaryExpression"))
(hash-set! mappings '+ '("plus" "BinaryExpression"))
(hash-set! mappings '- '("minus" "BinaryExpression"))
(hash-set! mappings '/ '("divide" "BinaryExpression"))
(hash-set! mappings '* '("multiply" "BinaryExpression"))
(hash-set! mappings 'quotient '("quotient" "BinaryExpression"))
(hash-set! mappings 'remainder '("modulo" "BinaryExpression"))
(hash-set! mappings 'abs '("absolute" "UnaryExpression"))
(hash-set! mappings 'ceiling '("ceiling" "UnaryExpression"))
(hash-set! mappings 'floor '("floor" "UnaryExpression"))
(hash-set! mappings 'truncate '("truncate" "UnaryExpression"))
(hash-set! mappings 'sign '("sign" "UnaryExpression"))
(hash-set! mappings 'if '("ifelse" "TernaryExpression"))
(hash-set! mappings 'substring '("substring" "BinaryExpression"))
(hash-set! mappings 'index-of '("index-of" "BinaryExpression"))
(hash-set! mappings 'length '("string-length" "UnaryExpression"))
(hash-set! mappings '== '("eq" "BinaryExpression"))
(hash-set! mappings 'in '("column" "Column"))
(hash-set! mappings 'is-null '("isnull" "UnaryExpression"))
(hash-set! mappings 'is-not-null '("is-not-null" "UnaryExpression"))
(hash-set! mappings 'date-subtract '("date-subtract" "BinaryExpression"))
(hash-set! mappings 'date-le '("date-le" "BinaryExpression"))
(hash-set! mappings 'date-lt '("date-lt" "BinaryExpression"))
(hash-set! mappings 'date-ge '("date-ge" "BinaryExpression"))
(hash-set! mappings 'date-gt '("date-gt" "BinaryExpression"))
(hash-set! mappings 'date-equal '("date-equal" "BinaryExpression"))
(hash-set! mappings 'add-seconds' '("add-seconds" "BinaryExpression"))
(hash-set! mappings 'add-minutes' '("add-minutes" "BinaryExpression"))
(hash-set! mappings 'add-hours' '("add-hours" "BinaryExpression"))
(hash-set! mappings 'add-days' '("add-days" "BinaryExpression"))
(hash-set! mappings 'add-months' '("add-months" "BinaryExpression"))
(hash-set! mappings 'add-years' '("add-years" "BinaryExpression"))
(hash-set! mappings 'subtract-seconds' '("subtract-seconds" "BinaryExpression"))
(hash-set! mappings 'subtract-minutes' '("subtract-minutes" "BinaryExpression"))
(hash-set! mappings 'subtract-hours' '("subtract-hours" "BinaryExpression"))
(hash-set! mappings 'subtract-days' '("subtract-days" "BinaryExpression"))
(hash-set! mappings 'subtract-months' '("subtract-months" "BinaryExpression"))
(hash-set! mappings 'subtract-years' '("subtract-years" "BinaryExpression"))
(hash-set! mappings 'extract-seconds '("extract-seconds" "UnaryExpression"))
(hash-set! mappings 'extract-minutes '("extract-minutes" "UnaryExpression"))
(hash-set! mappings 'extract-hours '("extract-hours" "UnaryExpression"))
(hash-set! mappings 'extract-days '("extract-days" "UnaryExpression"))
(hash-set! mappings 'extract-months '("extract-months" "UnaryExpression"))
(hash-set! mappings 'extract-years '("extract-years" "UnaryExpression"))
(hash-set! mappings 'date-to-epoch '("date-to-epoch" "UnaryExpression"))
(hash-set! mappings 'date-from-epoch '("date-from-epoch" "UnaryExpression"))


; define a set of mappings from functions to their pretty equivalents
(define pretty (make-hash))
(hash-set! pretty 'or "lhs or rhs")
(hash-set! pretty 'and "lhs and rhs")
(hash-set! pretty 'not "is not expression")
(hash-set! pretty '= "lhs = rhs")
(hash-set! pretty '<= "lhs <= rhs")
(hash-set! pretty '>= "lhs >= rhs")
(hash-set! pretty '< "(lhs < rhs)")
(hash-set! pretty '> "lhs > rhs")
(hash-set! pretty 'concat "lhs concatenated with rhs")
(hash-set! pretty '+ "lhs + rhs")
(hash-set! pretty '- "(lhs - rhs)")
(hash-set! pretty '/ "lhs / rhs")
(hash-set! pretty '* "lhs * rhs")
(hash-set! pretty 'quotient "'quotient when lhs / rhs" )
(hash-set! pretty 'remainder "<remainder when lhs / rhs")
(hash-set! pretty 'abs "absolute value of expression")
(hash-set! pretty 'ceiling "ceiling of expression")
(hash-set! pretty 'floor "floor of expression")
(hash-set! pretty 'truncate "truncated expression")
(hash-set! pretty 'sign "sign of expression")
(hash-set! pretty 'if "if one is true then two otherwise three")
(hash-set! pretty 'substring "a part of one from location two to location three")
(hash-set! pretty 'index-of "location of rhs in lhs")
(hash-set! pretty 'length "length of expression")
(hash-set! pretty '== "lhs = rhs")
(hash-set! pretty 'in "expression")
(hash-set! pretty 'is-null "expression is null")
(hash-set! pretty 'is-not-null "expression is not null")
(hash-set! pretty 'date-le "lhs <= rhs")
(hash-set! pretty 'date-lt "lhs < rhs")
(hash-set! pretty 'date-ge "lhs >= rhs")
(hash-set! pretty 'date-gt "lhs > rhs")
(hash-set! pretty 'date-equal "lhs = rhs")
(hash-set! pretty 'add-seconds' "lhs + rhs seconds")
(hash-set! pretty 'add-minutes' "lhs + rhs minutes")
(hash-set! pretty 'add-hours' "lhs + rhs hours")
(hash-set! pretty 'add-days' "lhs + rhs days")
(hash-set! pretty 'add-months' "lhs + rhs months")
(hash-set! pretty 'add-years' "lhs + rhs years")
(hash-set! pretty 'subtract-seconds' "lhs - rhs seconds")
(hash-set! pretty 'subtract-minutes' "lhs - rhs minutes")
(hash-set! pretty 'subtract-hours' "lhs - rhs hours")
(hash-set! pretty 'subtract-days' "lhs - rhs days")
(hash-set! pretty 'subtract-months' "lhs - rhs months")
(hash-set! pretty 'subtract-years' "lhs - rhs years")
(hash-set! pretty 'extract-seconds "extract seconds from expression")
(hash-set! pretty 'extract-minutes "extract minutes from expression")
(hash-set! pretty 'extract-hours "extract hours from expression")
(hash-set! pretty 'extract-days "extract days from expresison")
(hash-set! pretty 'extract-months "extract months from expression")
(hash-set! pretty 'extract-years "extract years from expression")
(hash-set! pretty 'date-to-epoch "extract unix time from expression")
(hash-set! pretty 'date-from-epoch "create date from unix time")



(define (check-exp type)
  (if (hash-has-key? mappings type)
      (hash-ref mappings type)
      (raise "No expression found")))

{'fraction_non_alphanumeric': 0.1905388978930308, 'fraction_numerical': 0.0, 'mean_word_length': 6.007097232079489, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5964118', 'n_tokens_mistral': 3360, 'n_tokens_neox': 3214, 'n_words': 970}