<commit_before>// A version of <NAME>'s lis.py (http://norvig.com/lispy.html) and
// lispy.py (http://norvig.com/lispy2.html) that compiles to Javascript

(function (sss) {
  "use strict";

  // Perl-like string formatting (use $0, $1... in the string, referring to
  // values passed as parameters)
  String.prototype.fmt = function () {
    var args = arguments;
    return this.replace(/\$(\d+)/g, function (s, p) {
      return args[p] === undefined || args[p] === null ? "" : args[p];
    });
  }

  // Tokenizer
  sss.tokenize = function(s) {
    var tokens = [];
    while (true) {
      var m = s
        .match(/\s*(,@|[('`,)]|"(?:[\\].|[^\\"])*"|;.*|[^\s('"`,;)]*)(.*)/);
      if (m && m[1].length > 0 && m[1][0] !== ";") {
        tokens.push(m[1]);
        s = m[2];
      } else {
        return tokens;
      }
    }
  }

  var symbol = {
    toString: function () {
      return "$0".fmt(this.symbol);
    }
  };

  sss.symbols = {};

  // Symbols
  // Get a symbol for a name, creating it if necessary
  sss.get_symbol = function (s) {
    if (!sss.symbols.hasOwnProperty(s)) {
      sss.symbols[s] = Object.create(symbol);
      sss.symbols[s].symbol = s;
    }
    return sss.symbols[s];
  }

  var s_if = sss.get_symbol("if");
  var s_set = sss.get_symbol("set!");
  var s_define = sss.get_symbol("define");
  var s_define_macro = sss.get_symbol("define-macro");
  var s_lambda = sss.get_symbol("lambda");
  var s_begin = sss.get_symbol("begin");

  var quotes = {
    "'": sss.get_symbol("quote"),
    "`": sss.get_symbol("quasiquote"),
    ",": sss.get_symbol("unquote"),
    ",@": sss.get_symbol("unquotesplicing")
  };

  // Create an atom from a token value (#t/#f, string, number, symbol)
  function atom(token) {
    if (token === "#t") {
      return true;
    } else if (token === "#f") {
      return false;
    } else if (token[0] === '"') {
      return JSON.parse(token);
    } else {
      var n = parseFloat(token);
      return isNaN(n) ? sss.get_symbol(token) : n;
    }
  }

  function cons(x, xs) {
    return [x].concat(xs);
  }

  function is_pair(x) {
    return Array.isArray(x) && x.length !== 0;
  }

  var macros = {};

  function add_macro(name, f) {
    var symbol = sss.get_symbol(name);
    macros[name] = f;
  }

  add_macro("let", function () {
    var args = Array.prototype.slice.call(arguments);
    var x = cons(sss.symbols["let"], args);
    check(x, args.length > 1);
    var bindings = args[0];
    var body = args.slice(1);
    check(x, bindings.every(function (b) {
      return Array.isArray(b) && b.length === 2 && is_symbol(b[0]);
    }), "illegal binding list");
    var vars = bindings.map(function (b) { return b[0]; });
    var vals = bindings.map(function (b) { return b[1]; });
    return cons([s_lambda, vars].concat(body.map(expand)), vals.map(expand));
  });

  // Compile a parsed expression to Javascript. This produces an anonymous
  // function of two arguments, env (the top-level environment) and set (a
  // function to set values in the environment)
  sss.compile = function (x) {
    return new Function("env", "get", "set", "symbols",
        "return $0;".fmt(sss.to_js(x, "env")));
  };

  sss.parse = function (tokens) {
    return expand(read(tokens), true);
  };

  function check(x, p, message) {
    if (!p) {
      throw "Syntax error in $0: $1".fmt(sss.to_sexp(x),
          message || "wrong length");
    }
    return x;
  }

  function expand(x, toplevel) {
    if (!Array.isArray(x)) {
      return x;
    }
    check(x, x.length > 0);
    if (x[0] === quotes["'"]) {
      return check(x, x.length === 2);
    } else if (x[0] === quotes["`"]) {
      check(x, x.length === 2);
      return expand_quasiquote(x[1]);
    } else if (x[0] === s_if) {
      check(x, x.length === 3 || x.length === 4);
      return x.map(expand);
    } else if (x[0] === s_set) {
      check(x, x.length === 3);
      check(x, is_symbol(x[1]), "can only set! a symbol");
      return [s_set, x[1], expand(x[2])];
    } else if (x[0] === s_define || x[0] === s_define_macro) {
      check(x, x.length >= 3);
      var v = x[1];
      if (Array.isArray(v)) {
        var lambda = [s_lambda, v.slice(1)].concat(x.slice(2));
        return expand([s_define, v[0], lambda]);
      } else {
        check(x, x.length === 3);
        check(x, is_symbol(v), "can only define a symbol");
        var exp = expand(x[2]);
        if (x[0] === s_define_macro) {
          check(x, toplevel, "define-macro is only allowed at top level");
          var f = sss.compile(exp)(sss.env, sss.get, sss.set, sss.symbols);
          check(x, typeof f === "function", "macro must be a function");
          macros[v.symbol] = f;
          return;
        }
        return [s_define, v, exp];
      }
    } else if (x[0] === s_begin) {<commit_msg>Simple test added to let<commit_after>// A version of <NAME>'s lis.py (http://norvig.com/lispy.html) and
// lispy.py (http://norvig.com/lispy2.html) that compiles to Javascript

(function (sss) {
  "use strict";

  // Perl-like string formatting (use $0, $1... in the string, referring to
  // values passed as parameters)
  String.prototype.fmt = function () {
    var args = arguments;
    return this.replace(/\$(\d+)/g, function (s, p) {
      return args[p] === undefined || args[p] === null ? "" : args[p];
    });
  }

  // Tokenizer
  sss.tokenize = function(s) {
    var tokens = [];
    while (true) {
      var m = s
        .match(/\s*(,@|[('`,)]|"(?:[\\].|[^\\"])*"|;.*|[^\s('"`,;)]*)(.*)/);
      if (m && m[1].length > 0 && m[1][0] !== ";") {
        tokens.push(m[1]);
        s = m[2];
      } else {
        return tokens;
      }
    }
  }

  var symbol = {
    toString: function () {
      return "$0".fmt(this.symbol);
    }
  };

  sss.symbols = {};

  // Symbols
  // Get a symbol for a name, creating it if necessary
  sss.get_symbol = function (s) {
    if (!sss.symbols.hasOwnProperty(s)) {
      sss.symbols[s] = Object.create(symbol);
      sss.symbols[s].symbol = s;
    }
    return sss.symbols[s];
  }

  var s_if = sss.get_symbol("if");
  var s_set = sss.get_symbol("set!");
  var s_define = sss.get_symbol("define");
  var s_define_macro = sss.get_symbol("define-macro");
  var s_lambda = sss.get_symbol("lambda");
  var s_begin = sss.get_symbol("begin");

  var quotes = {
    "'": sss.get_symbol("quote"),
    "`": sss.get_symbol("quasiquote"),
    ",": sss.get_symbol("unquote"),
    ",@": sss.get_symbol("unquotesplicing")
  };

  // Create an atom from a token value (#t/#f, string, number, symbol)
  function atom(token) {
    if (token === "#t") {
      return true;
    } else if (token === "#f") {
      return false;
    } else if (token[0] === '"') {
      return JSON.parse(token);
    } else {
      var n = parseFloat(token);
      return isNaN(n) ? sss.get_symbol(token) : n;
    }
  }

  function cons(x, xs) {
    return [x].concat(xs);
  }

  function is_pair(x) {
    return Array.isArray(x) && x.length !== 0;
  }

  var macros = {};

  function add_macro(name, f) {
    var symbol = sss.get_symbol(name);
    macros[name] = f;
  }

  add_macro("let", function () {
    var args = Array.prototype.slice.call(arguments);
    var x = cons(sss.symbols["let"], args);
    check(x, args.length > 1);
    var bindings = args[0];
    var body = args.slice(1);
    check(x, Array.isArray(bindings) && bindings.every(function (b) {
      return Array.isArray(b) && b.length === 2 && is_symbol(b[0]);
    }), "illegal binding list");
    var vars = bindings.map(function (b) { return b[0]; });
    var vals = bindings.map(function (b) { return b[1]; });
    return cons([s_lambda, vars].concat(body.map(expand)), vals.map(expand));
  });

  // Compile a parsed expression to Javascript. This produces an anonymous
  // function of two arguments, env (the top-level environment) and set (a
  // function to set values in the environment)
  sss.compile = function (x) {
    return new Function("env", "get", "set", "symbols",
        "return $0;".fmt(sss.to_js(x, "env")));
  };

  sss.parse = function (tokens) {
    return expand(read(tokens), true);
  };

  function check(x, p, message) {
    if (!p) {
      throw "Syntax error in $0: $1".fmt(sss.to_sexp(x),
          message || "wrong length");
    }
    return x;
  }

  function expand(x, toplevel) {
    if (!Array.isArray(x)) {
      return x;
    }
    check(x, x.length > 0);
    if (x[0] === quotes["'"]) {
      return check(x, x.length === 2);
    } else if (x[0] === quotes["`"]) {
      check(x, x.length === 2);
      return expand_quasiquote(x[1]);
    } else if (x[0] === s_if) {
      check(x, x.length === 3 || x.length === 4);
      return x.map(expand);
    } else if (x[0] === s_set) {
      check(x, x.length === 3);
      check(x, is_symbol(x[1]), "can only set! a symbol");
      return [s_set, x[1], expand(x[2])];
    } else if (x[0] === s_define || x[0] === s_define_macro) {
      check(x, x.length >= 3);
      var v = x[1];
      if (Array.isArray(v)) {
        var lambda = [s_lambda, v.slice(1)].concat(x.slice(2));
        return expand([s_define, v[0], lambda]);
      } else {
        check(x, x.length === 3);
        check(x, is_symbol(v), "can only define a symbol");
        var exp = expand(x[2]);
        if (x[0] === s_define_macro) {
          check(x, toplevel, "define-macro is only allowed at top level");
          console.log("expand macro >>> ", sss.to_js(exp, "env"));
          var f = sss.compile(exp)(sss.env, sss.get, sss.set, sss.symbols);
          check(x, typeof f === "function", "macro must be a function");
          macros[v.symbol] = f;
          return;
        }
        return [s_define, v, exp];
      }
    } else if (x[0] === s_begin) {
{'fraction_non_alphanumeric': 0.19723682853902463, 'fraction_numerical': 0.009691720795958347, 'mean_word_length': 2.529839883551674, 'pattern_counts': {'":': 8, '<': 5, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4753819', 'n_tokens_mistral': 3627, 'n_tokens_neox': 3519, 'n_words': 847}