<commit_before>var test = require('tape');
var xtend = require('xtend');
var srcToTokens = require('./src-to-tokens');
var tokensToAST = require('./tokens-to-ast');
var astToTarget = require('./ast-to-target');
var simStreamWrites = require('./simulate-stream-writes');

var setup = function(src, callback){
  var target_parts = [];

  var s = srcToTokens();
  var a = tokensToAST();
  var e = astToTarget();
  var defMacro = function(name, fn){
    e.macros[name] = function(ast, astToTarget){
      return astToTarget(fn(ast));
    };
  };
  var defTargetMacro = function(name, fn){
    e.macros[name] = fn;
  };

  defTargetMacro('$$es-no$$top-level-expression', function(ast, astToTarget){
    return astToTarget(ast);
  });
  defTargetMacro('$$es-no$$fn-call', function(ast, astToTarget){
    var args = [];
    var i;
    for(i=2; i<ast.value.length; i++){
      args.push(astToTarget(ast.value[i]));
    }
    return {fn_name: ast.value[1].value, args: args};
  });
  defTargetMacro('$$es-no$$make-type-string', function(ast, astToTarget){
    return {string: ast.value};
  });
  defTargetMacro('$$es-no$$make-type-number', function(ast, astToTarget){
    return {number: parseFloat(ast.value)};
  });
  defTargetMacro('list', function(ast, astToTarget){
    var vals = [];
    var i;
    for(i=1; i<ast.value.length; i++){
      vals.push(astToTarget(ast.value[i]));
    }
    return {list: vals};
  });
  defMacro('surround', function(ast){
    return xtend({}, ast, {
      type: 'list',
      value: [
        xtend({}, ast, {type: 'symbol', value: 'list'}),
        xtend({}, ast, {type: 'number', value: '1'}),
        ast.value[1],
        xtend({}, ast, {type: 'number', value: '3'})
      ]
    });
  });
  defMacro('surround-these', function(ast){
    return xtend({}, ast, {
      type: 'list',
      value: [
        xtend({}, ast, {type: 'symbol', value: 'list'}),
        xtend({}, ast, {
          type: 'list',
          value: [
            xtend({}, ast, {type: 'symbol', value: 'surround'}),
            ast.value[1]
          ]
        }),
        xtend({}, ast, {
          type: 'list',
          value: [
            xtend({}, ast, {type: 'symbol', value: 'surround'}),
            ast.value[2]<commit_msg>cleanup<commit_after>var test = require('tape');
var xtend = require('xtend');
var srcToTokens = require('./src-to-tokens');
var tokensToAST = require('./tokens-to-ast');
var astToTarget = require('./ast-to-target');
var simStreamWrites = require('./simulate-stream-writes');

var setup = function(src, callback){
  var target_parts = [];

  var s = srcToTokens();
  var a = tokensToAST();
  var e = astToTarget();
  var defMacro = function(name, fn){
    e.macros[name] = function(ast, astToTarget){
      return astToTarget(fn(ast));
    };
  };
  var defTargetMacro = function(name, fn){
    e.macros[name] = fn;
  };

  defTargetMacro('$$es-no$$top-level-expression', function(ast, astToTarget){
    return astToTarget(ast);
  });
  defTargetMacro('$$es-no$$fn-call', function(ast, astToTarget){
    return {fn_name: ast.value[1].value, args: ast.value.slice(2).map(astToTarget)};
  });
  defTargetMacro('$$es-no$$make-type-string', function(ast, astToTarget){
    return {string: ast.value};
  });
  defTargetMacro('$$es-no$$make-type-number', function(ast, astToTarget){
    return {number: parseFloat(ast.value)};
  });
  defTargetMacro('list', function(ast, astToTarget){
    return {list: ast.value.slice(1).map(astToTarget)};
  });
  defMacro('surround', function(ast){
    return xtend({}, ast, {
      type: 'list',
      value: [
        xtend({}, ast, {type: 'symbol', value: 'list'}),
        xtend({}, ast, {type: 'number', value: '1'}),
        ast.value[1],
        xtend({}, ast, {type: 'number', value: '3'})
      ]
    });
  });
  defMacro('surround-these', function(ast){
    return xtend({}, ast, {
      type: 'list',
      value: [
        xtend({}, ast, {type: 'symbol', value: 'list'}),
        xtend({}, ast, {
          type: 'list',
          value: [
            xtend({}, ast, {type: 'symbol', value: 'surround'}),
            ast.value[1]
          ]
        }),
        xtend({}, ast, {
          type: 'list',
          value: [
            xtend({}, ast, {type: 'symbol', value: 'surround'}),
            ast.value[2]
{'fraction_non_alphanumeric': 0.20186480186480185, 'fraction_numerical': 0.0037296037296037296, 'mean_word_length': 2.9403122130394856, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5177321', 'n_tokens_mistral': 1493, 'n_tokens_neox': 1489, 'n_words': 303}