<commit_before>var _ = require("lodash");
var e = require("estree-builder");
var parser = require("ecmaless-parser");
var compiler = require("ecmaless-compiler");
var escodegen = require("escodegen");

var toJSFile = function(estree, opts){
  opts = opts || {};

  if(!_.isArray(estree)){
    estree = [estree];
  }
  return escodegen.generate({
    "loc": _.size(estree) > 0 ? _.head(estree).loc : undefined,
    "type": "Program",
    "body": estree
  }, opts.escodegen);
};

var extractDeps = function(ast){
  var deps = {};
  var new_ast = ast;
  if(ast[0] && ast[0].type === "Dependencies"){
    _.each(ast[0].dependencies, function(d){
      deps[d.id.value] = d;
    });
    new_ast = ast.slice(1);
  }
  return {deps: deps, ast: new_ast};
};

module.exports = function(conf, callback){

  var modules = {};

  var load = function load(path, callback){
    if(_.has(modules, path)){
      return callback();
    }
    conf.loadPath(path, function(err, src){
      if(err)return callback(err);

      var p = extractDeps(parser(src, {filename: path}));
      var c = compiler(p.ast);

      modules[path] = {
        src: src,
        ast: p.ast,
        est: c.estree
      };

      if(_.isEmpty(p.deps)){
        return callback();
      }
      var done = (function(){
        var error;
        var after = _.after(_.size(p.deps), function(){
          callback(error);
        });
        return function(err){
          if(!error && err){
            error = err;
          }
          after();
        };
      }());
      _.each(p.deps, function(dep){
        load(dep.path.value, done);
      });
    });
  };

  load(conf.start_path, function(err){
    if(err)return callback(err);

    var mods = {};
    _.each(modules, function(m, path){
      mods[path] = e("array", [e("fn", [], m.est)]);
    });
    var est = e("call",
        e("fn", ["mdefs", "main"], []),
        [e("object", mods), e("string", conf.start_path)]);<commit_msg>proper compiling of the wrapped module functions<commit_after>var _ = require("lodash");
var e = require("estree-builder");
var parser = require("ecmaless-parser");
var compiler = require("ecmaless-compiler");
var escodegen = require("escodegen");

var toJSFile = function(estree, opts){
  opts = opts || {};

  if(!_.isArray(estree)){
    estree = [estree];
  }
  return escodegen.generate({
    "loc": _.size(estree) > 0 ? _.head(estree).loc : undefined,
    "type": "Program",
    "body": estree
  }, opts.escodegen);
};

var extractDeps = function(ast){
  var deps = {};
  var new_ast = ast;
  if(ast[0] && ast[0].type === "Dependencies"){
    _.each(ast[0].dependencies, function(d){
      deps[d.id.value] = d;
    });
    new_ast = ast.slice(1);
  }
  return {deps: deps, ast: new_ast};
};

module.exports = function(conf, callback){

  var modules = {};

  var load = function load(path, callback){
    if(_.has(modules, path)){
      return callback();
    }
    conf.loadPath(path, function(err, src){
      if(err)return callback(err);

      var ast = parser(src, {filename: path});
      var mloc;//TODO
      var mast = [
        {
          loc: mloc,
          type: "Function",
          params: [],
          block: {
            loc: mloc,
            type: "Block",
            body: []
          }
        }
      ];
      var deps = {};
      if(ast[0] && ast[0].type === "Dependencies"){
        _.each(ast[0].dependencies, function(d){
          deps[d.id.value] = d.path.value;
          mast[0].params.push(d.id)
        });
        mast[0].block.body = ast.slice(1);
      }else{
        mast[0].block.body = ast;
      }

      var c = compiler(mast);

      modules[path] = {
        src: src,
        ast: ast,
        est: c.estree[0],
        mast: mast,
        deps: deps,
      };

      if(_.isEmpty(deps)){
        return callback();
      }
      var done = (function(){
        var error;
        var after = _.after(_.size(deps), function(){
          callback(error);
        });
        return function(err){
          if(!error && err){
            error = err;
          }
          after();
        };
      }());
      _.each(deps, function(path){
        load(path, done);
      });
    });
  };

  load(conf.start_path, function(err){
    if(err)return callback(err);

    var toReqPath = function(path){
      return e("string", path);
    };

    var mods = {};
    _.each(modules, function(m, path){
      mods[path] = e("array", [m.est].concat(_.map(m.deps, toReqPath)));
    });
    var est = e("call",
        e("fn", ["mdefs", "main"], []),
        [e("object", mods), e("string", conf.start_path)]);
{'fraction_non_alphanumeric': 0.18507268387936646, 'fraction_numerical': 0.0039054024734215664, 'mean_word_length': 2.2170272156315423, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '992529', 'n_tokens_mistral': 1563, 'n_tokens_neox': 1543, 'n_words': 321}