<commit_before>var babel = require("babel-core");
var path  = require("path");
var url   = require("url");
var fs    = require("fs");
var _     = require("lodash");

module.exports = function (opts) {
  opts = _.defaults(opts || {}, {
    options: {},
    dest:    "cache",
    src:     "assets"
  });

  var cache = Object.create(null);

  return function (req, res, next) {
    if (!babel.canCompile(req.url)) return next();

    var pathname = path.normalize(url.parse(req.url).pathname);
    var dest = path.join(opts.dest, pathname);
    var src  = path.join(opts.src, pathname);
    var srcStat;

    var send = function (data) {
      res.set('Content-Type', 'application/javascript');
      res.end(data);
    };

    var write = function (transformed) {
      fs.writeFile(dest, transformed, function (err) {
        if (err) {
          next(err);
        } else {
          cache[pathname] = +srcStat.mtime;
          send(transformed);
        }
      });
    };

    var compile = function () {
      var transformOpts = _.clone(opts.options);
      babel.transformFile(src, transformOpts, function (err, result) {
        if (err) {
          next(err);
        } else {
          write(result.code);
        }
      });
    };

    var tryCache = function () {
      fs.readFile(dest, function (err, data) {
        if (err && err.code === 'ENOENT') {
          compile();
        } else if (err) {
          next(err);
        } else {
          send(data);
        }
      });
    };

    fs.stat(src, function (err, stat) {
      srcStat = stat;
      if (err && err.code === 'ENOENT') {
        next();
      } else if (err) {
        next(err);
      } else if (cache[pathname] === +stat.mtime) {
        tryCache();
      } else {<commit_msg>Transform all file urls to babel:///foo/bar.js for easier usage<commit_after>var babel = require("babel-core");
var path  = require("path");
var url   = require("url");
var fs    = require("fs");
var _     = require("lodash");

module.exports = function (opts) {
  opts = _.defaults(opts || {}, {
    options: {},
    dest:    "cache",
    src:     "assets"
  });

  var cache = Object.create(null);

  return function (req, res, next) {
    if (!babel.canCompile(req.url)) return next();

    var pathname = path.normalize(url.parse(req.url).pathname);
    var dest = path.join(opts.dest, pathname);
    var src  = path.join(opts.src, pathname);
    var srcStat;

    var send = function (data) {
      res.set('Content-Type', 'application/javascript');
      res.end(data);
    };

    var write = function (transformed) {
      fs.writeFile(dest, transformed, function (err) {
        if (err) {
          next(err);
        } else {
          cache[pathname] = +srcStat.mtime;
          send(transformed);
        }
      });
    };

    var compile = function () {
      var transformOpts = _.clone(opts.options);
      transformOpts.sourceFileName = 'babel:///' + src.replace(opts.src, '');
      babel.transformFile(src, transformOpts, function (err, result) {
        if (err) {
          next(err);
        } else {
          write(result.code);
        }
      });
    };

    var tryCache = function () {
      fs.readFile(dest, function (err, data) {
        if (err && err.code === 'ENOENT') {
          compile();
        } else if (err) {
          next(err);
        } else {
          send(data);
        }
      });
    };

    fs.stat(src, function (err, stat) {
      srcStat = stat;
      if (err && err.code === 'ENOENT') {
        next();
      } else if (err) {
        next(err);
      } else if (cache[pathname] === +stat.mtime) {
        tryCache();
      } else {
{'fraction_non_alphanumeric': 0.16338259441707717, 'fraction_numerical': 0.0, 'mean_word_length': 1.9959016393442623, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1604330', 'n_tokens_mistral': 1172, 'n_tokens_neox': 1145, 'n_words': 263}