<commit_before>var fs = require('fs')
  , http = require('http')

function getBase64 (url, fn) {
  var m = /^(https?):\/\/([^:\/]+)(?::(\d+))?([^:]*)$/.exec(url);
  if (m !== null) {
    http.get({host:m[2], port:parseInt(m[3]||80), path:m[4]}, function (res) {
      var buf = '';
      res.on('data', function (data) { buf += data; });
      res.on('end', function () {
        fn(new Buffer(buf).toString('base64'));
      });
    });
  } else {
    fs.readFile(url, function (err, data) {
      fn(data.toString('base64'));
    });
  }
}

function imgbase (sIn, sOut) {
  var buf = '';
  sIn.resume();
  sIn.on('data', function (data) {
    var re = /url\(["']?([^\)]*\.(png|gif|jpeg|jpg))["']?\)/g;
    buf += data;
    var parse = function (start) {
      var m = re.exec(buf);
      if (m !== null) {
        sIn.pause();
        var index = m.index
          , len = m[0].length
          , url = m[1]
          , type = m[2];
        getBase64(url, function (img) {
          sOut.write(buf.slice(start, index)
            +'url(data:image/'+type+';base64,'+img+')');
          parse(index + len);
        });
      } else {
        buf = buf.slice(start);
        sIn.resume();
      }
    }
    parse(0);
  });
  sIn.on('end', function () {
    sOut.write(buf);
  });
}

module.exports = imgbase;

if (!module.parent) {
  imgbase(process.stdin, process.stdout);
}<commit_msg>Added support for in file as parameter + relative and base path<commit_after>var fs = require('fs')
  , http = require('http');

function getBase64 (url, fn, opt) {
  var m = /^(https?):\/\/([^:\/]+)(?::(\d+))?([^:]*)$/.exec(url);
  if (opt === undefined) opt = {};
  if (m !== null) {
    http.get({host:m[2], port:parseInt(m[3]||80), path:m[4]}, function (res) {
      var buf = ''
      res.setEncoding('binary');
      res.on('data', function (data) { buf += data; });
      res.on('end', function () {
        fn(new Buffer(buf, 'binary').toString('base64'));
      });
    });
  } else {
    if (url[0] === '/') {
      url = (opt.base ? opt.base+'/' : '')+url.slice(1);
    } else {
      url = (opt.rel ? opt.rel+'/' : '')+url;
    }
    fs.readFile(url, function (err, data) {
      fn(data.toString('base64'));
    });
  }
}

function imgbase (sIn, sOut, opt) {
  var buf = '';
  sIn.resume();
  sIn.on('data', function (data) {
    var re = /url\(["']?\s*([^\)]*\.(png|gif|jpeg|jpg))\s*["']?\)/g;
    buf += data;
    var parse = function (start) {
      var m = re.exec(buf);
      if (m !== null) {
        sIn.pause();
        var index = m.index
          , len = m[0].length
          , url = m[1]
          , type = m[2];
        getBase64(url, function (img) {
          sOut.write(buf.slice(start, index)
            +'url(data:image/'+type+';base64,'+img+')');
          parse(index + len);
        }, opt);
      } else {
        buf = buf.slice(start);
        sIn.resume();
      }
    }
    parse(0);
  });
  sIn.on('end', function () {
    sOut.write(buf);
  });
}

module.exports = imgbase;

if (!module.parent) {
  var file = process.argv[2];
  if (file !== undefined) {
    var path = require('path');
    imgbase(fs.createReadStream(file), process.stdout, {
      rel: path.dirname(file)
    });
  } else {
    imgbase(process.stdin, process.stdout);
  }
}
{'fraction_non_alphanumeric': 0.23601345154387038, 'fraction_numerical': 0.012534393151941303, 'mean_word_length': 2.255721393034826, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3244601', 'n_tokens_mistral': 1235, 'n_tokens_neox': 1220, 'n_words': 232}