<commit_before>'use strict';

var redis      = require('redis')
  , xtend      = require('xtend')
  , hyperquest = require('hyperquest')
  ;

module.exports = FetchAndCache;

function FetchAndCache(opts) {
  if (!(this instanceof FetchAndCache)) return new FetchAndCache(opts);

  var redisOpts = xtend({ port: 6379, host: '127.0.0.1' }, opts.redis)
    , serviceOpts = xtend({ url: 'http://127.0.0.1' }, opts.service)

  this._defaultExpire = opts.defaultExpire || 15 * 60
  this._serviceOpts   = serviceOpts;
  this._markCached    = opts.markCached !== false;
  this._client        = redis.createClient(redisOpts.port, redisOpts.host, redisOpts)
}

var proto = FetchAndCache.prototype;

proto.fetch = function (uri, opts, cb) {
  var self = this;
  if (!self._client) return cb(new Error('fetchncache was stopped and can no longer be used to fetch data'));

  if (typeof opts === 'function') {
    cb = opts;
    opts = null;
  }
  opts = xtend({ expire: self._defaultExpire, transform: function (x) { return x } }, opts);

  self._client.get(uri, function (err, res) {
    if (err) return cb(err);
    if (res) return cb(null, res, true);  

    self._get(uri, function (err, res) {
      if (err) return cb(err);
      self._cache(uri, res, opts, cb);
    })
  });
}

proto.stop = function (force) {
  if (!this._client) throw new Error('fetchncache was stopped previously and cannot be stopped again');
  if (force || typeof this._client.unref !== 'function') this._client.end(); else this._client.unref();  
  this._client = null;
}

proto.monitor = function (monitorfn, cb) {
  if (!this._client) return cb(new Error('fetchncache was stopped and can no longer be used to monitor data'));

  this._client.monitor(function (err) {<commit_msg>forcing javascript parser for node 0.8<commit_after>'use strict';

var redis      = require('redis')
  , xtend      = require('xtend')
  , hyperquest = require('hyperquest')
  ;

module.exports = FetchAndCache;

function FetchAndCache(opts) {
  if (!(this instanceof FetchAndCache)) return new FetchAndCache(opts);

  var redisOpts = xtend({ port: 6379, host: '127.0.0.1' }, opts.redis)
    , serviceOpts = xtend({ url: 'http://127.0.0.1' }, opts.service)

  // node 0.8 has problems with hiredis
  if (/^v0\.8/.test(process.version)) redisOpts.parser = redisOpts.parser || 'javascript';

  this._defaultExpire = opts.defaultExpire || 15 * 60
  this._serviceOpts   = serviceOpts;
  this._markCached    = opts.markCached !== false;
  this._client        = redis.createClient(redisOpts.port, redisOpts.host, redisOpts)
}

var proto = FetchAndCache.prototype;

proto.fetch = function (uri, opts, cb) {
  var self = this;
  if (!self._client) return cb(new Error('fetchncache was stopped and can no longer be used to fetch data'));

  if (typeof opts === 'function') {
    cb = opts;
    opts = null;
  }
  opts = xtend({ expire: self._defaultExpire, transform: function (x) { return x } }, opts);

  self._client.get(uri, function (err, res) {
    if (err) return cb(err);
    if (res) return cb(null, res, true);  

    self._get(uri, function (err, res) {
      if (err) return cb(err);
      self._cache(uri, res, opts, cb);
    })
  });
}

proto.stop = function (force) {
  if (!this._client) throw new Error('fetchncache was stopped previously and cannot be stopped again');
  if (force) this._client.end(); else this._client.unref();  
  this._client = null;
}

proto.monitor = function (monitorfn, cb) {
  if (!this._client) return cb(new Error('fetchncache was stopped and can no longer be used to monitor data'));

  this._client.monitor(function (err) {
{'fraction_non_alphanumeric': 0.1447186027169393, 'fraction_numerical': 0.01275298031605212, 'mean_word_length': 3.6375321336760926, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7668212', 'n_tokens_mistral': 1234, 'n_tokens_neox': 1199, 'n_words': 345}