<commit_before>var parseUrl = require('url').parse,
    parse = require('ua-parser2')().parse
  , zlib = require('zlib')
  , cache_manager = require('cache-manager')
  , mongo = require('mongodb')
  , MongoClient = require('mongodb').MongoClient
  , mongoUri = process.env.MONGOLAB_URI || process.env.MONGOHQ_URL || 'mongodb://localhost/prerender'
  , statusMatch = /<meta.*?name=['"]prerender-status-code['"] content=['"]([0-9]{3})['"].*?\/?>/i
  , database;

MongoClient.connect(mongoUri, function(err, db) {
  database = db;
});

function bytesToSize(bytes) {
  var k = 1000
    , sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB']
    , i;
   if (bytes === 0) return '0 Bytes';
   i = parseInt(Math.floor(Math.log(bytes) / Math.log(k)), 10);
   return (bytes / Math.pow(k, i)).toPrecision(3) + ' ' + sizes[i];
}

function getKeyFromUrl(url) {
  var parsed = parseUrl(url);
  return parsed.host + parsed.path;
}

var mongo_cache = {
  get: function(key, callback) {
    database.collection('pages', function(err, collection) {
      collection.findOne({ _id: key, status: 200 }, function (err, item) {
        if (item && item.value) {
          var buffer = new Buffer(item.value, 'base64');
          zlib.inflate(buffer, function (err, inflated) {
            callback(err, inflated.toString('utf8'));
          });
        } else {
          callback(err, null);
        }
      });
    });
  },
  set: function(key, value, info, callback) {
    database.collection('pages', function(err, collection) {
      var preSize = bytesToSize(Buffer.byteLength(value, 'utf8'));
      zlib.deflate(value, function (err, deflated) {
        var encoded = deflated.toString('base64');
        info._id = key;
        info.value = encoded;
        info.created = new Date();
        collection.save(info, { w: 1 }, function (err) {
          if (!err) {
            var postSize = bytesToSize(Buffer.byteLength(encoded, 'base64'));
            console.log('Cached | ' + key + ' | ' + preSize + ' > ' + postSize);
          }
        });
      });
    });
  }
};

module.exports = {
  init: function() {
    this.cache = cache_manager.caching({
        store: mongo_cache
    });
  },<commit_msg>add value only if 200 code<commit_after>var parseUrl = require('url').parse,
    parse = require('ua-parser2')().parse
  , zlib = require('zlib')
  , cache_manager = require('cache-manager')
  , mongo = require('mongodb')
  , MongoClient = require('mongodb').MongoClient
  , mongoUri = process.env.MONGOLAB_URI || process.env.MONGOHQ_URL || 'mongodb://localhost/prerender'
  , statusMatch = /<meta.*?name=['"]prerender-status-code['"] content=['"]([0-9]{3})['"].*?\/?>/i
  , database;

MongoClient.connect(mongoUri, function(err, db) {
  database = db;
});

function bytesToSize(bytes) {
  var k = 1000
    , sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB']
    , i;
   if (bytes === 0) return '0 Bytes';
   i = parseInt(Math.floor(Math.log(bytes) / Math.log(k)), 10);
   return (bytes / Math.pow(k, i)).toPrecision(3) + ' ' + sizes[i];
}

function getKeyFromUrl(url) {
  var parsed = parseUrl(url);
  return parsed.host + parsed.path;
}

var mongo_cache = {
  get: function(key, callback) {
    database.collection('pages', function(err, collection) {
      collection.findOne({ _id: key, status: 200 }, function (err, item) {
        if (item && item.value) {
          var buffer = new Buffer(item.value, 'base64');
          zlib.inflate(buffer, function (err, inflated) {
            callback(err, inflated.toString('utf8'));
          });
        } else {
          callback(err, null);
        }
      });
    });
  },
  set: function(key, value, info, callback) {
    database.collection('pages', function(err, collection) {
      var preSize = bytesToSize(Buffer.byteLength(value, 'utf8'));
      zlib.deflate(value, function (err, deflated) {
        var encoded = deflated.toString('base64');
        info._id = key;
        if (info.status == 200) {
          info.value = encoded;
        }
        info.created = new Date();
        collection.save(info, { w: 1 }, function (err) {
          if (!err) {
            var postSize = bytesToSize(Buffer.byteLength(encoded, 'base64'));
            console.log('Cached | ' + key + ' | ' + preSize + ' > ' + postSize);
          }
        });
      });
    });
  }
};

module.exports = {
  init: function() {
    this.cache = cache_manager.caching({
        store: mongo_cache
    });
  },
{'fraction_non_alphanumeric': 0.17982356932820628, 'fraction_numerical': 0.012666817462112644, 'mean_word_length': 2.7506361323155217, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '575062', 'n_tokens_mistral': 1518, 'n_tokens_neox': 1461, 'n_words': 292}