<commit_before>// Download a webpage and cache it locally for future retrieval
// moduled after the Github.com/UnitedStates utils.py download function

var request = require('request'),
	fs = require('fs'),
	mkdirp = require('mkdirp');

var downcache = module.exports = exports;

var isVerbose = false;

// try to find a page in cache. If not found, retrieve and cache
module.exports = function(url, opts, callback) {
	if (arguments.length < 3) {
		callback = opts;
		opts = {};
	}

	if (opts.dir) {
		if (opts.dir.slice(-1) != "/") {
			opts.dir += "/";
		}
	} else {
		opts.dir = "";
	}

	opts.dir += "cache/";

	announce("directory for cache is " + opts.dir)

	var dir = opts.dir + url.split("/").slice(2, -1).join("/"),
		path = opts.dir + url.split("/").slice(2).join("/");

	if (path === dir + "/") {
		path += "index.html";
	}

	fs.readFile(path, {encoding: "utf-8" }, function(err, body) {
		if (err) {
			announce("Couldn't find " + url + " in cache. Calling live.");
			request(url, function(err, resp, body) {
				if (err) {
					console.log("Error retrieving", url);
					return null;
				};<commit_msg>Fixed bug with cache retrieval<commit_after>// Download a webpage and cache it locally for future retrieval
// moduled after the Github.com/UnitedStates utils.py download function

var request = require('request'),
	fs = require('fs'),
	mkdirp = require('mkdirp');

var downcache = module.exports = exports;

var isVerbose = false;

// try to find a page in cache. If not found, retrieve and cache
module.exports = function(url, opts, callback) {
	if (arguments.length < 3) {
		callback = opts;
		opts = {};
	}

	if (!callback) {
		callback = function() {};
	}

	if (opts.dir) {
		if (opts.dir.slice(-1) != "/") {
			opts.dir += "/";
		}
	} else {
		opts.dir = "";
	}

	opts.dir += "cache/";

	announce("directory for cache is " + opts.dir)

	var dir = opts.dir + url.split("/").slice(2, -1).join("/"),
		path = opts.dir + url.split("/").slice(2).join("/");

	if (path === dir + "/") {
		path += "index.html";
	}

	fs.readFile(path, {encoding: "utf-8" }, function(err, body) {
		if (err) {
			announce("Couldn't find " + url + " in cache. Calling live.");
			request(url, function(err, resp, body) {
				if (err) {
					console.log("Error retrieving", url);
					return null;
				};
{'fraction_non_alphanumeric': 0.18676534610361342, 'fraction_numerical': 0.00522420548541576, 'mean_word_length': 3.1630434782608696, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6373969', 'n_tokens_mistral': 871, 'n_tokens_neox': 783, 'n_words': 222}