<commit_before>var sax =  require("sax"),
	readability = require("../readabilitySAX"),
	url = require("url");

exports.get = function(uri, cb){
	if(!uri || uri.trim() === ""){cb({
			title:	"Error",
	    	text:	"No URI specified!",
	    	html:	"<b>No URI specified!</b>",
	    	error: true
	    });
	    return;
	}
	
	var link = url.parse(uri),
		client = require("http").createClient(80, link.host),
		req = client.request("GET", link.href, { host: link.host, accept:"text/html" });
	
	client.addListener("error", function(err) {
		cb({
			title:	"Error",
	    	text:	"Couldn't find uri!",
	    	html:	"<b>Couldn't find uri!</b>",
	    	error: true
	    });
	});
	
	req.addListener("response", function(connection){
		if(connection.statusCode === 301 || connection.statusCode === 302)
			return exports.get(connection.headers.location, cb);
		var parser = sax.parser(false, {
		    lowercasetags : true
		});
		var data = "";
		
		var readable = new readability.process(parser, {
			convertLinks: function(a){
				return url.resolve(link, a);
			},
			pageURL: url.format(link)
		});
		
		connection.addListener("data", function(chunk){
			parser.write(chunk.toString("utf8"));
			data += chunk.toString("utf8");
		});
		connection.addListener("end", function(){
			parser.close();
			var ret = readable.getArticle();
			if(ret.textLength < 250){
				ret = exports.process(data, 1, {
					convertLinks: function(a){
						return url.resolve(link, a);
					},
					pageURL: url.format(link)
				});	
			}
			ret.link = link.href;
			cb(ret);
		});
	});
	req.end();
};
exports.process = function(data, skipLevel, readabilitySettings, type){
	skipLevel = skipLevel || 0;
	readabilitySettings = readabilitySettings || {};
	
	var contentLength = 0,
		parser, readable, ret;
	
	while(contentLength < 250 && skipLevel < 4){
	    parser = sax.parser(false, {
		    lowercasetags : true
		});
	    
	    readabilitySettings.skipLevel = skipLevel;
	    <commit_msg>just added caching in getReadableContent when receiving & writing data<commit_after>var sax =  require("sax"),
	readability = require("../readabilitySAX"),
	url = require("url");

exports.get = function(uri, cb){
	if(!uri || uri.trim() === ""){cb({
			title:	"Error",
	    	text:	"No URI specified!",
	    	html:	"<b>No URI specified!</b>",
	    	error: true
	    });
	    return;
	}
	
	var link = url.parse(uri),
		client = require("http").createClient(80, link.host),
		req = client.request("GET", link.href, { host: link.host, accept:"text/html" });
	
	client.addListener("error", function(err) {
		cb({
			title:	"Error",
	    	text:	"Couldn't find uri!",
	    	html:	"<b>Couldn't find uri!</b>",
	    	error: true
	    });
	});
	
	req.addListener("response", function(connection){
		if(connection.statusCode === 301 || connection.statusCode === 302)
			return exports.get(connection.headers.location, cb);
		var parser = sax.parser(false, {
		    lowercasetags : true
		});
		var data = "";
		
		var readable = new readability.process(parser, {
			convertLinks: function(a){
				return url.resolve(link, a);
			},
			pageURL: url.format(link)
		});
		
		connection.on("data", function(chunk){
			data += chunk = chunk.toString("utf8");
			parser.write(chunk);
		});
		connection.on("end", function(){
			parser.close();
			var ret = readable.getArticle();
			if(ret.textLength < 250){
				ret = exports.process(data, 1, {
					convertLinks: function(a){
						return url.resolve(link, a);
					},
					pageURL: url.format(link)
				});	
			}
			ret.link = link.href;
			cb(ret);
		});
	});
	req.end();
};
exports.process = function(data, skipLevel, readabilitySettings, type){
	skipLevel = skipLevel || 0;
	readabilitySettings = readabilitySettings || {};
	
	var contentLength = 0,
		parser, readable, ret;
	
	while(contentLength < 250 && skipLevel < 4){
	    parser = sax.parser(false, {
		    lowercasetags : true
		});
	    
	    readabilitySettings.skipLevel = skipLevel;
	    
{'fraction_non_alphanumeric': 0.1732623033992897, 'fraction_numerical': 0.00989345509893455, 'mean_word_length': 3.2905331882480957, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7570802', 'n_tokens_mistral': 1545, 'n_tokens_neox': 1287, 'n_words': 275}