<commit_before>var http = require('http');
var https = require('https');
var util = require('util');
var crypto = require('crypto');
var couch = require('./couch.js');
var timers = require('timers');

process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0"

function getHttpAndHttps(url, cb) {
	if(/^https/.test(url)) {
		https.get(url, cb).on('error', function(e) {
			util.puts(JSON.stringify(e));
			util.puts(url);
		});
	} else {
		http.get(url, cb).on('error', function(e) {
			util.puts(JSON.stringify(e));
			util.puts(url);
		});
	}
}

function putToCouch() {
	var spaces = http.get('http://localhost:5984/spaces/_design/all/_view/json', function(res) {
		var data = "";
		res.on('data', function(chunk){
			data = data + chunk;
		});
		res.on('end', function() {
			spaces = JSON.parse(data).rows;
			util.puts(JSON.stringify(spaces));
			getListItems(spaces);
		});
	});
}

function getListItems(spaces) {
	var current = spaces.pop();
	
	if(!current)
		return;

	getHttpAndHttps(current.key, function(res) {
		data = "";
		res.on('data', function(chunk){
			data = data + chunk;
		});
		res.on('end', function() {
			try {
				data = JSON.parse(data);
				couch.update(data, "/" + current.id + "/", function(res) {
					res.on('data', util.puts);
					res.on('end', function() {
						getListItems(spaces);
					});
				});
			} catch (e) {
				util.puts(current.key);
				getListItems(spaces);
			}
		});
	});
};

function hash(str) {
	var hash = crypto.createHash('sha256');
	hash.update("" + str);
	return hash.digest('hex');
};

putToCouch();
timers.setInterval(putToCouch, 1000 * 60 * 10);<commit_msg>put only relevant info to couch<commit_after>var http = require('http');
var https = require('https');
var util = require('util');
var crypto = require('crypto');
var couch = require('./couch.js');
var timers = require('timers');

process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0"

function getHttpAndHttps(url, cb) {
	if(/^https/.test(url)) {
		https.get(url, cb).on('error', function(e) {
			util.puts(JSON.stringify(e));
			util.puts(url);
		});
	} else {
		http.get(url, cb).on('error', function(e) {
			util.puts(JSON.stringify(e));
			util.puts(url);
		});
	}
}

function putToCouch() {
	var spaces = http.get('http://localhost:5984/spaces/_design/all/_view/json', function(res) {
		var data = "";
		res.on('data', function(chunk){
			data = data + chunk;
		});
		res.on('end', function() {
			spaces = JSON.parse(data).rows;
			util.puts(JSON.stringify(spaces));
			getListItems(spaces);
		});
	});
}

function getListItems(spaces) {
	var current = spaces.pop();
	
	if(!current)
		return;

	getHttpAndHttps(current.key, function(res) {
		data = "";
		res.on('data', function(chunk){
			data = data + chunk;
		});
		res.on('end', function() {
			try {
				data = JSON.parse(data);
				util.puts(util.inspect(data));
				var spacedate = {
					name : data.name,
					open : false,
					lastchange : (new Date()).getTime() / 1000,
				}
				if(data.lastchange)
					spacedate.lastchange = data.lastchange;
				if(data.open) {
					spacedate.open = true;
				}
				if(data.state) {
					if(data.state.lastchange) {
						spacedate.lastchange = data.state.lastchange;
					}
					if(data.state.open) {
						spacedate.open = true;
					}
				}
				couch.update(data, "/" + current.id + "/", function(res) {
					res.on('data', util.puts);
					res.on('end', function() {
						getListItems(spaces);
					});
				});
			} catch (e) {
				util.puts(current.key);
				getListItems(spaces);
			}
		});
	});
};

function hash(str) {
	var hash = crypto.createHash('sha256');
	hash.update("" + str);
	return hash.digest('hex');
};

putToCouch();
timers.setInterval(putToCouch, 1000 * 60 * 10);
{'fraction_non_alphanumeric': 0.19794205253181696, 'fraction_numerical': 0.00974817221770918, 'mean_word_length': 3.434573829531813, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1635774', 'n_tokens_mistral': 1622, 'n_tokens_neox': 1360, 'n_words': 228}