<commit_before>// this runs on all vikings with an etcd client to listen for events
var etcdjs = require('etcdjs')
var config = require('./config')()
var utils = require('component-consoler');
var HostMonitor = require('./hostmonitor')

module.exports = function(etcdhosts){
	var etcd = etcdjs(etcdhosts)

	var vikingHostname = config.network.hostname

	var hostMonitor = HostMonitor(config, etcd)

	var leaderRunning = false

	// this will be trigged in another viking master if the etcd leadership changes
	function runLeader(){
		if(leaderRunning) return
		utils.log('leader', 'starting')
		leaderRunning = true

		hostMonitor.leader(function(){
			// the leader module has all the servies like registry and git push
			var leader = require('./leader')(config, etcd)
			leader.start(function(err){
				if(err){
					console.error('ERROR STARTING LEADER ' + err)
					return
				}
				
				utils.log('leader', 'running')
			})
		})
	}

	function runSlave(){
		var slave = require('./slave')(config, etcd)
		slave.start(function(err){
			if(err){
				utils.error('ERROR STARTING SLAVE ' + err)
				return
			}
			
			utils.log('slave', 'running')
		})
	}

	function masterLoop(){
		etcd.stats.leader(function(err, stats){
			if(err){
				//utils.error(err)
				setTimeout(masterLoop, 200)
				return
			}
			if(!stats){
				utils.error('no stats were returned from the etcd server')
				setTimeout(masterLoop, 200)
				return
			}
			console.log('leader stats')
			console.dir(stats)
			if(stats.leader==vikingHostname){
				runLeader()
			}
			setTimeout(masterLoop, 2000)
		})
	}

	setTimeout(function(){

		hostMonitor.start(function(){
			utils.log('host monitor', 'running')
		})

		setTimeout(function(){
			
			if(config.master){
				masterLoop()	
			}

			runSlave()

		}, 500)
	}, 500)
	
}
<commit_msg>remove logging each host hit<commit_after>// this runs on all vikings with an etcd client to listen for events
var etcdjs = require('etcdjs')
var config = require('./config')()
var utils = require('component-consoler');
var HostMonitor = require('./hostmonitor')

module.exports = function(etcdhosts){
	var etcd = etcdjs(etcdhosts)

	var vikingHostname = config.network.hostname

	var hostMonitor = HostMonitor(config, etcd)

	var leaderRunning = false

	// this will be trigged in another viking master if the etcd leadership changes
	function runLeader(){
		if(leaderRunning) return
		utils.log('leader', 'starting')
		leaderRunning = true

		hostMonitor.leader(function(){
			// the leader module has all the servies like registry and git push
			var leader = require('./leader')(config, etcd)
			leader.start(function(err){
				if(err){
					console.error('ERROR STARTING LEADER ' + err)
					return
				}
				
				utils.log('leader', 'running')
			})
		})
	}

	function runSlave(){
		var slave = require('./slave')(config, etcd)
		slave.start(function(err){
			if(err){
				utils.error('ERROR STARTING SLAVE ' + err)
				return
			}
			
			utils.log('slave', 'running')
		})
	}

	function masterLoop(){
		etcd.stats.leader(function(err, stats){
			if(err){
				//utils.error(err)
				setTimeout(masterLoop, 200)
				return
			}
			if(!stats){
				utils.error('no stats were returned from the etcd server')
				setTimeout(masterLoop, 200)
				return
			}
			
			if(stats.leader==vikingHostname){
				runLeader()
			}
			setTimeout(masterLoop, 2000)
		})
	}

	setTimeout(function(){

		hostMonitor.start(function(){
			utils.log('host monitor', 'running')
		})

		setTimeout(function(){
			
			if(config.master){
				masterLoop()	
			}

			runSlave()

		}, 500)
	}, 500)
	
}

{'fraction_non_alphanumeric': 0.13481894150417828, 'fraction_numerical': 0.008913649025069638, 'mean_word_length': 3.4224137931034484, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2517319', 'n_tokens_mistral': 1449, 'n_tokens_neox': 1144, 'n_words': 285}