import time, os, threading, argparse
import ingest, process, sync

def main():
	"""Ingest, process, sync"""

	if not os.path.exists("data"):
		os.makedirs("data")

	parser = argparse.ArgumentParser(description='Ingest virus sequences')
	parser.add_argument('--headless', action='store_true', help='Run firefox in headless state (requires xvfb and x11vnc)', required=False)
	parser.add_argument('--clock', action='store_true', help='Run ntpdate to fix date', required=False)
	args = vars(parser.parse_args())
	headless = args['headless']
	clock = args['clock']

	ingest.main([headless])
	process.main()
	sync.main([clock])

if __name__ == "__main__":
	main()

#!/usr/bin/env python
import os, re, time, datetime, csv, sys, argparse, subprocess
import rethinkdb as r
from vdb_download import vdb_download
from Bio import SeqIO, AlignIO

def local_count(params):
	handle = open(params.path + params.fstem + "." + params.ftype, "rU")
	records = list(SeqIO.parse(handle, "fasta"))
	return len(records)

def build(params):
	'''
	run build for single virus / resolution combination
	'''
	vdb.download_all_documents()
	vdb.output()

	process = 'src/' + params.virus + '_process.py'
	call = map(str, [params.bin, process, '--resolution', params.resolution])
	print call
	subprocess.call(call)

def tick():
	print "tick"

if __name__=="__main__":
	parser = argparse.ArgumentParser(description = "download and process")
	parser.add_argument('--bin', type = str, default = "python")	
	parser.add_argument('-v', '--virus', default='Zika', help="virus table to interact with")
	parser.add_argument('-r', '--resolution', default='', help="build params")	
	parser.add_argument('--host', default=None, help="rethink host url")
	parser.add_argument('--auth_key', default=None, help="auth_key for rethink database")
	parser.add_argument('--build', action="store_true", default=False, help ="single rebuild")
	parser.add_argument('--watch', action="store_true", default=False, help ="watch database and rebuild when updated")	
	params = parser.parse_args()

	if params.virus is None:
		params.virus = 'Zika'

	if params.resolution is None:
		params.resolution = ''

	params.database = 'vdb'
	params.path = 'data/'
	params.ftype = 'fasta'	
	params.fstem = params.virus
	
	vdb = vdb_download(**params.__dict__)

	# rebuild
	if (params.build):
		build(params)

	# watch
	if (params.watch):
		while True:
			rcount = vdb.count_documents()
			lcount = local_count(params)
			print rcount, "remote documents and", lcount, "local documents"
			if (rcount > lcount):
				build(params)
			time.sleep(60)

Include run.py script that pulls from rethink and processes.