<commit_before>import saga
import os
import re
import random
import time

def submit_remote_job(mod, username, params):

	ctx = saga.Context("UserPass")
	ctx.user_id = username

	session = saga.Session()
	session.add_context(ctx)

	service = saga.job.Service("slurm://localhost",session=session)

	# job info from database...
	job_id = 'saga.'+str(random.randint(10000,90000))
	job = saga.job.Description()
	job.working_directory = "/lustre/janus_scratch/%s/webapi"%username
	
	# Look up job information
	job.wall_time_limit   = 5 # minutes
	job.total_cpu_count   = 12 
	job.queue             = "janus-admin"
	job.output            = mod.name + '.' + job_id + '.out'
	job.error             = mod.name + '.' + job_id + '.err'
	job.executable        = mod.cmd
	job.arguments         = params

	touchjob = service.create_job(job)
	touchjob.run()

	jid = re.findall(r'\[([0-9]+)\]', touchjob.get_id())[0]
	jout = os.path.join(job.working_directory,job.output)
	jerr = os.path.join(job.working_directory,job.error)

	service.close()

	return jid, jout, jerr

def get_job_ouput(job):

	ctx = saga.Context("UserPass")
	ctx.user_id = username

	session = saga.Session()
	session.add_context(ctx)
	print job.output
	# output = 'sftp://localhost' + job.output 
	# tmp_ = '/Users/mlunacek/projects/myhat/app/tmp/tmp.output'
	# target = 'file://localhost' + tmp_
	target = os.path.join(job.working_directory,job.output)


	out = saga.filesystem.File(output, session=session)
	if out is None:
		return '\nFile does not exist yet\n'

	out.copy(target)

	while not os.path.exists(tmp_):
		time.sleep(1)

	with open(tmp_, 'r') as infile:
		data = infile.read()

	os.remove(tmp_)
	return data


	    #return jsonify( {'success': 'True'} )
	# except:
	# 	print 'fail'

	    #return jsonify( {'success': 'False'} )
	# print "Job State   : %s" % (touchjob.state)
	# print "Exitcode    : %s" % (touchjob.exit_code)
	# # print "output      : %s" % (touchjob.output)
	# print 'id          : %s' % (touchjob.get_id())

	# print 'output      : %s' % (job.output)
	# print 'error       : %s' % (job.error)
	# print ' '.join(job.arguments)
	# print job.executable

	# print re.findall(r'\[([0-9]+)\]', touchjob.get_id())[0]
	# print os.path.join(job.working_directory,job.output)

	


<commit_msg>Corrected file handling for job output.<commit_after>import saga
import os
import re
import random
import time

def submit_remote_job(mod, username, params):

	ctx = saga.Context("UserPass")
	ctx.user_id = username

	session = saga.Session()
	session.add_context(ctx)

	service = saga.job.Service("slurm://localhost",session=session)

	# job info from database...
	job_id = 'saga.'+str(random.randint(10000,90000))
	job = saga.job.Description()
	job.working_directory = "/lustre/janus_scratch/%s/webapi"%username
	
	# Look up job information
	job.wall_time_limit   = 5 # minutes
	job.total_cpu_count   = 12 
	# job.queue             = "janus-admin" # NOT USED WITH SLURM
	job.output            = mod.name + '.' + job_id + '.out'
	job.error             = mod.name + '.' + job_id + '.err'
	job.executable        = mod.cmd
	job.arguments         = params

	touchjob = service.create_job(job)
	touchjob.run()

	jid = re.findall(r'\[([0-9]+)\]', touchjob.get_id())[0]
	jout = os.path.join(job.working_directory,job.output)
	jerr = os.path.join(job.working_directory,job.error)

	service.close()

	return jid, jout, jerr

def get_job_ouput(job):

	ctx = saga.Context("UserPass")
	ctx.user_id = username

	session = saga.Session()
	session.add_context(ctx)
	print job.output
	# output = 'sftp://localhost' + job.output 
	# tmp_ = '/Users/mlunacek/projects/myhat/app/tmp/tmp.output'
	# target = 'file://localhost' + tmp_
	target = os.path.join(job.working_directory,job.output)


	out = saga.filesystem.File(output, session=session)
	if out is None:
		return '\nFile does not exist yet\n'

	out.copy(target)

	# while not os.path.exists(tmp_):
	# 	time.sleep(1)

	with open(target, 'r') as infile:
		data = infile.read()

	# os.remove(tmp_)
	return data


	    #return jsonify( {'success': 'True'} )
	# except:
	# 	print 'fail'

	    #return jsonify( {'success': 'False'} )
	# print "Job State   : %s" % (touchjob.state)
	# print "Exitcode    : %s" % (touchjob.exit_code)
	# # print "output      : %s" % (touchjob.output)
	# print 'id          : %s' % (touchjob.get_id())

	# print 'output      : %s' % (job.output)
	# print 'error       : %s' % (job.error)
	# print ' '.join(job.arguments)
	# print job.executable

	# print re.findall(r'\[([0-9]+)\]', touchjob.get_id())[0]
	# print os.path.join(job.working_directory,job.output)

	



{'fraction_non_alphanumeric': 0.1613745106568073, 'fraction_numerical': 0.008699434536755112, 'mean_word_length': 3.3386792452830187, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4065113', 'n_tokens_mistral': 1799, 'n_tokens_neox': 1699, 'n_words': 339}