<commit_before>"""
Job module for pyppl
"""
import json
import sys
from collections import OrderedDict
from glob import glob
from os import makedirs, path, remove, symlink, utime, readlink, listdir
from shutil import copyfile, copytree, move, rmtree
from subprocess import Popen, PIPE

from . import utils, logger

class job (object):
	
	"""
	Job class, defining a job in a process

	@static variables:
		`FAILED_RC`: Jobs failed to submit, no return code available
		`EMPTY_RC`:  Rc file not generated, not is empty
		`NOOUT_RC`:  Outfile not generated
		`RC_MSGS`:   The messages when job failed
	"""
	
	FAILED_RC = 9999
	EMPTY_RC  = 9998
	NOOUT_RC  = -1000
	RC_MSGS   = {
		9998:  "No rcfile generated or empty",
		9999:  "Failed to submit/run the jobs",
		-1000: "Output files not generated or expections didn't meet",
		1:     "Script error"
	}
		
	def __init__(self, index, proc):
		"""
		Constructor
		@params:
			`index`:   The index of the job in a process
			`proc`:    The process
		"""
		self.dir       = path.abspath(path.join (proc.workdir, str(index)))
		self.indir     = path.join (self.dir, "input")
		self.outdir    = path.join (self.dir, "output")
		self.script    = path.join (self.dir, "job.script")
		self.oscript   = path.join (self.dir, "job.oscript")
		self.rcfile    = path.join (self.dir, "job.rc")
		self.outfile   = path.join (self.dir, "job.stdout")
		self.errfile   = path.join (self.dir, "job.stderr")
		self.cachefile = path.join (self.dir, "job.cache")
		self.idfile    = path.join (self.dir, "job.id")
		#self.input   = {'var':[], 'file':[], 'files':[]} if input is None else input
		#self.output  = {'var':[], 'file':[]} if output is None else input
		self.index     = index
		self.proc      = proc
		self.input     = {}
		# need to pass this to next procs, so have to keep order
		self.output    = OrderedDict()
		self.brings    = {}
		self.data      = {
			'#':           index,
			'job.index':   index,
			'job.id':      '',
			'job.indir':   self.indir,
			'job.outdir':  self.outdir,
			'job.dir':     self.dir,
			'job.outfile': self.outfile,
			'job.errfile': self.errfile,
			'job.idfile':  self.idfile
		}
		
	def init (self):
		"""
		Initiate a job, make directory and prepare input, brings, output and script.
		"""
		if not path.exists(self.dir):
			makedirs (self.dir)
		self.data.update (self.proc.procvars)
		self._prepInput ()
		self._prepBrings ()
		self._prepOutput ()
		self._prepScript ()

	def report (self):
		"""
		Report the job information to logger
		"""
		for key in sorted(self.input.keys()):
			if self.input[key]['type'] in self.proc.IN_FILESTYPE:
				self.proc.log ("[%s/%s] %s => [%s, ...]" % (self.index, self.proc.length - 1, key, self.input[key]['data'][0]), 'input')
				if 'orig' in self.input[key]:
					self.proc.log ("[%s/%s] %s.orig => [%s, ...]" % (self.index, self.proc.length - 1, key, self.input[key]['orig'][0]), 'input')
			else:
				self.proc.log ("[%s/%s] %s => %s" % (self.index, self.proc.length - 1, key, self.input[key]['data']), 'input')
				if 'orig' in self.input[key]:
					self.proc.log ("[%s/%s] %s.orig => %s" % (self.index, self.proc.length - 1, key, self.input[key]['orig']), 'input')
		for key in sorted(self.brings.keys()):
			self.proc.log ("[%s/%s] %s => %s" % (self.index, self.proc.length - 1, key, self.brings[key]), 'brings')
		for key in sorted(self.output.keys()):
			self.proc.log ("[%s/%s] %s => %s" % (self.index, self.proc.length - 1, key, self.output[key]['data']), 'output')	
	<commit_msg>Make it clearer in logs for input files<commit_after>"""
Job module for pyppl
"""
import json
import sys
from collections import OrderedDict
from glob import glob
from os import makedirs, path, remove, symlink, utime, readlink, listdir
from shutil import copyfile, copytree, move, rmtree
from subprocess import Popen, PIPE

from . import utils, logger

class job (object):
	
	"""
	Job class, defining a job in a process

	@static variables:
		`FAILED_RC`: Jobs failed to submit, no return code available
		`EMPTY_RC`:  Rc file not generated, not is empty
		`NOOUT_RC`:  Outfile not generated
		`RC_MSGS`:   The messages when job failed
	"""
	
	FAILED_RC = 9999
	EMPTY_RC  = 9998
	NOOUT_RC  = -1000
	RC_MSGS   = {
		9998:  "No rcfile generated or empty",
		9999:  "Failed to submit/run the jobs",
		-1000: "Output files not generated or expections didn't meet",
		1:     "Script error"
	}
		
	def __init__(self, index, proc):
		"""
		Constructor
		@params:
			`index`:   The index of the job in a process
			`proc`:    The process
		"""
		self.dir       = path.abspath(path.join (proc.workdir, str(index)))
		self.indir     = path.join (self.dir, "input")
		self.outdir    = path.join (self.dir, "output")
		self.script    = path.join (self.dir, "job.script")
		self.oscript   = path.join (self.dir, "job.oscript")
		self.rcfile    = path.join (self.dir, "job.rc")
		self.outfile   = path.join (self.dir, "job.stdout")
		self.errfile   = path.join (self.dir, "job.stderr")
		self.cachefile = path.join (self.dir, "job.cache")
		self.idfile    = path.join (self.dir, "job.id")
		#self.input   = {'var':[], 'file':[], 'files':[]} if input is None else input
		#self.output  = {'var':[], 'file':[]} if output is None else input
		self.index     = index
		self.proc      = proc
		self.input     = {}
		# need to pass this to next procs, so have to keep order
		self.output    = OrderedDict()
		self.brings    = {}
		self.data      = {
			'#':           index,
			'job.index':   index,
			'job.id':      '',
			'job.indir':   self.indir,
			'job.outdir':  self.outdir,
			'job.dir':     self.dir,
			'job.outfile': self.outfile,
			'job.errfile': self.errfile,
			'job.idfile':  self.idfile
		}
		
	def init (self):
		"""
		Initiate a job, make directory and prepare input, brings, output and script.
		"""
		if not path.exists(self.dir):
			makedirs (self.dir)
		self.data.update (self.proc.procvars)
		self._prepInput ()
		self._prepBrings ()
		self._prepOutput ()
		self._prepScript ()

	def report (self):
		"""
		Report the job information to logger
		"""
		for key in sorted(self.input.keys()):
			if self.input[key]['type'] in self.proc.IN_FILESTYPE:
				data1 = self.input[key]['data'][0]
				data3 = self.input[key]['data'][1:3]
				dataR = self.input[key]['data'][3:]
				orig1 = self.input[key]['orig'][0]
				orig3 = self.input[key]['orig'][1:3]
				origR = self.input[key]['orig'][3:]
				
				self.proc.log ("[%s/%s] %s => %s, " % (self.index, self.proc.length - 1, key, data1), 'input')
				for data in data3:
					self.proc.log ("[%s/%s] %s    %s, " % (self.index, self.proc.length - 1, ' ' * len(key), data), 'input')
				if dataR:
					self.proc.log ("[%s/%s] %s    ... %s more not shown." % (self.index, self.proc.length - 1, ' ' * len(key), len(dataR)), 'input')
					
				self.proc.log ("[%s/%s] %s.orig => %s, " % (self.index, self.proc.length - 1, key, orig1), 'input')
				for orig in orig3:
					self.proc.log ("[%s/%s] %s         %s, " % (self.index, self.proc.length - 1, ' ' * len(key), orig), 'input')
				if origR:
					self.proc.log ("[%s/%s] %s         ... %s more not shown." % (self.index, self.proc.length - 1, ' ' * len(key), len(origR)), 'input')
			else:
				self.proc.log ("[%s/%s] %s => %s" % (self.index, self.proc.length - 1, key, self.input[key]['data']), 'input')
				if 'orig' in self.input[key]:
					self.proc.log ("[%s/%s] %s.orig => %s" % (self.index, self.proc.length - 1, key, self.input[key]['orig']), 'input')
		for key in sorted(self.brings.keys()):
			self.proc.log ("[%s/%s] %s => %s" % (self.index, self.proc.length - 1, key, self.brings[key]), 'brings')
		for key in sorted(self.output.keys()):
			self.proc.log ("[%s/%s] %s => %s" % (self.index, self.proc.length - 1, key, self.output[key]['data']), 'output')	
	
{'fraction_non_alphanumeric': 0.18151390319258495, 'fraction_numerical': 0.010813594232749742, 'mean_word_length': 3.031655422937208, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3058455', 'n_tokens_mistral': 3211, 'n_tokens_neox': 2875, 'n_words': 769}