<commit_before>"""
Job module for pyppl
"""
import json
from collections import OrderedDict
from glob import glob
from os import makedirs, path, remove, symlink, utime, readlink, listdir
from shutil import copyfile, copytree, move, rmtree
from multiprocessing import Lock
from . import utils, logger

class Job (object):
	
	"""
	Job class, defining a job in a process
	"""
		
	def __init__(self, index, proc):
		"""
		Constructor
		@params:
			`index`:   The index of the job in a process
			`proc`:    The process
		"""
		self.dir       = path.abspath(path.join (proc.workdir, str(index)))
		self.indir     = path.join (self.dir, "input")
		self.outdir    = path.join (self.dir, "output")
		self.script    = path.join (self.dir, "job.script")
		self.rcfile    = path.join (self.dir, "job.rc")
		self.outfile   = path.join (self.dir, "job.stdout")
		self.errfile   = path.join (self.dir, "job.stderr")
		self.cachefile = path.join (self.dir, "job.cache")
		self.pidfile   = path.join (self.dir, "job.pid")
		self.index     = index
		self.proc      = proc
		self.input     = {}
		# need to pass this to next procs, so have to keep order
		self.output    = OrderedDict()
		self.brings    = {}
		self.data      = {
			'job': {
				'index'   : self.index,
				'indir'   : self.indir,
				'outdir'  : self.outdir,
				'dir'     : self.dir,
				'outfile' : self.outfile,
				'errfile' : self.errfile,
				'pidfile' : self.pidfile
			},
			'in'   : {},
			'out'  : {},
			'bring': {}
		}
		
	def init (self):
		"""
		Initiate a job, make directory and prepare input, brings, output and script.
		"""
		if not path.exists(self.dir):
			makedirs (self.dir)
		self.data.update (self.proc.procvars)<commit_msg>Fix output channel key sequence does not keep.<commit_after>"""
Job module for pyppl
"""
import json
from collections import OrderedDict
from glob import glob
from os import makedirs, path, remove, symlink, utime, readlink, listdir
from shutil import copyfile, copytree, move, rmtree
from multiprocessing import Lock
from . import utils, logger

class Job (object):
	
	"""
	Job class, defining a job in a process
	"""
		
	def __init__(self, index, proc):
		"""
		Constructor
		@params:
			`index`:   The index of the job in a process
			`proc`:    The process
		"""
		self.dir       = path.abspath(path.join (proc.workdir, str(index)))
		self.indir     = path.join (self.dir, "input")
		self.outdir    = path.join (self.dir, "output")
		self.script    = path.join (self.dir, "job.script")
		self.rcfile    = path.join (self.dir, "job.rc")
		self.outfile   = path.join (self.dir, "job.stdout")
		self.errfile   = path.join (self.dir, "job.stderr")
		self.cachefile = path.join (self.dir, "job.cache")
		self.pidfile   = path.join (self.dir, "job.pid")
		self.index     = index
		self.proc      = proc
		self.input     = {}
		# need to pass this to next procs, so have to keep order
		self.output    = OrderedDict()
		self.brings    = {}
		self.data      = {
			'job': {
				'index'   : self.index,
				'indir'   : self.indir,
				'outdir'  : self.outdir,
				'dir'     : self.dir,
				'outfile' : self.outfile,
				'errfile' : self.errfile,
				'pidfile' : self.pidfile
			},
			'in'   : {},
			'out'  : OrderedDict(),
			'bring': {}
		}
		
	def init (self):
		"""
		Initiate a job, make directory and prepare input, brings, output and script.
		"""
		if not path.exists(self.dir):
			makedirs (self.dir)
		self.data.update (self.proc.procvars)
{'fraction_non_alphanumeric': 0.14264962231260894, 'fraction_numerical': 0.0, 'mean_word_length': 2.812846068660022, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2427851', 'n_tokens_mistral': 1363, 'n_tokens_neox': 1225, 'n_words': 339}