<commit_before>#!/usr/bin/env python

import subprocess, os, shutil, sys, glob
from xml.etree import ElementTree

max_number_jobs = 2000

# execute a shell command
def call_shell(cmd, sh=False):
  p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=sh)
  p.wait()
  return p.poll(), p.stdout.read(), p.stderr.read()

def number_running_jobs():
  number_jobs = 0
  num, out, err = call_shell(['qstat -u `whoami` | egrep " (Q|R) " | wc -l'], sh=True)
  return int(out)
#  tree = ElementTree.fromstring(out)
#  for job in tree.getiterator('Job'):
#    state = job.find('job_state')
#    if state.text == 'R':
#      number_jobs += 1
#      print '.'
#  return number_jobs

def submit_jobs(files):
  n_running = number_running_jobs()
  n_submit = max_number_jobs - n_running
  print 'There are', n_running, 'jobs currently running. We can submit another', n_submit, 'jobs.'
  while n_submit > 0 and len(files) > 0:
    file = os.path.expanduser(os.path.abspath(files.pop(0)))
    print 'Submitting job', file
    if os.path.exists(file):
      subprocess.check_call(['qsub',file])
      done_dir = os.path.join(os.path.dirname(file),'done_jobs')
      if not os.path.isdir(done_dir):
        os.makedirs(done_dir)
      shutil.move(file, os.path.join(os.path.dirname(file),'done_jobs',os.path.basename(file)))
      n_submit -= 1
    else: 
      raise Exception("Job " + file + " does not exist!")

def auto_submit(control_file):
  print 'Submitting jobs from control file', os.path.expanduser(os.path.abspath(control_file))
  f = open(control_file, 'r')
  for line in f:
    print glob.glob(line)
    submit_jobs(glob.glob(line))

if __name__ == "__main__":
  if len(sys.argv) > 2:
    submit_jobs(sys.argv[1:])
  elif len(sys.argv) == 2:
    auto_submit(sys.argv[1])
  else:
    print 'Usage', sys.argv[0], 'control_file or list of jobs'<commit_msg>bugfixes and improvements to job management scripts<commit_after>#!/usr/bin/env python

import subprocess, os, shutil, sys, glob
from xml.etree import ElementTree

max_number_jobs = 2000

# execute a shell command
def call_shell(cmd, sh=False):
  p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=sh)
  p.wait()
  return p.poll(), p.stdout.read(), p.stderr.read()

def number_running_jobs():
  number_jobs = 0
  num, out, err = call_shell(['qstat -u `whoami` | egrep " (Q|R) " | wc -l'], sh=True)
  return int(out)
#  tree = ElementTree.fromstring(out)
#  for job in tree.getiterator('Job'):
#    state = job.find('job_state')
#    if state.text == 'R':
#      number_jobs += 1
#      print '.'
#  return number_jobs

def submit_jobs(files):
  n_running = number_running_jobs()
  n_submit = max_number_jobs - n_running
  print 'There are', n_running, 'jobs currently running. We can submit another', n_submit, 'jobs.'
  while n_submit > 0 and len(files) > 0:
    file = os.path.expanduser(os.path.abspath(files.pop(0)))
    print 'Submitting job', file
    if os.path.exists(file):
      subprocess.check_call(['qsub',file])
      done_dir = os.path.join(os.path.dirname(file),'done_jobs')
      if not os.path.isdir(done_dir):
        os.makedirs(done_dir)
      shutil.move(file, os.path.join(os.path.dirname(file),'done_jobs',os.path.basename(file)))
      n_submit -= 1
    else: 
      raise Exception("Job " + file + " does not exist!")

def auto_submit(control_file):
  print 'Submitting jobs from control file', os.path.expanduser(os.path.abspath(control_file))
  f = open(control_file, 'r')
  for line in f:
    submit_jobs(glob.glob(line))

if __name__ == "__main__":
  if len(sys.argv) > 2:
    submit_jobs(sys.argv[1:])
  elif len(sys.argv) == 2:
    auto_submit(sys.argv[1])
  else:
    print 'Usage', sys.argv[0], 'control_file or list of jobs'
{'fraction_non_alphanumeric': 0.14851222104144526, 'fraction_numerical': 0.007970244420828906, 'mean_word_length': 3.864341085271318, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4994805', 'n_tokens_mistral': 1422, 'n_tokens_neox': 1365, 'n_words': 313}