<commit_before>#!/usr/bin/env python
# QSubDriver.py
#
#   Copyright (C) 2006 CCLRC, <NAME>
#
#   This code is distributed under the BSD license, a copy of which is
#   included in the root directory of this package.
#
# 16th June 2006
#
# A Driver implementation to work with sun grid engine clusters via the
# "qsub" shell command. This is based on ScriptDriver. This works like...
#
# > qsub -cwd process-lrem.sh
# Your job 1871 ("process-lrem.sh") has been submitted.
#
# Applicability: Linux with Sun Grid Engine
#
# How This Works
# --------------
#
# qsub to get the job id, then repeated qstat calls to find out what's
# happening with this job. Raise exception if it looks like something has
# gone wrong (e.g. all of the queues are disabled or something.)
#
# To find out when this has finished, keep calling qstat -j job_id
# until the following is seen:
#
# Following jobs do not exist: 164459
#
# The rest of the Driver stuff can then follow.
#
# This class has been deprecated. See SunGridEngineClusterDriver.
#

import os
import sys
import subprocess
import time
import random

from DefaultDriver import DefaultDriver
from DriverHelper import script_writer

# Now depend on Phil scope from xia2...
if not os.environ.has_key('XIA2_ROOT'):
  raise RuntimeError, 'XIA2_ROOT not defined'

if not os.environ['XIA2_ROOT'] in sys.path:
  sys.path.append(os.environ['XIA2_ROOT'])

def get_qsub_command():
  from Handlers.Phil import PhilIndex
  params = PhilIndex.get_python_object()
  mp_params = params.xia2.settings.multiprocessing
  if mp_params.qsub_command:
    return mp_params.qsub_command
  return None

class QSubDriver(DefaultDriver):

  def __init__(self):

    if os.name != 'posix':
      raise RuntimeError, 'os "%s" not supported' % os.name

    DefaultDriver.__init__(self)

    self._script_command_line = []
    self._script_standard_input = []

    self.set_qsub_name(self._name)

    self._script_status = 0

    # this is opened by the close() method and read by output
    # from self._script_name.xout

    self._output_file = None

    return

  def set_qsub_name(self, name):
    '''Set the name to something sensible.'''
    self._script_name = 'J%s' % name

    return

  def start(self):
    '''This is pretty meaningless in terms of running things through
    scripts...'''

    for c in self._command_line:
      self._script_command_line.append(c)

  def check(self):
    '''NULL overloading of the default check method.'''
    return True

  def check_sge_errors(self, sge_stderr_list):
    '''Check through the standard error from Sun Grid Engine
    for indications that something went wrong...'''

    for o in sge_stderr_list:
      if 'command not found' in o:
        missing_program = o.split(':')[2].strip()
        raise RuntimeError, 'executable "%s" missing' % \
              missing_program

  def _input(self, record):
    self._script_standard_input.append(record)

    return

  def _output(self):
    return self._output_file.readline()

  def _status(self):
    return self._script_status

  def close(self):
    '''This is where most of the work will be done - in here is
    where the script itself gets written and run, and the output
    file channel opened when the process has finished...'''

    script_name = os.path.join('jobs', self._script_name)

    try:
      os.mkdir(os.path.join(self._working_directory, 'jobs'))
    except OSError, e:
      if not os.path.exists(os.path.join(self._working_directory, 'jobs')):
        raise

    # copy in LD_LIBRARY_PATH - SGE squashes this
    if 'LD_LIBRARY_PATH' in os.environ and \
       not 'LD_LIBRARY_PATH' in self._working_environment:
      self._working_environment['LD_LIBRARY_PATH'] = os.environ[
        'LD_LIBRARY_PATH'].split(os.pathsep)

    script_writer(self._working_directory,
                  script_name,
                  self._executable,
                  self._script_command_line,
                  self._working_environment,
                  self._script_standard_input)

    # this will return almost instantly, once the job is in
    # the queue

    qsub_command = get_qsub_command()
    if not qsub_command:
      qsub_command = 'qsub'

    if self._cpu_threads > 1:
      pipe = subprocess.Popen(qsub_command.split() +
                              ['-V', '-cwd',
                               '-pe', 'smp', '%d' % self._cpu_threads,
                               '%s.sh' % script_name],
                               cwd = self._working_directory,
                               stdout = subprocess.PIPE,
                               stderr = subprocess.PIPE)
    else:
      pipe = subprocess.Popen(qsub_command.split() +
                              ['-V', '-cwd',
                               '%s.sh' % script_name],
                               cwd = self._working_directory,
                               stdout = subprocess.PIPE,
                               stderr = subprocess.PIPE)

    # this will get all of the output as a tuple (stdout, stderr)
    stdout, stderr = pipe.communicate()

    # check the standard error
    if len(stderr) > 0:
      # something probably went wrong
      if 'error opening' in stderr:
        raise RuntimeError, 'executable "%s" does not exist' % \
              stdout.split('\n')[0].split(':')[0].replace(
            'error opening ', '')

    # probably everything is ok then

    # the job id etc go to the standard output

    for record in stdout.split('\n'):
      if 'Your job' in record:<commit_msg>use shlex to split the qsub_command rather than reyling on string.split()<commit_after>#!/usr/bin/env python
# QSubDriver.py
#
#   Copyright (C) 2006 CCLRC, <NAME>
#
#   This code is distributed under the BSD license, a copy of which is
#   included in the root directory of this package.
#
# 16th June 2006
#
# A Driver implementation to work with sun grid engine clusters via the
# "qsub" shell command. This is based on ScriptDriver. This works like...
#
# > qsub -cwd process-lrem.sh
# Your job 1871 ("process-lrem.sh") has been submitted.
#
# Applicability: Linux with Sun Grid Engine
#
# How This Works
# --------------
#
# qsub to get the job id, then repeated qstat calls to find out what's
# happening with this job. Raise exception if it looks like something has
# gone wrong (e.g. all of the queues are disabled or something.)
#
# To find out when this has finished, keep calling qstat -j job_id
# until the following is seen:
#
# Following jobs do not exist: 164459
#
# The rest of the Driver stuff can then follow.
#
# This class has been deprecated. See SunGridEngineClusterDriver.
#

import os
import sys
import subprocess
import time
import random

from DefaultDriver import DefaultDriver
from DriverHelper import script_writer

# Now depend on Phil scope from xia2...
if not os.environ.has_key('XIA2_ROOT'):
  raise RuntimeError, 'XIA2_ROOT not defined'

if not os.environ['XIA2_ROOT'] in sys.path:
  sys.path.append(os.environ['XIA2_ROOT'])

def get_qsub_command():
  from Handlers.Phil import PhilIndex
  params = PhilIndex.get_python_object()
  mp_params = params.xia2.settings.multiprocessing
  if mp_params.qsub_command:
    return mp_params.qsub_command
  return None

class QSubDriver(DefaultDriver):

  def __init__(self):

    if os.name != 'posix':
      raise RuntimeError, 'os "%s" not supported' % os.name

    DefaultDriver.__init__(self)

    self._script_command_line = []
    self._script_standard_input = []

    self.set_qsub_name(self._name)

    self._script_status = 0

    # this is opened by the close() method and read by output
    # from self._script_name.xout

    self._output_file = None

    return

  def set_qsub_name(self, name):
    '''Set the name to something sensible.'''
    self._script_name = 'J%s' % name

    return

  def start(self):
    '''This is pretty meaningless in terms of running things through
    scripts...'''

    for c in self._command_line:
      self._script_command_line.append(c)

  def check(self):
    '''NULL overloading of the default check method.'''
    return True

  def check_sge_errors(self, sge_stderr_list):
    '''Check through the standard error from Sun Grid Engine
    for indications that something went wrong...'''

    for o in sge_stderr_list:
      if 'command not found' in o:
        missing_program = o.split(':')[2].strip()
        raise RuntimeError, 'executable "%s" missing' % \
              missing_program

  def _input(self, record):
    self._script_standard_input.append(record)

    return

  def _output(self):
    return self._output_file.readline()

  def _status(self):
    return self._script_status

  def close(self):
    '''This is where most of the work will be done - in here is
    where the script itself gets written and run, and the output
    file channel opened when the process has finished...'''

    script_name = os.path.join('jobs', self._script_name)

    try:
      os.mkdir(os.path.join(self._working_directory, 'jobs'))
    except OSError, e:
      if not os.path.exists(os.path.join(self._working_directory, 'jobs')):
        raise

    # copy in LD_LIBRARY_PATH - SGE squashes this
    if 'LD_LIBRARY_PATH' in os.environ and \
       not 'LD_LIBRARY_PATH' in self._working_environment:
      self._working_environment['LD_LIBRARY_PATH'] = os.environ[
        'LD_LIBRARY_PATH'].split(os.pathsep)

    script_writer(self._working_directory,
                  script_name,
                  self._executable,
                  self._script_command_line,
                  self._working_environment,
                  self._script_standard_input)

    # this will return almost instantly, once the job is in
    # the queue

    qsub_command = get_qsub_command()
    if not qsub_command:
      qsub_command = 'qsub'

    import shlex
    qsub_command = shlex.split(qsub_command)
    if self._cpu_threads > 1:
      pipe = subprocess.Popen(qsub_command +
                              ['-V', '-cwd',
                               '-pe', 'smp', '%d' % self._cpu_threads,
                               '%s.sh' % script_name],
                               cwd = self._working_directory,
                               stdout = subprocess.PIPE,
                               stderr = subprocess.PIPE)
    else:
      pipe = subprocess.Popen(qsub_command +
                              ['-V', '-cwd',
                               '%s.sh' % script_name],
                               cwd = self._working_directory,
                               stdout = subprocess.PIPE,
                               stderr = subprocess.PIPE)

    # this will get all of the output as a tuple (stdout, stderr)
    stdout, stderr = pipe.communicate()

    # check the standard error
    if len(stderr) > 0:
      # something probably went wrong
      if 'error opening' in stderr:
        raise RuntimeError, 'executable "%s" does not exist' % \
              stdout.split('\n')[0].split(':')[0].replace(
            'error opening ', '')

    # probably everything is ok then

    # the job id etc go to the standard output

    for record in stdout.split('\n'):
      if 'Your job' in record:
{'fraction_non_alphanumeric': 0.09493443506376864, 'fraction_numerical': 0.005748158792886654, 'mean_word_length': 2.443104514533086, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1342061', 'n_tokens_mistral': 3372, 'n_tokens_neox': 3221, 'n_words': 1051}