<commit_before>#!/usr/bin/env python
from __future__ import print_function
from __future__ import division
import argparse
import math
import os
import pkg_resources
import re
import subprocess
import stat
import sys
import fnmatch
import errno


def _setupVars():

    # setup defaults (let environment override)
    global SYSTEM
    SYSTEM = os.environ.get("QBATCH_SYSTEM", "local")
    if SYSTEM == "slurm":
        which('sbatch') or sys.exit("qbatch: error: QBATCH_SYSTEM set to slurm"
                                    " but sbatch not found")
        which('squeue') or sys.exit("qbatch: error: QBATCH_SYSTEM set to slurm"
                                    " but squeue not found")
    elif (SYSTEM == "pbs") or (SYSTEM == "sge"):
        which('qsub') or sys.exit("qbatch: error: QBATCH_SYSTEM set to pbs/sge"
                                  " but qsub not found")
        which('qstat') or sys.exit("qbatch: error: QBATCH_SYSTEM set to"
                                   " pbs/sge but qstat not found")
    else:
        which('parallel') or sys.exit("qbatch: error: QBATCH_SYSTEM set to"
                                      " local but parallel not found")
    global PPJ
    PPJ = os.environ.get("QBATCH_PPJ", "1")
    global CHUNKSIZE
    CHUNKSIZE = os.environ.get("QBATCH_CHUNKSIZE", PPJ)
    global CORES
    CORES = os.environ.get("QBATCH_CORES", PPJ)
    global NODES
    NODES = os.environ.get("QBATCH_NODES", 1)
    global SGE_PE
    SGE_PE = os.environ.get("QBATCH_SGE_PE", "smp")
    global MEMVARS
    MEMVARS = os.environ.get("QBATCH_MEMVARS", "mem")
    global MEM
    MEM = os.environ.get("QBATCH_MEM", "0")
    global SCRIPT_FOLDER
    SCRIPT_FOLDER = os.environ.get("QBATCH_SCRIPT_FOLDER", ".qbatch/")
    global QUEUE
    QUEUE = os.environ.get("QBATCH_QUEUE", None)
    global SHELL
    SHELL = os.environ.get("QBATCH_SHELL", "/bin/sh")
    global OPTIONS
    OPTIONS = [os.environ.get("QBATCH_OPTIONS")] if os.environ.get(
        "QBATCH_OPTIONS") else []

    # environment vars to ignore when copying the environment to the job script
    global IGNORE_ENV_VARS
    IGNORE_ENV_VARS = ['PWD', 'SGE_TASK_ID', 'PBS_ARRAYID', 'ARRAY_IND',
                       'BASH_FUNC_*']

    global PBS_HEADER_TEMPLATE
    PBS_HEADER_TEMPLATE = """
    #!{shell}
    #PBS -l nodes={nodes}:{nodes_spec}ppn={ppj}
    #PBS -j oe
    #PBS -o {logdir}
    #PBS -d {workdir}
    #PBS -N {job_name}
    #PBS {o_memopts}
    #PBS {o_queue}
    #PBS {o_array}
    #PBS {o_walltime}
    #PBS {o_dependencies}
    #PBS {o_options}
    #PBS {o_env}
    {env}
    {header_commands}
    ARRAY_IND=$PBS_ARRAYID
    """.strip()

    global SGE_HEADER_TEMPLATE
    SGE_HEADER_TEMPLATE = """
    #!{shell}
    #$ {ppj}
    #$ -j y
    #$ -o {logdir}
    #$ -wd {workdir}
    #$ -N {job_name}
    #$ {o_memopts}
    #$ {o_queue}
    #$ {o_array}
    #$ {o_walltime}
    #$ {o_dependencies}
    #$ {o_options}
    #$ {o_env}
    {env}
    {header_commands}
    ARRAY_IND=$SGE_TASK_ID
    """.strip()

    global SLURM_HEADER_TEMPLATE
    SLURM_HEADER_TEMPLATE = """
    #!{shell}
    #SBATCH --nodes={nodes}
    #SBATCH {ppj}
    #SBATCH {logfile}
    #SBATCH --chdir={workdir}
    #SBATCH --job-name={job_name}
    #SBATCH {o_memopts}
    #SBATCH {o_queue}
    #SBATCH {o_array}
    #SBATCH {o_walltime}
    #SBATCH {o_dependencies}
    #SBATCH {o_options}
    #SBATCH {o_env}
    {env}
    {header_commands}
    ARRAY_IND=$SLURM_ARRAY_TASK_ID
    """.strip()

    global LOCAL_TEMPLATE
    LOCAL_TEMPLATE = """
    #!{shell}
    {env}
    {header_commands}
    cd {workdir}
    """.strip()


def run_command(command, logfile=None):
    # Run command and collect stdout
    # http://blog.endpoint.com/2015/01/getting-realtime-output-using-python.html # noqa
    process = subprocess.Popen(
        command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    if logfile:
        filehandle = open(logfile, 'wb')
    while True:
        output = process.stdout.readline()
        if output.decode('UTF-8') == '' and process.poll() is not None:
            break
        if output and logfile:
            print(output.strip().decode('UTF-8'))
            filehandle.write(output.strip())
            filehandle.write("\n".encode('UTF-8'))
        elif output:
            print(output.strip().decode('UTF-8'))
    rc = process.poll()
    if logfile:
        filehandle.close()
    return rc<commit_msg>Add quick and dirty way to avoid setting defaults twice<commit_after>#!/usr/bin/env python
from __future__ import print_function
from __future__ import division
import argparse
import math
import os
import pkg_resources
import re
import subprocess
import stat
import sys
import fnmatch
import errno


def _setupVars():
    try:
        __varsSet
    except NameError:
        pass
    else:
        return
    # setup defaults (let environment override)
    global SYSTEM
    SYSTEM = os.environ.get("QBATCH_SYSTEM", "local")
    if SYSTEM == "slurm":
        which('sbatch') or sys.exit("qbatch: error: QBATCH_SYSTEM set to slurm"
                                    " but sbatch not found")
        which('squeue') or sys.exit("qbatch: error: QBATCH_SYSTEM set to slurm"
                                    " but squeue not found")
    elif (SYSTEM == "pbs") or (SYSTEM == "sge"):
        which('qsub') or sys.exit("qbatch: error: QBATCH_SYSTEM set to pbs/sge"
                                  " but qsub not found")
        which('qstat') or sys.exit("qbatch: error: QBATCH_SYSTEM set to"
                                   " pbs/sge but qstat not found")
    else:
        which('parallel') or sys.exit("qbatch: error: QBATCH_SYSTEM set to"
                                      " local but parallel not found")
    global PPJ
    PPJ = os.environ.get("QBATCH_PPJ", "1")
    global CHUNKSIZE
    CHUNKSIZE = os.environ.get("QBATCH_CHUNKSIZE", PPJ)
    global CORES
    CORES = os.environ.get("QBATCH_CORES", PPJ)
    global NODES
    NODES = os.environ.get("QBATCH_NODES", 1)
    global SGE_PE
    SGE_PE = os.environ.get("QBATCH_SGE_PE", "smp")
    global MEMVARS
    MEMVARS = os.environ.get("QBATCH_MEMVARS", "mem")
    global MEM
    MEM = os.environ.get("QBATCH_MEM", "0")
    global SCRIPT_FOLDER
    SCRIPT_FOLDER = os.environ.get("QBATCH_SCRIPT_FOLDER", ".qbatch/")
    global QUEUE
    QUEUE = os.environ.get("QBATCH_QUEUE", None)
    global SHELL
    SHELL = os.environ.get("QBATCH_SHELL", "/bin/sh")
    global OPTIONS
    OPTIONS = [os.environ.get("QBATCH_OPTIONS")] if os.environ.get(
        "QBATCH_OPTIONS") else []

    # environment vars to ignore when copying the environment to the job script
    global IGNORE_ENV_VARS
    IGNORE_ENV_VARS = ['PWD', 'SGE_TASK_ID', 'PBS_ARRAYID', 'ARRAY_IND',
                       'BASH_FUNC_*']

    global PBS_HEADER_TEMPLATE
    PBS_HEADER_TEMPLATE = """
    #!{shell}
    #PBS -l nodes={nodes}:{nodes_spec}ppn={ppj}
    #PBS -j oe
    #PBS -o {logdir}
    #PBS -d {workdir}
    #PBS -N {job_name}
    #PBS {o_memopts}
    #PBS {o_queue}
    #PBS {o_array}
    #PBS {o_walltime}
    #PBS {o_dependencies}
    #PBS {o_options}
    #PBS {o_env}
    {env}
    {header_commands}
    ARRAY_IND=$PBS_ARRAYID
    """.strip()

    global SGE_HEADER_TEMPLATE
    SGE_HEADER_TEMPLATE = """
    #!{shell}
    #$ {ppj}
    #$ -j y
    #$ -o {logdir}
    #$ -wd {workdir}
    #$ -N {job_name}
    #$ {o_memopts}
    #$ {o_queue}
    #$ {o_array}
    #$ {o_walltime}
    #$ {o_dependencies}
    #$ {o_options}
    #$ {o_env}
    {env}
    {header_commands}
    ARRAY_IND=$SGE_TASK_ID
    """.strip()

    global SLURM_HEADER_TEMPLATE
    SLURM_HEADER_TEMPLATE = """
    #!{shell}
    #SBATCH --nodes={nodes}
    #SBATCH {ppj}
    #SBATCH {logfile}
    #SBATCH --chdir={workdir}
    #SBATCH --job-name={job_name}
    #SBATCH {o_memopts}
    #SBATCH {o_queue}
    #SBATCH {o_array}
    #SBATCH {o_walltime}
    #SBATCH {o_dependencies}
    #SBATCH {o_options}
    #SBATCH {o_env}
    {env}
    {header_commands}
    ARRAY_IND=$SLURM_ARRAY_TASK_ID
    """.strip()

    global LOCAL_TEMPLATE
    LOCAL_TEMPLATE = """
    #!{shell}
    {env}
    {header_commands}
    cd {workdir}
    """.strip()

    global __varsSet
    __varsSet = True


def run_command(command, logfile=None):
    # Run command and collect stdout
    # http://blog.endpoint.com/2015/01/getting-realtime-output-using-python.html # noqa
    process = subprocess.Popen(
        command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    if logfile:
        filehandle = open(logfile, 'wb')
    while True:
        output = process.stdout.readline()
        if output.decode('UTF-8') == '' and process.poll() is not None:
            break
        if output and logfile:
            print(output.strip().decode('UTF-8'))
            filehandle.write(output.strip())
            filehandle.write("\n".encode('UTF-8'))
        elif output:
            print(output.strip().decode('UTF-8'))
    rc = process.poll()
    if logfile:
        filehandle.close()
    return rc
{'fraction_non_alphanumeric': 0.12759808825163943, 'fraction_numerical': 0.0028898521729465378, 'mean_word_length': 2.4396024464831805, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6670908', 'n_tokens_mistral': 3292, 'n_tokens_neox': 3195, 'n_words': 701}