<commit_before>#!/usr/bin/env python

__copyright__ = "Copyright 2015, http://radical.rutgers.edu"
__license__   = "MIT"

import sys
import time
import os
import radical.pilot as rp
import random

# "Global" configs

# agent_config = {
#     "number_of_workers": {
#         "StageinWorker"   : 1,
#         "ExecWorker"      : 1,
#         "StageoutWorker"  : 1,
#         "UpdateWorker"    : 1
#     },
#     "blowup_factor": {
#         "Agent"           : 1,
#         "stagein_queue"   : 1,
#         "StageinWorker"   : 1,
#         "schedule_queue"  : 1,
#         "Scheduler"       : 1,
#         "execution_queue" : 1,
#         "ExecWorker"      : 1,
#         "watch_queue"     : 1,
#         "ExecWatcher"     : 1,
#         "stageout_queue"  : 1,
#         "StageoutWorker"  : 1,
#         "update_queue"    : 1,
#         "UpdateWorker"    : 1
#     },
#     "drop_clones": {
#         "Agent"           : 1,
#         "stagein_queue"   : 1,
#         "StageinWorker"   : 1,
#         "schedule_queue"  : 1,
#         "Scheduler"       : 1,
#         "execution_queue" : 1,
#         "ExecWorker"      : 1,
#         "watch_queue"     : 1,
#         "ExecWatcher"     : 1,
#         "stageout_queue"  : 1,
#         "StageoutWorker"  : 1,
#         "update_queue"    : 1,
#         "UpdateWorker"    : 1
#     }
# }

# Whether and how to install new RP remotely
RP_VERSION = "local" # debug, installed, local
VIRTENV_MODE = "create" # create, use, update

# Schedule CUs directly to a Pilot, assumes single Pilot
SCHEDULER = rp.SCHED_DIRECT_SUBMISSION

resource_config = {
    #
    # XE nodes have 2 "Interlagos" Processors with 8 "Bulldozer" cores each.
    # Every "Bulldozer" core consists of 2 schedualable integer cores.
    # XE nodes therefore have a PPN=32.
    #
    # Depending on the type of application,
    # one generally chooses to have 16 or 32 instances per node.

    'LOCAL': {
        'RESOURCE': 'local.localhost',
        'LAUNCH_METHOD': 'FORK',
        #'AGENT_SPAWNER': 'POPEN',
        'AGENT_SPAWNER': 'SHELL',
        'PPN': 4
    },
    'APRUN': {
        'RESOURCE': 'ncsa.bw',
        'LAUNCH_METHOD': 'APRUN',
        'AGENT_SPAWNER': 'SHELL',
        #'AGENT_SPAWNER': 'POPEN',
        'QUEUE': 'debug', # Maximum 30 minutes
        'PPN': 32
    },
    'CCM': {
        'RESOURCE': 'ncsa.bw_ccm',
        #'LAUNCH_METHOD': 'SSH',
        'LAUNCH_METHOD': 'MPIRUN',
        #'AGENT_SPAWNER': 'SHELL',
        'AGENT_SPAWNER': 'POPEN',
        'QUEUE': 'debug', # Maximum 30 minutes
        'PPN': 32,
        'PRE_EXEC_PREPEND': [
            'module use --append /u/sciteam/marksant/privatemodules',
            'module load use.own',
            'module load openmpi/1.8.4_ccm'
        ]
    },
    'ORTE': {
        'RESOURCE': 'ncsa.bw',
        'LAUNCH_METHOD': "ORTE",
        'AGENT_SPAWNER': 'SHELL',
    #nodes_var = [1]
        #'AGENT_SPAWNER': 'POPEN',
        #'QUEUE': 'debug', # Maximum 30 minutes
        'PPN': 32,
        'PRE_EXEC_PREPEND': [
            'module use --append /u/sciteam/marksant/privatemodules',
            'module load use.own',
            'module load openmpi/git'
        ]
    },
    'TITAN': {
        'RESOURCE': 'ornl.titan',
        'SCHEMA': 'local',
        'LAUNCH_METHOD': "ORTE",
        'AGENT_SPAWNER': 'SHELL',
        #'AGENT_SPAWNER': 'POPEN',
        #'QUEUE': 'debug', # Maximum 60 minutes
        'PROJECT': 'csc168',
        'PPN': 16,
        'PRE_EXEC_PREPEND': [
            #'module use --append /u/sciteam/marksant/privatemodules',
            #'module load use.own',
            #'module load openmpi/git'
        ]
    }
    'STAMPEDE': {
        'RESOURCE': 'xsede.stampede',
        #'SCHEMA': 'local',
        #'LAUNCH_METHOD': "ORTE",
        #'AGENT_SPAWNER': 'SHELL',
        #'AGENT_SPAWNER': 'POPEN',
        #'QUEUE': 'debug', # Maximum 30 minutes
        'PROJECT': 'TG-MCB090174',
        'PPN': 16,
        'PRE_EXEC_PREPEND': [
            #'module use --append /u/sciteam/marksant/privatemodules',
            #'module load use.own',
            #'module load openmpi/git'
        ]
    }
}



#------------------------------------------------------------------------------
#
def pilot_state_cb (pilot, state):

    if not pilot:
        return

    print "[Callback]: ComputePilot '%s' state: %s." % (pilot.uid, state)

    if state == rp.FAILED:
        sys.exit (1)<commit_msg>Add comma.<commit_after>#!/usr/bin/env python

__copyright__ = "Copyright 2015, http://radical.rutgers.edu"
__license__   = "MIT"

import sys
import time
import os
import radical.pilot as rp
import random

# "Global" configs

# agent_config = {
#     "number_of_workers": {
#         "StageinWorker"   : 1,
#         "ExecWorker"      : 1,
#         "StageoutWorker"  : 1,
#         "UpdateWorker"    : 1
#     },
#     "blowup_factor": {
#         "Agent"           : 1,
#         "stagein_queue"   : 1,
#         "StageinWorker"   : 1,
#         "schedule_queue"  : 1,
#         "Scheduler"       : 1,
#         "execution_queue" : 1,
#         "ExecWorker"      : 1,
#         "watch_queue"     : 1,
#         "ExecWatcher"     : 1,
#         "stageout_queue"  : 1,
#         "StageoutWorker"  : 1,
#         "update_queue"    : 1,
#         "UpdateWorker"    : 1
#     },
#     "drop_clones": {
#         "Agent"           : 1,
#         "stagein_queue"   : 1,
#         "StageinWorker"   : 1,
#         "schedule_queue"  : 1,
#         "Scheduler"       : 1,
#         "execution_queue" : 1,
#         "ExecWorker"      : 1,
#         "watch_queue"     : 1,
#         "ExecWatcher"     : 1,
#         "stageout_queue"  : 1,
#         "StageoutWorker"  : 1,
#         "update_queue"    : 1,
#         "UpdateWorker"    : 1
#     }
# }

# Whether and how to install new RP remotely
RP_VERSION = "local" # debug, installed, local
VIRTENV_MODE = "create" # create, use, update

# Schedule CUs directly to a Pilot, assumes single Pilot
SCHEDULER = rp.SCHED_DIRECT_SUBMISSION

resource_config = {
    #
    # XE nodes have 2 "Interlagos" Processors with 8 "Bulldozer" cores each.
    # Every "Bulldozer" core consists of 2 schedualable integer cores.
    # XE nodes therefore have a PPN=32.
    #
    # Depending on the type of application,
    # one generally chooses to have 16 or 32 instances per node.

    'LOCAL': {
        'RESOURCE': 'local.localhost',
        'LAUNCH_METHOD': 'FORK',
        #'AGENT_SPAWNER': 'POPEN',
        'AGENT_SPAWNER': 'SHELL',
        'PPN': 4
    },
    'APRUN': {
        'RESOURCE': 'ncsa.bw',
        'LAUNCH_METHOD': 'APRUN',
        'AGENT_SPAWNER': 'SHELL',
        #'AGENT_SPAWNER': 'POPEN',
        'QUEUE': 'debug', # Maximum 30 minutes
        'PPN': 32
    },
    'CCM': {
        'RESOURCE': 'ncsa.bw_ccm',
        #'LAUNCH_METHOD': 'SSH',
        'LAUNCH_METHOD': 'MPIRUN',
        #'AGENT_SPAWNER': 'SHELL',
        'AGENT_SPAWNER': 'POPEN',
        'QUEUE': 'debug', # Maximum 30 minutes
        'PPN': 32,
        'PRE_EXEC_PREPEND': [
            'module use --append /u/sciteam/marksant/privatemodules',
            'module load use.own',
            'module load openmpi/1.8.4_ccm'
        ]
    },
    'ORTE': {
        'RESOURCE': 'ncsa.bw',
        'LAUNCH_METHOD': "ORTE",
        'AGENT_SPAWNER': 'SHELL',
    #nodes_var = [1]
        #'AGENT_SPAWNER': 'POPEN',
        #'QUEUE': 'debug', # Maximum 30 minutes
        'PPN': 32,
        'PRE_EXEC_PREPEND': [
            'module use --append /u/sciteam/marksant/privatemodules',
            'module load use.own',
            'module load openmpi/git'
        ]
    },
    'TITAN': {
        'RESOURCE': 'ornl.titan',
        'SCHEMA': 'local',
        'LAUNCH_METHOD': "ORTE",
        'AGENT_SPAWNER': 'SHELL',
        #'AGENT_SPAWNER': 'POPEN',
        #'QUEUE': 'debug', # Maximum 60 minutes
        'PROJECT': 'csc168',
        'PPN': 16,
        'PRE_EXEC_PREPEND': [
            #'module use --append /u/sciteam/marksant/privatemodules',
            #'module load use.own',
            #'module load openmpi/git'
        ]
    },
    'STAMPEDE': {
        'RESOURCE': 'xsede.stampede',
        #'SCHEMA': 'local',
        #'LAUNCH_METHOD': "ORTE",
        #'AGENT_SPAWNER': 'SHELL',
        #'AGENT_SPAWNER': 'POPEN',
        #'QUEUE': 'debug', # Maximum 30 minutes
        'PROJECT': 'TG-MCB090174',
        'PPN': 16,
        'PRE_EXEC_PREPEND': [
            #'module use --append /u/sciteam/marksant/privatemodules',
            #'module load use.own',
            #'module load openmpi/git'
        ]
    }
}



#------------------------------------------------------------------------------
#
def pilot_state_cb (pilot, state):

    if not pilot:
        return

    print "[Callback]: ComputePilot '%s' state: %s." % (pilot.uid, state)

    if state == rp.FAILED:
        sys.exit (1)
{'fraction_non_alphanumeric': 0.16515426497277677, 'fraction_numerical': 0.017695099818511795, 'mean_word_length': 1.7527318139244459, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4681668', 'n_tokens_mistral': 3317, 'n_tokens_neox': 3068, 'n_words': 654}