<commit_before>#! /usr/bin/env python
# runpbs_evol.py
# runs evolutionary algorithm on arm2dms model using PBS Torque in HPC

import os, sys
from numpy import mean
import csv
import copy
from inspyred.ec.variators import mutator
from random import Random
from time import time, sleep
import inspyred
import logging
from popen2 import popen2
import pickle
import multiprocessing
import Queue
import socket

ngen = -1 #global variable keeping number of generations

###############################################################################
### Simulation options
###############################################################################  
evolAlgorithm = 'evolutionStrategy' #'diffEvolution' # 'evolutionStrategy' #'krichmarCustom' #'genetic'#'particleSwarm100'#'estimationDist' #'diffEvolution' # 'evolutionStrategy' # 'krichmarCustom', 'genetic'
simdatadir = '../data/15mar20b_'+evolAlgorithm # folder to save sim results

num_islands = 1 # number of islands
numproc = 4 # number of cores per job
max_migrants = 1 #
migration_interval = 5
pop_size = 100 # population size per island
num_elites = pop_size/10 # num of top individuals kept each generation - maybe set to pop_size/10? 
max_generations = 1000
max_evaluations = max_generations *  num_islands * pop_size
mutation_rate = 0.4 # only for custom EC  
crossover = 0.5 # only for custom EC
targets_eval = [0] # center-out reaching target to evaluate

# parameter names and ranges
pNames = []
pRanges = []

# single target 
# pNames.append('trainTime'); pRanges.append([10*1e3,120*1e3]) #int
# pNames.append('plastConnsType'); pRanges.append([0,1,2,3]) # int
# #pNames.append('stdpFactor'); pRanges.append([0,1])
# pNames.append('RLfactor'); pRanges.append([0.5,5])
# #pNames.append('stdpwin'); pRanges.append([10,30])
# pNames.append('eligwin'); pRanges.append([50,100])
# #pNames.append('RLinterval'); pRanges.append([50,100])
# #pNames.append('maxweight'); pRanges.append([15,75])
# pNames.append('backgroundrate'); pRanges.append([30,100])
# pNames.append('backgroundrateExplor'); pRanges.append([500,1500])
# #pNames.append('minRLerror'); pRanges.append([0.0,0.01])
# pNames.append('cmdmaxrate'); pRanges.append([10,30])
# #pNames.append('cmdtimewin'); pRanges.append([50,150])
# #pNames.append('explorMovsFactor'); pRanges.append([1,10])
# #pNames.append('explorMovsDur'); pRanges.append([500,1500])

# multiple targets simultaenously
pNames.append('trainTime'); pRanges.append([40*1e3,160*1e3]) #int
pNames.append('plastConnsType'); pRanges.append([4,5,6,7]) # int
pNames.append('RLfactor'); pRanges.append([3,6])
pNames.append('backgroundrate'); pRanges.append([20,50])
pNames.append('backgroundrateExplor'); pRanges.append([300,600])
pNames.append('backgroundrateTest'); pRanges.append([30,100])
pNames.append('cmdmaxrate'); pRanges.append([10,30])
pNames.append('cmdmaxrateTest'); pRanges.append([10,30])
pNames.append('maxPMdRate'); pRanges.append([300,500])

num_inputs = len(pNames)

# Set bounds and allowed ranges for params
def bound_params(candidate, args = []):
    cBound = []
    for i,p in enumerate(candidate):
        cBound.append(max(min(p, max(pRanges[i])), min(pRanges[i])))

    # need to be integer 
    cBound[0] = round(max(min(pRanges[0], max(pRanges[0])), min(pRanges[0])))
    cBound[1] = round(max(min(pRanges[1], max(pRanges[1])), min(pRanges[1])))
  
    # fixed values from list
    #param14 = min(param14_range, key=lambda x:abs(x-c[13]))

    candidate = cBound
    return candidate


###############################################################################
### Generate new set of random values for params
###############################################################################  
def generate_rastrigin(random, args):
    size = args.get('num_inputs', 10)
    paramsRand = []
    for iparam in range(len(pNames)):
        paramsRand.append(random.uniform(min(pRanges[iparam]),max(pRanges[iparam])))

    # need to be integer 
    paramsRand[0] = round(paramsRand[0])
    paramsRand[1] = round(paramsRand[1])

    # fixed values from list
    #param[14] = min(param14_range, key=lambda x:abs(x-param14))

    return paramsRand


###############################################################################
### Observer
###############################################################################  
def my_observer(population, num_generations, num_evaluations, args):
    #ngen=num_generations
    best = max(population)
    print('{0:6} -- {1} : {2}'.format(num_generations, 
                                      best.fitness, 
                                      str(best.candidate)))

###############################################################################
### Custom mutator (nonuniform taking into account bounds)
###############################################################################  
@mutator
def nonuniform_bounds_mutation(random, candidate, args):
    """Return the mutants produced by nonuniform mutation on the candidates.
    .. Arguments:
       random -- the random number generator object
       candidate -- the candidate solution
       args -- a dictionary of keyword arguments
    Required keyword arguments in args:       
    Optional keyword arguments in args:    
    - *mutation_strength* -- the strength of the mutation, where higher
      values correspond to greater variation (default 1)
    
    """
    #bounder = args['_ec'].bounder
    #num_gens = args['_ec'].num_generations
    lower_bound = [x[0] for x in pRanges]
    upper_bound = [x[1] for x in pRanges]
    strength = args.setdefault('mutation_strength', 1)
    exponent = strength
    mutant = copy.copy(candidate)
    for i, (c, lo, hi) in enumerate(zip(candidate, lower_bound, upper_bound)):
        if random.random() <= 0.5:
            new_value = c + (hi - c) * (1.0 - random.random() ** exponent)
        else:
            new_value = c - (c - lo) * (1.0 - random.random() ** exponent)
        mutant[i] = new_value
    mutant_bounded = bound_params(mutant)
    return mutant_bounded

###############################################################################
### Parallel evaluation
###############################################################################   
def parallel_evaluation_pbs(candidates, args):
    global ngen, targets_eval
    simdatadir = args.get('simdatadir') # load params
    ngen += 1 # increase number of generations
    maxiter_wait=args.get('maxiter_wait',1000) # 
    default_error=args.get('default_error',0.3)

    #run pbs jobs
    total_jobs = 0
    for i, c in enumerate(candidates): 
        for itarget in targets_eval:
            outfilestem=simdatadir+"/gen_"+str(ngen)+"_cand_"+str(i) # set filename
            with open('%s_params'% (outfilestem), 'w') as f: # save current candidate params to file 
                pickle.dump(c, f)
            command = 'mpiexec -np %d nrniv -python -mpi main.py outfilestem="%s" targetid=%d'%(numproc, outfilestem, itarget) # set command to run
            #c[0] = 1000 # temporarily set train time to 1 second to debug !!! 
            for iparam, param in enumerate(c): # add all param names and values dynamically
                paramstring = ' %s=%r' % (pNames[iparam], param)
                command += paramstring

            output, input = popen2('qsub') # Open a pipe to the qsub command.
            job_name = outfilestem # Customize your options here
            walltime = "01:00:00"
            processors = "nodes=1:ppn=%d"%(numproc)

            if socket.gethostname()[0:2] == 'ma': 
                queueName == 'longq'
            elif socket.gethostname()[0:2] == 'do': 
                queueName == 'batch'

            job_string = """#!/bin/bash 
            #PBS -N %s
            #PBS -l walltime=%s
            #PBS -q %s
            #PBS -l %s
            #PBS -o %s.run
            #PBS -e %s.err
            cd $PBS_O_WORKDIR
            echo $PBS_O_WORKDIR
            %s""" % (job_name, walltime, queueName processors, job_name, job_name, command)

            # Send job_string to qsub
            input.write(job_string)
            input.close()

            #os.system('screen '+command) # temporarily run commands to debug !!!

            # Print your job and the response to the screen
            print job_string
            #print output.read()+": "+command
            total_jobs+=1
            sleep(0.1)

    #read results from file
    targetFitness = [[None for j in targets_eval] for i in range(len(candidates))]
    num_iters = 0
    jobs_completed=0
    while jobs_completed < total_jobs:
        #print outfilestem
        print str(jobs_completed)+" / "+str(total_jobs)+" jobs completed"
        unfinished = [[(i,j) for j,y in enumerate(x) if y is None] for i, x in enumerate(targetFitness)]<commit_msg>15mar20b evol sim; and adapted evol.py to change queue name as a function of hpc<commit_after>#! /usr/bin/env python
# runpbs_evol.py
# runs evolutionary algorithm on arm2dms model using PBS Torque in HPC

import os, sys
from numpy import mean
import csv
import copy
from inspyred.ec.variators import mutator
from random import Random
from time import time, sleep
import inspyred
import logging
from popen2 import popen2
import pickle
import multiprocessing
import Queue
import socket

ngen = -1 #global variable keeping number of generations

###############################################################################
### Simulation options
###############################################################################  
evolAlgorithm = 'evolutionStrategy' #'diffEvolution' # 'evolutionStrategy' #'krichmarCustom' #'genetic'#'particleSwarm100'#'estimationDist' #'diffEvolution' # 'evolutionStrategy' # 'krichmarCustom', 'genetic'
simdatadir = '../data/15mar20b_'+evolAlgorithm # folder to save sim results

num_islands = 1 # number of islands
numproc = 4 # number of cores per job
max_migrants = 1 #
migration_interval = 5
pop_size = 100 # population size per island
num_elites = pop_size/10 # num of top individuals kept each generation - maybe set to pop_size/10? 
max_generations = 1000
max_evaluations = max_generations *  num_islands * pop_size
mutation_rate = 0.4 # only for custom EC  
crossover = 0.5 # only for custom EC
targets_eval = [0] # center-out reaching target to evaluate

# parameter names and ranges
pNames = []
pRanges = []

# single target 
# pNames.append('trainTime'); pRanges.append([10*1e3,120*1e3]) #int
# pNames.append('plastConnsType'); pRanges.append([0,1,2,3]) # int
# #pNames.append('stdpFactor'); pRanges.append([0,1])
# pNames.append('RLfactor'); pRanges.append([0.5,5])
# #pNames.append('stdpwin'); pRanges.append([10,30])
# pNames.append('eligwin'); pRanges.append([50,100])
# #pNames.append('RLinterval'); pRanges.append([50,100])
# #pNames.append('maxweight'); pRanges.append([15,75])
# pNames.append('backgroundrate'); pRanges.append([30,100])
# pNames.append('backgroundrateExplor'); pRanges.append([500,1500])
# #pNames.append('minRLerror'); pRanges.append([0.0,0.01])
# pNames.append('cmdmaxrate'); pRanges.append([10,30])
# #pNames.append('cmdtimewin'); pRanges.append([50,150])
# #pNames.append('explorMovsFactor'); pRanges.append([1,10])
# #pNames.append('explorMovsDur'); pRanges.append([500,1500])

# multiple targets simultaenously
pNames.append('trainTime'); pRanges.append([40*1e3,160*1e3]) #int
pNames.append('plastConnsType'); pRanges.append([4,5,6,7]) # int
pNames.append('RLfactor'); pRanges.append([3,6])
pNames.append('backgroundrate'); pRanges.append([20,50])
pNames.append('backgroundrateExplor'); pRanges.append([300,600])
pNames.append('backgroundrateTest'); pRanges.append([30,100])
pNames.append('cmdmaxrate'); pRanges.append([10,30])
pNames.append('cmdmaxrateTest'); pRanges.append([10,30])
pNames.append('maxPMdRate'); pRanges.append([300,500])

num_inputs = len(pNames)

# Set bounds and allowed ranges for params
def bound_params(candidate, args = []):
    cBound = []
    for i,p in enumerate(candidate):
        cBound.append(max(min(p, max(pRanges[i])), min(pRanges[i])))

    # need to be integer 
    cBound[0] = round(max(min(pRanges[0], max(pRanges[0])), min(pRanges[0])))
    cBound[1] = round(max(min(pRanges[1], max(pRanges[1])), min(pRanges[1])))
  
    # fixed values from list
    #param14 = min(param14_range, key=lambda x:abs(x-c[13]))

    candidate = cBound
    return candidate


###############################################################################
### Generate new set of random values for params
###############################################################################  
def generate_rastrigin(random, args):
    size = args.get('num_inputs', 10)
    paramsRand = []
    for iparam in range(len(pNames)):
        paramsRand.append(random.uniform(min(pRanges[iparam]),max(pRanges[iparam])))

    # need to be integer 
    paramsRand[0] = round(paramsRand[0])
    paramsRand[1] = round(paramsRand[1])

    # fixed values from list
    #param[14] = min(param14_range, key=lambda x:abs(x-param14))

    return paramsRand


###############################################################################
### Observer
###############################################################################  
def my_observer(population, num_generations, num_evaluations, args):
    #ngen=num_generations
    best = max(population)
    print('{0:6} -- {1} : {2}'.format(num_generations, 
                                      best.fitness, 
                                      str(best.candidate)))

###############################################################################
### Custom mutator (nonuniform taking into account bounds)
###############################################################################  
@mutator
def nonuniform_bounds_mutation(random, candidate, args):
    """Return the mutants produced by nonuniform mutation on the candidates.
    .. Arguments:
       random -- the random number generator object
       candidate -- the candidate solution
       args -- a dictionary of keyword arguments
    Required keyword arguments in args:       
    Optional keyword arguments in args:    
    - *mutation_strength* -- the strength of the mutation, where higher
      values correspond to greater variation (default 1)
    
    """
    #bounder = args['_ec'].bounder
    #num_gens = args['_ec'].num_generations
    lower_bound = [x[0] for x in pRanges]
    upper_bound = [x[1] for x in pRanges]
    strength = args.setdefault('mutation_strength', 1)
    exponent = strength
    mutant = copy.copy(candidate)
    for i, (c, lo, hi) in enumerate(zip(candidate, lower_bound, upper_bound)):
        if random.random() <= 0.5:
            new_value = c + (hi - c) * (1.0 - random.random() ** exponent)
        else:
            new_value = c - (c - lo) * (1.0 - random.random() ** exponent)
        mutant[i] = new_value
    mutant_bounded = bound_params(mutant)
    return mutant_bounded

###############################################################################
### Parallel evaluation
###############################################################################   
def parallel_evaluation_pbs(candidates, args):
    global ngen, targets_eval
    simdatadir = args.get('simdatadir') # load params
    ngen += 1 # increase number of generations
    maxiter_wait=args.get('maxiter_wait',1000) # 
    default_error=args.get('default_error',0.3)

    #run pbs jobs
    total_jobs = 0
    for i, c in enumerate(candidates): 
        for itarget in targets_eval:
            outfilestem=simdatadir+"/gen_"+str(ngen)+"_cand_"+str(i) # set filename
            with open('%s_params'% (outfilestem), 'w') as f: # save current candidate params to file 
                pickle.dump(c, f)
            command = 'mpiexec -np %d nrniv -python -mpi main.py outfilestem="%s" targetid=%d'%(numproc, outfilestem, itarget) # set command to run
            #c[0] = 1000 # temporarily set train time to 1 second to debug !!! 
            for iparam, param in enumerate(c): # add all param names and values dynamically
                paramstring = ' %s=%r' % (pNames[iparam], param)
                command += paramstring

            output, input = popen2('qsub') # Open a pipe to the qsub command.
            job_name = outfilestem # Customize your options here
            walltime = "01:00:00"
            processors = "nodes=1:ppn=%d"%(numproc)

            queueName = 'default'
            if socket.gethostname()[0:2] == 'ma': 
                queueName == 'longq'
            elif socket.gethostname()[0:2] == 'do': 
                queueName == 'batch'

            job_string = """#!/bin/bash 
            #PBS -N %s
            #PBS -l walltime=%s
            #PBS -q %s
            #PBS -l %s
            #PBS -o %s.run
            #PBS -e %s.err
            cd $PBS_O_WORKDIR
            echo $PBS_O_WORKDIR
            %s""" % (job_name, walltime, queueName, processors, job_name, job_name, command)

            # Send job_string to qsub
            input.write(job_string)
            input.close()

            #os.system('screen '+command) # temporarily run commands to debug !!!

            # Print your job and the response to the screen
            print job_string
            #print output.read()+": "+command
            total_jobs+=1
            sleep(0.1)

    #read results from file
    targetFitness = [[None for j in targets_eval] for i in range(len(candidates))]
    num_iters = 0
    jobs_completed=0
    while jobs_completed < total_jobs:
        #print outfilestem
        print str(jobs_completed)+" / "+str(total_jobs)+" jobs completed"
        unfinished = [[(i,j) for j,y in enumerate(x) if y is None] for i, x in enumerate(targetFitness)]
{'fraction_non_alphanumeric': 0.2076923076923077, 'fraction_numerical': 0.024886877828054297, 'mean_word_length': 3.718708299973312, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '703843', 'n_tokens_mistral': 5703, 'n_tokens_neox': 5268, 'n_words': 1315}