<commit_before>"""
batch.py 

Class to setup and run batch simulations

Contributors: <EMAIL>
"""


import datetime
from itertools import izip, product
from popen2 import popen2
from time import sleep
import imp
from netpyne import specs
from neuron import h
pc = h.ParallelContext() # use bulletin board master/slave
if pc.id()==0: pc.master_works_on_jobs(0) 

# function to run single job using ParallelContext bulletin board (master/slave) 
# func needs to be outside of class
def runJob(script, cfgSavePath, netParamsSavePath):
    from subprocess import Popen, PIPE

    print '\nJob in rank id: ',pc.id()
    command = 'nrniv %s simConfig=%s netParams=%s' % (script, cfgSavePath, netParamsSavePath) 
    print command+'\n'
    proc = Popen(command.split(' '), stdout=PIPE, stderr=PIPE)
    print proc.stdout.read()

def tupleToStr (obj):
    if type(obj) == list:
        for item in obj:
            if type(item) in [list, dict]:
                tupleToStr(item)
    elif type(obj) == dict:
        for key,val in obj.iteritems():
            if type(val) in [list, dict]:
                tupleToStr(val)
            elif type(key) == tuple:
                obj[str(key)] = obj.pop(key) 
    return obj


class Batch(object):

    def __init__(self, cfgFile='cfg.py', netParamsFile='netParams.py', params=None, grouped=None, initCfg={}):
        self.batchLabel = 'batch_'+str(datetime.date.today())
        self.cfgFile = cfgFile
        self.initCfg = initCfg
        self.netParamsFile = netParamsFile
        self.saveFolder = '/'+self.batchLabel
        self.method = 'grid'
        self.runCfg = {}
        self.params = []
        if params:
            for k,v in params.iteritems():
                self.params.append({'label': k, 'values': v})
        if grouped:
            for p in self.params:
                if p['label'] in grouped: p['group'] = True
    
    def save(self, filename):
        import os
        basename = os.path.basename(filename)
        folder = filename.split(basename)[0]
        ext = basename.split('.')[1]

        # make dir
        try:
            os.mkdir(folder)
        except OSError:
            if not os.path.exists(folder):
                print ' Could not create', folder

        dataSave = {'batch': tupleToStr(self.__dict__)}
        if ext == 'json':
            import json
            #from json import encoder
            #encoder.FLOAT_REPR = lambda o: format(o, '.12g')
            print('Saving batch to %s ... ' % (filename))
            with open(filename, 'w') as fileObj:
                json.dump(dataSave, fileObj, indent=4, sort_keys=True)


    def setCfgNestedParam(self, paramLabel, paramVal):
        if isinstance(paramLabel, tuple):
            container = self.cfg<commit_msg>Added option to set grouped params in Batch constructor<commit_after>"""
batch.py 

Class to setup and run batch simulations

Contributors: <EMAIL>
"""


import datetime
from itertools import izip, product
from popen2 import popen2
from time import sleep
import imp
from netpyne import specs
from neuron import h
pc = h.ParallelContext() # use bulletin board master/slave
if pc.id()==0: pc.master_works_on_jobs(0) 

# function to run single job using ParallelContext bulletin board (master/slave) 
# func needs to be outside of class
def runJob(script, cfgSavePath, netParamsSavePath):
    from subprocess import Popen, PIPE

    print '\nJob in rank id: ',pc.id()
    command = 'nrniv %s simConfig=%s netParams=%s' % (script, cfgSavePath, netParamsSavePath) 
    print command+'\n'
    proc = Popen(command.split(' '), stdout=PIPE, stderr=PIPE)
    print proc.stdout.read()

def tupleToStr (obj):
    if type(obj) == list:
        for item in obj:
            if type(item) in [list, dict]:
                tupleToStr(item)
    elif type(obj) == dict:
        for key,val in obj.iteritems():
            if type(val) in [list, dict]:
                tupleToStr(val)
            elif type(key) == tuple:
                obj[str(key)] = obj.pop(key) 
    return obj


class Batch(object):

    def __init__(self, cfgFile='cfg.py', netParamsFile='netParams.py', params=None, groupedParams=None, initCfg={}):
        self.batchLabel = 'batch_'+str(datetime.date.today())
        self.cfgFile = cfgFile
        self.initCfg = initCfg
        self.netParamsFile = netParamsFile
        self.saveFolder = '/'+self.batchLabel
        self.method = 'grid'
        self.runCfg = {}
        self.params = []
        if params:
            for k,v in params.iteritems():
                self.params.append({'label': k, 'values': v})
        if groupedParams:
            for p in self.params:
                if p['label'] in groupedParams: p['group'] = True
    
    def save(self, filename):
        import os
        basename = os.path.basename(filename)
        folder = filename.split(basename)[0]
        ext = basename.split('.')[1]

        # make dir
        try:
            os.mkdir(folder)
        except OSError:
            if not os.path.exists(folder):
                print ' Could not create', folder

        dataSave = {'batch': tupleToStr(self.__dict__)}
        if ext == 'json':
            import json
            #from json import encoder
            #encoder.FLOAT_REPR = lambda o: format(o, '.12g')
            print('Saving batch to %s ... ' % (filename))
            with open(filename, 'w') as fileObj:
                json.dump(dataSave, fileObj, indent=4, sort_keys=True)


    def setCfgNestedParam(self, paramLabel, paramVal):
        if isinstance(paramLabel, tuple):
            container = self.cfg
{'fraction_non_alphanumeric': 0.1145311381531854, 'fraction_numerical': 0.0032211882605583395, 'mean_word_length': 2.3169139465875372, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '716673', 'n_tokens_mistral': 1706, 'n_tokens_neox': 1646, 'n_words': 485}