<commit_before>'''
This module contains two sections:
1. Experiment stage definitions for experiments.
2. Parameter group definitions for experiments.

@author: mgormley
'''

import sys
import os
import getopt
import math
import tempfile
import stat
import shlex
import subprocess
from subprocess import Popen
from optparse import OptionParser
from pypipeline.util import get_new_file, sweep_mult, fancify_cmd, frange
from pypipeline.util import head_sentences
import platform
from glob import glob
from pypipeline.experiment_runner import ExpParamsRunner, get_subset
from pypipeline import experiment_runner
from pypipeline import pipeline
import re
import random
from pypipeline.pipeline import write_script, RootStage, Stage
import multiprocessing
from experiments.exp_util import *
from experiments.path_defs import *
from experiments.srl_stages import ScrapeSrl, SrlExpParams

# ---------------------------- Definitions Classes ----------------------------------

class ParamDefinitions():
    ''' This class exposes only a single public method (get_param_groups_and_lists),
        which returns a single ParamGroups object and a single ParamGroupLists object.
        These objects are defined by calling a number of private methods. Those defining
        parameter groups are prefixed by '_define_groups'. Those defining parameter group
        lists are prefixed by '_define_lists'.
    '''
    
    def __init__(self, options):
        self.root_dir = os.path.abspath(get_root_dir())
        self.fast = options.fast
        self.expname = options.expname
        self.path_defs = PathDefinitions(options)
        self.queue = options.queue
        self.big_machine = (multiprocessing.cpu_count() > 2)
        
    def get_param_groups_and_lists_and_paths(self):        
        g = ParamGroups()
        l = ParamGroupLists()
        p = self.path_defs.get_paths()
        g.langs = {} # Language-specific parameter groups. (always reference as "gl")
        l.langs = {} # Language-specific parameter lists.  (always reference as "ll")
        for lang_short in p.c09_lang_short_names + p.cx_lang_short_names + p.c07_lang_short_names:            
            g.langs[lang_short] = ParamGroups()
            l.langs[lang_short] = ParamGroups()
        
        # Define all the parameter groups.
        self._define_groups_features(g, p)
        self._define_groups_optimizer(g)
        self._define_groups_model(g)
        self._define_groups_parser_output(g, p)
        
        # Define the special group (g.defaults) which may 
        # utilize the parameter groups defined above.
        self._define_groups_defaults(g)

        # Define all the parameter lists
        self._define_lists_features(g, l)
        self._define_lists_optimizer(g, l)
        self._define_lists_model(g, l)
        self._define_lists_parser_output(g, l, p)
        self._define_lists_parse_and_srl(g, l, p)
        
        return g, l, p

    def _define_groups_defaults(self, g):
        g.defaults = SrlExpParams()
        
        g.defaults.set("expname", self.expname, False, False)
        g.defaults.set("timeoutSeconds", 48*60*60, incl_arg=False, incl_name=False)
        g.defaults.set("work_mem_megs", 1.5*1024, incl_arg=False, incl_name=False)
        g.defaults.update(seed=random.getrandbits(63))
        if self.queue:
            threads = 15
        elif self.big_machine:
            threads = 2
        else:
            threads = 1
        g.defaults.set("threads", threads, incl_name=False)
        g.defaults.set("sgdBatchSize", 30)<commit_msg>Updating default number of threads and batch size<commit_after>'''
This module contains two sections:
1. Experiment stage definitions for experiments.
2. Parameter group definitions for experiments.

@author: mgormley
'''

import sys
import os
import getopt
import math
import tempfile
import stat
import shlex
import subprocess
from subprocess import Popen
from optparse import OptionParser
from pypipeline.util import get_new_file, sweep_mult, fancify_cmd, frange
from pypipeline.util import head_sentences
import platform
from glob import glob
from pypipeline.experiment_runner import ExpParamsRunner, get_subset
from pypipeline import experiment_runner
from pypipeline import pipeline
import re
import random
from pypipeline.pipeline import write_script, RootStage, Stage
import multiprocessing
from experiments.exp_util import *
from experiments.path_defs import *
from experiments.srl_stages import ScrapeSrl, SrlExpParams

# ---------------------------- Definitions Classes ----------------------------------

class ParamDefinitions():
    ''' This class exposes only a single public method (get_param_groups_and_lists),
        which returns a single ParamGroups object and a single ParamGroupLists object.
        These objects are defined by calling a number of private methods. Those defining
        parameter groups are prefixed by '_define_groups'. Those defining parameter group
        lists are prefixed by '_define_lists'.
    '''
    
    def __init__(self, options):
        self.root_dir = os.path.abspath(get_root_dir())
        self.fast = options.fast
        self.expname = options.expname
        self.path_defs = PathDefinitions(options)
        self.queue = options.queue
        self.big_machine = (multiprocessing.cpu_count() > 2)
        
    def get_param_groups_and_lists_and_paths(self):        
        g = ParamGroups()
        l = ParamGroupLists()
        p = self.path_defs.get_paths()
        g.langs = {} # Language-specific parameter groups. (always reference as "gl")
        l.langs = {} # Language-specific parameter lists.  (always reference as "ll")
        for lang_short in p.c09_lang_short_names + p.cx_lang_short_names + p.c07_lang_short_names:            
            g.langs[lang_short] = ParamGroups()
            l.langs[lang_short] = ParamGroups()
        
        # Define all the parameter groups.
        self._define_groups_features(g, p)
        self._define_groups_optimizer(g)
        self._define_groups_model(g)
        self._define_groups_parser_output(g, p)
        
        # Define the special group (g.defaults) which may 
        # utilize the parameter groups defined above.
        self._define_groups_defaults(g)

        # Define all the parameter lists
        self._define_lists_features(g, l)
        self._define_lists_optimizer(g, l)
        self._define_lists_model(g, l)
        self._define_lists_parser_output(g, l, p)
        self._define_lists_parse_and_srl(g, l, p)
        
        return g, l, p

    def _define_groups_defaults(self, g):
        g.defaults = SrlExpParams()
        
        g.defaults.set("expname", self.expname, False, False)
        g.defaults.set("timeoutSeconds", 48*60*60, incl_arg=False, incl_name=False)
        g.defaults.set("work_mem_megs", 1.5*1024, incl_arg=False, incl_name=False)
        g.defaults.update(seed=random.getrandbits(63))
        if self.queue:
            threads = 7
        elif self.big_machine:
            threads = 2
        else:
            threads = 1
        g.defaults.set("threads", threads, incl_name=False)
        g.defaults.set("sgdBatchSize", 14)
{'fraction_non_alphanumeric': 0.09158173949462516, 'fraction_numerical': 0.007399134440876727, 'mean_word_length': 2.822838847385272, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3290470', 'n_tokens_mistral': 2064, 'n_tokens_neox': 2017, 'n_words': 564}