<commit_before>import argparse
from tfutils.utils import frozendict, format_devices
import os
import logging
import tensorflow as tf
import copy
import numpy as np
from tfutils.optimizer import ClipOptimizer, MinibatchOptimizer
import tfutils.utils as utils
from tfutils.utils import aggregate_outputs
import pdb
import inspect
from six import string_types

from tfutils.defaults import \
        BRANCH_QUEUE_NAME, DEFAULT_HOST, \
        DEFAULT_DEVICES, DEFAULT_MODEL_SEED, DEFAULT_DONT_RUN, \
        DEFAULT_SKIP_CHECK, DEFAULT_LOG_DEVICE_PLACEMENT, \
        DEFAULT_TRAIN_THRES_LOSS, DEFAULT_LOAD_PARAMS, \
        DEFAULT_LEARNING_RATE_PARAMS, DEFAULT_LOSS_PARAMS, \
        DEFAULT_OPTIMIZER_PARAMS, DEFAULT_SAVE_PARAMS, DEFAULT_PARAMS, \
        train_loop, mean_and_reg_loss
from tfutils.multi_gpu import easy_variable_mgr as variable_mgr


if 'TFUTILS_LOGFILE' in os.environ:
    logging.basicConfig(filename=os.environ['TFUTILS_LOGFILE'])
    print ("USING LOGFILE: %s" % os.environ['TFUTILS_LOGFILE'])
else:
    logging.basicConfig()
log = logging.getLogger('tfutils')
log.setLevel('DEBUG')


def get_model_base(inputs, func, seed=0, train=False, **old_model_params):
    model_params = copy.deepcopy(old_model_params)
    model_params['seed'] = seed
    model_params['train'] = train
    # Your model function should return:
    #   1. the outputs
    #   2. onfiguration files you want to put to database
    outputs, cfg_to_database = func(inputs=inputs,
                              **model_params)
    model_params['func'] = func
    model_params['cfg_to_database'] = cfg_to_database
    # For supporting previous behaviors
    model_params['cfg_final'] = cfg_to_database
    return model_params, outputs


def get_model(inputs, model_params, var_manager=None, param=None, trarg=None):
    """Return model and any other targets (loss + optimizer) specified.

    Args:
        inputs (tf.Operation): Model inputs
        model_params (dict): Specifies model configuration and must contain:
            'devices' (list): device specs (e.g. '/gpu:0')
            'train' (bool): whether getting model for training
        param (None, optional): Description.
        trarg (None, optional): Description.

    Returns:
        tuple: Description.

    """
    devices = model_params['devices']
    num_gpus = model_params['num_gpus']
    model_prefix = model_params['prefix']

    # var_manager is used for variable management in multiple gpu training
    if not var_manager:
        trainable_scopes = model_params['trainable_scopes']
        if isinstance(trainable_scopes, string_types):<commit_msg>Model params fix<commit_after>import argparse
from tfutils.utils import frozendict, format_devices
import os
import logging
import tensorflow as tf
import copy
import numpy as np
from tfutils.optimizer import ClipOptimizer, MinibatchOptimizer
import tfutils.utils as utils
from tfutils.utils import aggregate_outputs
import pdb
import inspect
from six import string_types

from tfutils.defaults import \
        BRANCH_QUEUE_NAME, DEFAULT_HOST, \
        DEFAULT_DEVICES, DEFAULT_MODEL_SEED, DEFAULT_DONT_RUN, \
        DEFAULT_SKIP_CHECK, DEFAULT_LOG_DEVICE_PLACEMENT, \
        DEFAULT_TRAIN_THRES_LOSS, DEFAULT_LOAD_PARAMS, \
        DEFAULT_LEARNING_RATE_PARAMS, DEFAULT_LOSS_PARAMS, \
        DEFAULT_OPTIMIZER_PARAMS, DEFAULT_SAVE_PARAMS, DEFAULT_PARAMS, \
        train_loop, mean_and_reg_loss
from tfutils.multi_gpu import easy_variable_mgr as variable_mgr


if 'TFUTILS_LOGFILE' in os.environ:
    logging.basicConfig(filename=os.environ['TFUTILS_LOGFILE'])
    print ("USING LOGFILE: %s" % os.environ['TFUTILS_LOGFILE'])
else:
    logging.basicConfig()
log = logging.getLogger('tfutils')
log.setLevel('DEBUG')


def get_model_base(inputs, func, seed=0, train=False, **old_model_params):
    model_params = copy.deepcopy(old_model_params)
    model_params['seed'] = seed
    model_params['train'] = train
    # Your model function should return:
    #   1. the outputs
    #   2. onfiguration files you want to put to database
    outputs, cfg_to_database = func(inputs=inputs,
                              **model_params)
    model_params['func'] = func
    model_params['cfg_to_database'] = cfg_to_database
    # For supporting previous behaviors
    model_params['cfg_final'] = cfg_to_database
    return model_params, outputs


def get_model(inputs, model_params, var_manager=None, param=None, trarg=None):
    """Return model and any other targets (loss + optimizer) specified.

    Args:
        inputs (tf.Operation): Model inputs
        model_params (dict): Specifies model configuration and must contain:
            'devices' (list): device specs (e.g. '/gpu:0')
            'train' (bool): whether getting model for training
        param (None, optional): Description.
        trarg (None, optional): Description.

    Returns:
        tuple: Description.

    """
    devices = model_params['devices']
    num_gpus = model_params['num_gpus']
    model_prefix = model_params['prefix']

    # var_manager is used for variable management in multiple gpu training
    if not var_manager:
        trainable_scopes = model_params.pop('trainable_scopes')
        if isinstance(trainable_scopes, string_types):
{'fraction_non_alphanumeric': 0.08217349857006673, 'fraction_numerical': 0.001525262154432793, 'mean_word_length': 3.8574074074074076, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '720309', 'n_tokens_mistral': 1575, 'n_tokens_neox': 1578, 'n_words': 409}