<commit_before>#Copyright (C) 2017 <NAME> <<EMAIL>>
#
#This Source Code Form is subject to the terms of the Mozilla Public
#License, v. 2.0. If a copy of the MPL was not distributed with this
#file, you can obtain one at http://mozilla.org/MPL/2.0/.
#Exhibit B is not attached; this software is compatible with the
#licenses expressed under Section 1.12 of the MPL v2.
"""Utils for models creation"""

import re
import tensorflow as tf
from .collections import MODEL_SUMMARIES, REQUIRED_NON_TRAINABLES


def legalize_name(name):
    """Made name a legal name to be used in tensorflow summaries
    Args:
        name: string
    Returns:
        name_legal
    """
    return re.sub(r"[^\w|/]", "_", name)


def tf_log(summary, collection=MODEL_SUMMARIES):
    """Add tf.summary object to collection named collection"""
    tf.add_to_collection(collection, summary)


def variables_to_save(add_list=[]):
    """Returns a list of variables to save.
    add_list variables are always added to the list
    """
    return tf.trainable_variables() + tf.get_collection_ref(
        REQUIRED_NON_TRAINABLES) + add_list


def variables_to_restore(add_list=[], exclude_scope_list=[]):
    """Returns a list of variables to restore to made the model working
    properly.
    The list is made by the trainable variables + required non trainable variables
    such as statistics of batch norm layers.
    Remove from the list variables that are in the exclude_scope_list.
    Add variables in the add_list<commit_msg>models/utils.py: add training_process_variables. Save and restore them<commit_after>#Copyright (C) 2017 <NAME> <<EMAIL>>
#
#This Source Code Form is subject to the terms of the Mozilla Public
#License, v. 2.0. If a copy of the MPL was not distributed with this
#file, you can obtain one at http://mozilla.org/MPL/2.0/.
#Exhibit B is not attached; this software is compatible with the
#licenses expressed under Section 1.12 of the MPL v2.
"""Utils for models creation"""

import re
import tensorflow as tf
from .collections import MODEL_SUMMARIES, REQUIRED_NON_TRAINABLES


def legalize_name(name):
    """Made name a legal name to be used in tensorflow summaries
    Args:
        name: string
    Returns:
        name_legal
    """
    return re.sub(r"[^\w|/]", "_", name)


def tf_log(summary, collection=MODEL_SUMMARIES):
    """Add tf.summary object to collection named collection"""
    tf.add_to_collection(collection, summary)


def training_process_variables():
    """global variables - trainable variables:
    it contains the variable defined by the optimizer and the others
    defined durint the training process.
    Those variables are useful when restoring a training process.
    Those variables are not trainable. can be saved and restored.
    """

    return [
        variable for variable in tf.global_variables()
        if variable.name not in (var.name for var in tf.trainable_variables())
    ]


def variables_to_save(add_list=[]):
    """Returns a list of variables to save.
    add_list variables are always added to the list
    """
    return tf.trainable_variables() + tf.get_collection_ref(
        REQUIRED_NON_TRAINABLES) + add_list + training_process_variables()


def variables_to_restore(add_list=[], exclude_scope_list=[]):
    """Returns a list of variables to restore to made the model working
    properly.
    The list is made by the trainable variables + required non trainable variables
    such as statistics of batch norm layers.
    Remove from the list variables that are in the exclude_scope_list.
    Add variables in the add_list
{'fraction_non_alphanumeric': 0.07857341877960435, 'fraction_numerical': 0.006687099470604625, 'mean_word_length': 3.7612732095490715, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '617291', 'n_tokens_mistral': 1054, 'n_tokens_neox': 1004, 'n_words': 420}