<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function

import os
import time

import numpy as np
import tensorflow as tf
from six.moves import range

from engine.input_buffer import TrainEvalInputBuffer
from engine.spatial_location_check import SpatialLocationCheckLayer
from engine.selective_sampler import SelectiveSampler
from engine.uniform_sampler import UniformSampler
from layer.loss import LossFunction
from utilities import misc_common as util
from utilities.input_placeholders import ImagePatch
import engine.logging

np.random.seed(seed=int(time.time()))


def run(net_class, param, volume_loader, device_str):
    # construct graph
    graph = tf.Graph()
    with graph.as_default(), tf.device('/cpu:0'):
        # defines a training element
        patch_holder = ImagePatch(
            spatial_rank=param.spatial_rank,
            image_size=param.image_size,
            label_size=param.label_size,
            weight_map_size=param.w_map_size,
            image_dtype=tf.float32,
            label_dtype=tf.int64,
            weight_map_dtype=tf.float32,
            num_image_modality=volume_loader.num_modality(0),
            num_label_modality=volume_loader.num_modality(1),
            num_weight_map=volume_loader.num_modality(2))
        # defines data augmentation for training
        augmentations = []
        if param.rotation:
            from layer.rand_rotation import RandomRotationLayer
            augmentations.append(RandomRotationLayer(
                min_angle=param.min_angle,
                max_angle=param.max_angle))
        if param.spatial_scaling:
            from layer.rand_spatial_scaling import RandomSpatialScalingLayer
            augmentations.append(RandomSpatialScalingLayer(
                min_percentage=param.min_percentage,
                max_percentage=param.max_percentage))
        # defines how to generate samples of the training element from volume
        if param.window_sampling == 'uniform':
            sampler = UniformSampler(patch=patch_holder,
                                     volume_loader=volume_loader,
                                     patch_per_volume=param.sample_per_volume,
                                     data_augmentation_methods=augmentations,
                                     name='uniform_sampler')
        elif param.window_sampling == 'selective':
            # TODO check param, this is for segmentation problems only
            spatial_location_check = SpatialLocationCheckLayer(
                compulsory=((0), (0)),
                minimum_ratio=param.min_sampling_ratio,
                min_numb_labels=param.min_numb_labels,
                padding=param.border,
                name='spatial_location_check')
            sampler = SelectiveSampler(
                patch=patch_holder,
                volume_loader=volume_loader,
                spatial_location_check=spatial_location_check,
                data_augmentation_methods=None,
                patch_per_volume=param.sample_per_volume,
                name="selective_sampler")

        w_regularizer = None
        b_regularizer = None
        if param.reg_type.lower() == 'l2':
            from tensorflow.contrib.layers.python.layers import regularizers
            w_regularizer = regularizers.l2_regularizer(param.decay)
            b_regularizer = regularizers.l2_regularizer(param.decay)
        elif param.reg_type.lower() == 'l1':
            from tensorflow.contrib.layers.python.layers import regularizers
            w_regularizer = regularizers.l1_regularizer(param.decay)
            b_regularizer = regularizers.l1_regularizer(param.decay)
        net = net_class(num_classes=param.num_classes,
                        w_regularizer=w_regularizer,
                        b_regularizer=b_regularizer,
                        acti_func=param.activation_function)
        loss_func = LossFunction(n_class=param.num_classes,
                                 loss_type=param.loss_type)
        # construct train queue
        train_batch_runner = TrainEvalInputBuffer(
            batch_size=param.batch_size,
            capacity=max(param.queue_length, param.batch_size),
            sampler=sampler,
            shuffle=True)
        # optimizer
        train_step = tf.train.AdamOptimizer(learning_rate=param.lr)
        tower_grads = []
        train_pairs = train_batch_runner.pop_batch_op
        images, labels = train_pairs['images'], train_pairs['labels']
        if "weight_maps" in train_pairs:
            weight_maps = train_pairs['weight_maps']
        else:
            weight_maps = None
        # Scalar summaries for the console are averaged over GPU runs
        console_outputs=graph.get_collection_ref(engine.logging.CONSOLE)
        console_outputs_cache=console_outputs[:]
        console_outputs.clear()
        tower_console_outputs=[]
        for i in range(0, max(param.num_gpus, 1)):
            with tf.device("/{}:{}".format(device_str, i)):
                predictions = net(images, is_training=True)
                loss = loss_func(predictions, labels, weight_maps)
                if param.decay > 0:
                    reg_losses = graph.get_collection(
                        tf.GraphKeys.REGULARIZATION_LOSSES)
                    reg_loss = tf.reduce_mean([tf.reduce_mean(reg_loss)
                                               for reg_loss in reg_losses])
                    loss = loss + reg_loss
                tf.summary.scalar('loss',loss,[engine.logging.CONSOLE,engine.logging.LOG])

                ##################
                # This should probably be refactored into an application class
                if param.application_type == 'segmentation':
                  # TODO compute miss for dfferent target types
                  miss = tf.reduce_mean(tf.cast(
                          tf.not_equal(tf.argmax(predictions, -1), labels[..., 0]),
                          dtype=tf.float32))
                  tf.summary.scalar('miss',miss,[engine.logging.CONSOLE,engine.logging.LOG])
                ################## 

                # record and clear summaries
                console_outputs=graph.get_collection_ref(engine.logging.CONSOLE)
                tower_console_outputs.append(console_outputs[:])
                console_outputs.clear()
                
                grads = train_step.compute_gradients(loss)
                tower_grads.append(grads)
                # note: only use batch stats from one GPU for batch_norm
                if i == 0:
                    bn_updates = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
        ave_grads = util.average_grads(tower_grads)
        apply_grad_op = train_step.apply_gradients(ave_grads)

        # Add averaged summaries
        console_outputs=graph.get_collection_ref(engine.logging.CONSOLE)
        console_outputs+=console_outputs_cache
        if len(tower_console_outputs)>1:
          for replicated_output in zip(*tower_console_outputs):
            avg=tf.reduce_mean([o.op.inputs[1] for o in replicated_output])
            tf.summary.scalar(replicated_output[0].op.name+'_avg', avg,[engine.logging.CONSOLE,engine.logging.LOG])
        # Track the moving averages of all trainable variables.
        variable_averages = tf.train.ExponentialMovingAverage(0.9)
        var_averages_op = variable_averages.apply(tf.trainable_variables())
        # batch norm variables moving mean and var
        batchnorm_updates_op = tf.group(*bn_updates)
        # primary operations
        init_op = tf.global_variables_initializer()
        train_op = tf.group(apply_grad_op,
                            var_averages_op,
                            batchnorm_updates_op)
        logged_summaries = list(set([s for c in [engine.logging.LOG,engine.logging.CONSOLE] for s in tf.get_collection(c)]))
        write_summary_op = tf.summary.merge(logged_summaries)
        # saver
        variables_to_restore = variable_averages.variables_to_restore()
        saver = tf.train.Saver(max_to_keep=20, var_list=variables_to_restore)
        tf.Graph.finalize(graph)
    # run session
    config = tf.ConfigProto()
    config.log_device_placement = False
    config.allow_soft_placement = True
    # config.gpu_options.allow_growth = True
    start_time = time.time()<commit_msg>Added name_scopes to structure output in tensorboard<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function

import os
import time

import numpy as np
import tensorflow as tf
from six.moves import range

from engine.input_buffer import TrainEvalInputBuffer
from engine.spatial_location_check import SpatialLocationCheckLayer
from engine.selective_sampler import SelectiveSampler
from engine.uniform_sampler import UniformSampler
from layer.loss import LossFunction
from utilities import misc_common as util
from utilities.input_placeholders import ImagePatch
import engine.logging

np.random.seed(seed=int(time.time()))


def run(net_class, param, volume_loader, device_str):
    # construct graph
    graph = tf.Graph()
    with graph.as_default(), tf.device('/cpu:0'):
        # defines a training element
        patch_holder = ImagePatch(
            spatial_rank=param.spatial_rank,
            image_size=param.image_size,
            label_size=param.label_size,
            weight_map_size=param.w_map_size,
            image_dtype=tf.float32,
            label_dtype=tf.int64,
            weight_map_dtype=tf.float32,
            num_image_modality=volume_loader.num_modality(0),
            num_label_modality=volume_loader.num_modality(1),
            num_weight_map=volume_loader.num_modality(2))
        # defines data augmentation for training
        augmentations = []
        if param.rotation:
            from layer.rand_rotation import RandomRotationLayer
            augmentations.append(RandomRotationLayer(
                min_angle=param.min_angle,
                max_angle=param.max_angle))
        if param.spatial_scaling:
            from layer.rand_spatial_scaling import RandomSpatialScalingLayer
            augmentations.append(RandomSpatialScalingLayer(
                min_percentage=param.min_percentage,
                max_percentage=param.max_percentage))
        # defines how to generate samples of the training element from volume
        with tf.name_scope('Sampling'):
            if param.window_sampling == 'uniform':
                sampler = UniformSampler(patch=patch_holder,
                                        volume_loader=volume_loader,
                                        patch_per_volume=param.sample_per_volume,
                                        data_augmentation_methods=augmentations,
                                        name='uniform_sampler')
            elif param.window_sampling == 'selective':
                # TODO check param, this is for segmentation problems only
                spatial_location_check = SpatialLocationCheckLayer(
                    compulsory=((0), (0)),
                    minimum_ratio=param.min_sampling_ratio,
                    min_numb_labels=param.min_numb_labels,
                    padding=param.border,
                    name='spatial_location_check')
                sampler = SelectiveSampler(
                    patch=patch_holder,
                    volume_loader=volume_loader,
                    spatial_location_check=spatial_location_check,
                    data_augmentation_methods=None,
                    patch_per_volume=param.sample_per_volume,
                    name="selective_sampler")
        w_regularizer = None
        b_regularizer = None
        if param.reg_type.lower() == 'l2':
            from tensorflow.contrib.layers.python.layers import regularizers
            w_regularizer = regularizers.l2_regularizer(param.decay)
            b_regularizer = regularizers.l2_regularizer(param.decay)
        elif param.reg_type.lower() == 'l1':
            from tensorflow.contrib.layers.python.layers import regularizers
            w_regularizer = regularizers.l1_regularizer(param.decay)
            b_regularizer = regularizers.l1_regularizer(param.decay)
        net = net_class(num_classes=param.num_classes,
                            w_regularizer=w_regularizer,
                            b_regularizer=b_regularizer,
                            acti_func=param.activation_function)
        loss_func = LossFunction(n_class=param.num_classes,
                                loss_type=param.loss_type)
        # construct train queue
        with tf.name_scope('DataQueue'):
            train_batch_runner = TrainEvalInputBuffer(
                batch_size=param.batch_size,
                capacity=max(param.queue_length, param.batch_size),
                sampler=sampler,
                shuffle=True)
        # optimizer
        with tf.name_scope('Optimizer'):
            train_step = tf.train.AdamOptimizer(learning_rate=param.lr)
        tower_grads = []
        train_pairs = train_batch_runner.pop_batch_op
        images, labels = train_pairs['Sampling/images'], train_pairs['Sampling/labels']
        if "weight_maps" in train_pairs:
            weight_maps = train_pairs['Sampling/weight_maps']
        else:
            weight_maps = None
        # Scalar summaries for the console are averaged over GPU runs
        console_outputs=graph.get_collection_ref(engine.logging.CONSOLE)
        console_outputs_cache=console_outputs[:]
        console_outputs.clear()
        tower_console_outputs=[]
        
        for i in range(0, max(param.num_gpus, 1)):
            with tf.device("/{}:{}".format(device_str, i)):
                predictions = net(images, is_training=True)
                with tf.name_scope('Loss'):
                    loss = loss_func(predictions, labels, weight_maps)
                    if param.decay > 0:
                        reg_losses = graph.get_collection(
                            tf.GraphKeys.REGULARIZATION_LOSSES)
                        reg_loss = tf.reduce_mean([tf.reduce_mean(reg_loss)
                                                for reg_loss in reg_losses])
                        loss = loss + reg_loss
                tf.summary.scalar('loss',loss,[engine.logging.CONSOLE,engine.logging.LOG])

                ##################
                # This should probably be refactored into an application class
                # Averages are in name_scope for Tensorboard naming; summaries are outside for console naming
                with tf.name_scope('ConsoleLogging'):
                    logs=[]
                    if param.application_type == 'segmentation':
                        # TODO compute miss for dfferent target types
                        logs.append(['miss', tf.reduce_mean(tf.cast(
                              tf.not_equal(tf.argmax(predictions, -1), labels[..., 0]),
                              dtype=tf.float32))])
                for tag,val in logs:
                    tf.summary.scalar(tag,val,[engine.logging.CONSOLE,engine.logging.LOG])
                ################## 

                # record and clear summaries
                console_outputs=graph.get_collection_ref(engine.logging.CONSOLE)
                tower_console_outputs.append(console_outputs[:])
                console_outputs.clear()
                
                with tf.name_scope('ComputeGradients'):
                    grads = train_step.compute_gradients(loss)
                tower_grads.append(grads)
                # note: only use batch stats from one GPU for batch_norm
                if i == 0:
                    bn_updates = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
        with tf.name_scope('AccumulateGradients'):
            ave_grads = util.average_grads(tower_grads)
            apply_grad_op = train_step.apply_gradients(ave_grads)

        # Add averaged summaries
        console_outputs=graph.get_collection_ref(engine.logging.CONSOLE)
        console_outputs+=console_outputs_cache
        if len(tower_console_outputs)>1:
            # Averages are in name_scope for Tensorboard naming; summaries are outside for console naming
            with tf.name_scope('AccumulateConsoleLogs'): 
                averaged_summaries=[]
                for replicated_output in zip(*tower_console_outputs):
                    averaged_summaries.append([replicated_output[0].op.name+'_avg',tf.reduce_mean([o.op.inputs[1] for o in replicated_output])])
            for tag,avg in averaged_summaries:
                tf.summary.scalar(tag, avg,[engine.logging.CONSOLE,engine.logging.LOG])
        # Track the moving averages of all trainable variables.
        with tf.name_scope('MovingAverages'):
            variable_averages = tf.train.ExponentialMovingAverage(0.9)
            var_averages_op = variable_averages.apply(tf.trainable_variables())
            # batch norm variables moving mean and var
            batchnorm_updates_op = tf.group(*bn_updates)
        # primary operations
        init_op = tf.global_variables_initializer()
        train_op = tf.group(apply_grad_op,
                            var_averages_op,
                            batchnorm_updates_op)
        logged_summaries = list(set([s for c in [engine.logging.LOG,engine.logging.CONSOLE] for s in tf.get_collection(c)]))
        write_summary_op = tf.summary.merge(logged_summaries)
        # saver
        variables_to_restore = variable_averages.variables_to_restore()
        saver = tf.train.Saver(max_to_keep=20, var_list=variables_to_restore)
        tf.Graph.finalize(graph)
    # run session
    config = tf.ConfigProto()
    config.log_device_placement = False
    config.allow_soft_placement = True
    # config.gpu_options.allow_growth = True
    start_time = time.time()
{'fraction_non_alphanumeric': 0.07876096244659321, 'fraction_numerical': 0.0038228018889138746, 'mean_word_length': 2.153518879631271, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2312012', 'n_tokens_mistral': 4900, 'n_tokens_neox': 4700, 'n_words': 859}