<commit_before>import os, glob, csv, re
from abc import ABCMeta, abstractmethod
import numpy as np
from itertools import combinations
import SimpleITK as sitk

from ContinuousRegistration.Source.metrics import tre, hausdorff, inverse_consistency_labels, inverse_consistency_points, dice
from ContinuousRegistration.Source.util import take, sort_file_names, copy_information_from_images_to_labels, merge_dicts
from ContinuousRegistration.Source.util import create_mask_by_thresholding, create_mask_by_size, create_identity_world_information
from ContinuousRegistration.Source.util import create_disp_field_names, warp_image
from ContinuousRegistration.Source.util import logging


class Dataset(object):
    """
    Base class for datasets. The derived classes need only to implement the file
    layout on disk and how to evaluate the dataset.  Everything else is handled
    by this class. See metrics.py for available metrics.
    """
    __metaclass__ = ABCMeta

    def generator(self):
        for file_name in self.file_names:
            yield file_name

    def make_shell_scripts(self, superelastix, blueprint_file_name, file_names, output_directory):
        if not os.path.exists(os.path.join(output_directory, 'sh')):
            os.mkdir(os.path.join(output_directory, 'sh'))
        COMMAND_TEMPLATE = '%s --conf \'%s\' --in %s %s --out DisplacementField=\'%s\' --loglevel trace --logfile \'%s\''
        #COMMAND_TEMPLATE = '%s --conf %s --in %s %s --out DisplacementField=%s --loglevel trace --logfile %s'

        # Fixed to moving
        root = os.path.splitext(file_names['disp_field_file_names'][0])[0]
        shell_script_file_name = os.path.join(output_directory, 'sh',
                                              root.replace('/', '_').replace('\\', '_').replace('.', '_') + '.sh')

        if 'mask_file_names' not in file_names:
            file_names['mask_file_names'] = ('', '')
        with open(shell_script_file_name, 'w') as shell_script:
            shell_script.write(COMMAND_TEMPLATE % (superelastix, blueprint_file_name,
                'FixedImage=\'%s\' MovingImage=\'%s\' ' % tuple(file_names['image_file_names']),
                'FixedMask=\'%s\' MovingMask=\'%s\'' % tuple(file_names['mask_file_names']),
                os.path.join(output_directory, file_names['disp_field_file_names'][0]),
                os.path.splitext(shell_script_file_name)[0] + '.log'))

        # Moving to fixed
        root = os.path.splitext(file_names['disp_field_file_names'][1])[0]
        shell_script_file_name = os.path.join(output_directory, 'sh',
                                              root.replace('/', '_').replace('\\', '_').replace('.', '_') + '.sh')<commit_msg>STY: remove commented line<commit_after>import os, glob, csv, re
from abc import ABCMeta, abstractmethod
import numpy as np
from itertools import combinations
import SimpleITK as sitk

from ContinuousRegistration.Source.metrics import tre, hausdorff, inverse_consistency_labels, inverse_consistency_points, dice
from ContinuousRegistration.Source.util import take, sort_file_names, copy_information_from_images_to_labels, merge_dicts
from ContinuousRegistration.Source.util import create_mask_by_thresholding, create_mask_by_size, create_identity_world_information
from ContinuousRegistration.Source.util import create_disp_field_names, warp_image
from ContinuousRegistration.Source.util import logging


class Dataset(object):
    """
    Base class for datasets. The derived classes need only to implement the file
    layout on disk and how to evaluate the dataset.  Everything else is handled
    by this class. See metrics.py for available metrics.
    """
    __metaclass__ = ABCMeta

    def generator(self):
        for file_name in self.file_names:
            yield file_name

    def make_shell_scripts(self, superelastix, blueprint_file_name, file_names, output_directory):
        if not os.path.exists(os.path.join(output_directory, 'sh')):
            os.mkdir(os.path.join(output_directory, 'sh'))
        COMMAND_TEMPLATE = '%s --conf \'%s\' --in %s %s --out DisplacementField=\'%s\' --loglevel trace --logfile \'%s\''

        # Fixed to moving
        root = os.path.splitext(file_names['disp_field_file_names'][0])[0]
        shell_script_file_name = os.path.join(output_directory, 'sh',
                                              root.replace('/', '_').replace('\\', '_').replace('.', '_') + '.sh')

        if 'mask_file_names' not in file_names:
            file_names['mask_file_names'] = ('', '')
        with open(shell_script_file_name, 'w') as shell_script:
            shell_script.write(COMMAND_TEMPLATE % (superelastix, blueprint_file_name,
                'FixedImage=\'%s\' MovingImage=\'%s\' ' % tuple(file_names['image_file_names']),
                'FixedMask=\'%s\' MovingMask=\'%s\'' % tuple(file_names['mask_file_names']),
                os.path.join(output_directory, file_names['disp_field_file_names'][0]),
                os.path.splitext(shell_script_file_name)[0] + '.log'))

        # Moving to fixed
        root = os.path.splitext(file_names['disp_field_file_names'][1])[0]
        shell_script_file_name = os.path.join(output_directory, 'sh',
                                              root.replace('/', '_').replace('\\', '_').replace('.', '_') + '.sh')
{'fraction_non_alphanumeric': 0.12185031966904851, 'fraction_numerical': 0.0022564874012786762, 'mean_word_length': 3.289516129032258, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4497654', 'n_tokens_mistral': 1588, 'n_tokens_neox': 1546, 'n_words': 357}