<commit_before>from __future__ import division, print_function, absolute_import

import inspect
import logging
import os

from dipy.workflows.multi_io import io_iterator_


class Workflow(object):
    def __init__(self, output_strategy='append', mix_names=False,
                 force=False, skip=False):
        """ The basic workflow object, it takes care of any workflow operation that
        is common. Every new workflow should extend this class.
        """
        self._output_strategy = output_strategy
        self._mix_names = mix_names
        self.last_generated_outputs = None
        self._force_overwrite = force
        self._skip = skip

    def get_io_iterator(self):
        """ Use a couple of inspection tricks to build an IOIterator using the
        previous frame(values of local variables and other contextuals) and the
        run method's docstring.
        """

        # To manage python 2/3
        try:
            frame = inspect.currentframe(1)
        except:
            frame = inspect.stack()[1].frame

        io_it = io_iterator_(frame, self.run,
                             output_strategy=self._output_strategy,
                             mix_names=self._mix_names)

        self.last_generated_outputs = io_it.outputs
        if self.manage_output_overwrite():
            return io_it
        else:
            return []

    def manage_output_overwrite(self):
        """ Check if a file will be overwritten upon processing the inputs.
        If it is bound to happen, an action is taken depending on
        self._force_overwrite (or --force via command line). A log message is
        output independently of the outcome to tell the user something happened.
        """
        duplicates = []
        for output_list in self.last_generated_outputs:
            for output in output_list:
                if os.path.isfile(output):
                    duplicates.append(output)
<commit_msg>Newer version of get_io_iterator compatible with all pythons.<commit_after>from __future__ import division, print_function, absolute_import

import inspect
import logging
import os

from dipy.workflows.multi_io import io_iterator_


class Workflow(object):
    def __init__(self, output_strategy='append', mix_names=False,
                 force=False, skip=False):
        """ The basic workflow object, it takes care of any workflow operation that
        is common. Every new workflow should extend this class.
        """
        self._output_strategy = output_strategy
        self._mix_names = mix_names
        self.last_generated_outputs = None
        self._force_overwrite = force
        self._skip = skip

    def get_io_iterator(self):
        """ Use a couple of inspection tricks to build an IOIterator using the
        previous frame(values of local variables and other contextuals) and the
        run method's docstring.
        """

        # To manage different python versions.
        frame = inspect.stack()[1]
        if isinstance(frame, tuple):
            frame = frame[0]
        else:
            frame = frame.frame

        io_it = io_iterator_(frame, self.run,
                             output_strategy=self._output_strategy,
                             mix_names=self._mix_names)

        self.last_generated_outputs = io_it.outputs
        if self.manage_output_overwrite():
            return io_it
        else:
            return []

    def manage_output_overwrite(self):
        """ Check if a file will be overwritten upon processing the inputs.
        If it is bound to happen, an action is taken depending on
        self._force_overwrite (or --force via command line). A log message is
        output independently of the outcome to tell the user something happened.
        """
        duplicates = []
        for output_list in self.last_generated_outputs:
            for output in output_list:
                if os.path.isfile(output):
                    duplicates.append(output)

{'fraction_non_alphanumeric': 0.06282985674792661, 'fraction_numerical': 0.0015079165619502387, 'mean_word_length': 2.1942215088282504, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6169992', 'n_tokens_mistral': 1020, 'n_tokens_neox': 992, 'n_words': 338}