<commit_before>import os

from caput import pipeline, config


class SingleTask(pipeline.TaskBase, pipeline.BasicContMixin):
    """Process a task with at most one input and output.

    Both input and output are expected to be :class:`memh5.BasicCont` objects.
    This class allows writing of the output when requested.

    Tasks inheriting from this class should override `process` and optionally
    :meth:`setup` or :meth:`finish`. They should not override :meth:`next`.

    If the value of :attr:`input_root` is anything other than the string "None"
    then the input will be read (using :meth:`read_input`) from the file
    ``self.input_root + self.input_filename``.  If the input is specified both as
    a filename and as a product key in the pipeline configuration, an error
    will be raised upon initialization.


    If the value of :attr:`output_root` is anything other than the string
    "None" then the output will be written (using :meth:`write_output`) to the
    file ``self.output_root + self.output_filename``.

    Attributes
    ----------
    save : bool
        Whether to save the output to disk or not.
    output_root : string
        Pipeline settable parameter giving the first part of the output path.
        If set to 'None' no output is written.

    Methods
    -------
    next
    setup
    process
    finish
    read_input
    cast_input
    write_output

    """

    save = config.Property(default=False, proptype=bool)
    output_root = config.Property(default='', proptype=str)

    _count = 0

    done = False
    _no_input = False

    def __init__(self):
        """Checks inputs and outputs and stuff."""

        import inspect

        # Inspect the `process` method to see how many arguments it takes.
        pro_argspec = inspect.getargspec(self.process)
        n_args = len(pro_argspec.args) - 1
        if n_args  > 1:
            msg = ("`process` method takes more than 1 argument, which is not"
                   " allowed.")
            raise PipelineConfigError(msg)
        if pro_argspec.varargs or pro_argspec.keywords or pro_argspec.defaults:
            msg = ("`process` method may not have variable length or optional"
                   " arguments.")
            raise PipelineConfigError(msg)
        if n_args == 0:
            self._no_input = True
        else:
            self._no_input = False


    def next(self, input=None):
        """Should not need to override. Implement `process` instead."""

        # This should only be called once.
        try:
            if self.done:
                raise pipeline.PipelineStopIteration()
        except AttributeError:
            self.done = True

        # Process input and fetch ouput
        if self._no_input:
            if input is not None:
                # This should never happen.  Just here to catch bugs.
                raise RuntimeError("Somehow `input` was set.")
            output = self.process()
        else:
            #if input is not None:
            #    input = self.cast_input(input)
            output = self.process(input)

        # Return immediately if output is None to skip writing phase.
        if output is None:
            return

        # Set a tag in output if needed
        if 'tag' not in output.attrs and input is not None and 'tag' in input.attrs:
            output.attrs['tag'] = input.attrs['tag']

        # Write the output if needed
        self._save_output(output)

        # Increment internal counter
        self._count = self._count + 1

        # Return the output for the next task
        return output

    def finish(self):
        """Should not need to override. Implement `process_finish` instead."""

        try:
            output = self.process_finish()
            
            # Write the output if needed
            self._save_output(output)

            return output

        except AttributeError:
            pass<commit_msg>Changed task base class to allow multiple inputs, and modified calibration task to take in inputmap on every call.<commit_after>import os

from caput import pipeline, config


class SingleTask(pipeline.TaskBase, pipeline.BasicContMixin):
    """Process a task with at most one input and output.

    Both input and output are expected to be :class:`memh5.BasicCont` objects.
    This class allows writing of the output when requested.

    Tasks inheriting from this class should override `process` and optionally
    :meth:`setup` or :meth:`finish`. They should not override :meth:`next`.

    If the value of :attr:`input_root` is anything other than the string "None"
    then the input will be read (using :meth:`read_input`) from the file
    ``self.input_root + self.input_filename``.  If the input is specified both as
    a filename and as a product key in the pipeline configuration, an error
    will be raised upon initialization.


    If the value of :attr:`output_root` is anything other than the string
    "None" then the output will be written (using :meth:`write_output`) to the
    file ``self.output_root + self.output_filename``.

    Attributes
    ----------
    save : bool
        Whether to save the output to disk or not.
    output_root : string
        Pipeline settable parameter giving the first part of the output path.
        If set to 'None' no output is written.

    Methods
    -------
    next
    setup
    process
    finish
    read_input
    cast_input
    write_output

    """

    save = config.Property(default=False, proptype=bool)
    output_root = config.Property(default='', proptype=str)

    _count = 0

    done = False
    _no_input = False

    def __init__(self):
        """Checks inputs and outputs and stuff."""

        import inspect

        # Inspect the `process` method to see how many arguments it takes.
        pro_argspec = inspect.getargspec(self.process)
        n_args = len(pro_argspec.args) - 1

        #if n_args  > 1:
        #    msg = ("`process` method takes more than 1 argument, which is not"
        #           " allowed.")
        #    raise PipelineConfigError(msg)

        if pro_argspec.varargs or pro_argspec.keywords or pro_argspec.defaults:
            msg = ("`process` method may not have variable length or optional"
                   " arguments.")
            raise PipelineConfigError(msg)
        
        if n_args == 0:
            self._no_input = True
        else:
            self._no_input = False


    def next(self, *input):
        """Should not need to override. Implement `process` instead."""

        # This should only be called once.
        try:
            if self.done:
                raise pipeline.PipelineStopIteration()
        except AttributeError:
            self.done = True

        # Process input and fetch ouput
        if self._no_input:
            if len(input) > 0:
                # This should never happen.  Just here to catch bugs.
                raise RuntimeError("Somehow `input` was set.")
            output = self.process()
        else:
            output = self.process(*input)

        # Return immediately if output is None to skip writing phase.
        if output is None:
            return

        # Set a tag in output if needed
        if 'tag' not in output.attrs and len(input) > 0 and 'tag' in input[0].attrs:
            output.attrs['tag'] = input[0].attrs['tag']

        # Write the output if needed
        self._save_output(output)

        # Increment internal counter
        self._count = self._count + 1

        # Return the output for the next task
        return output

    def finish(self):
        """Should not need to override. Implement `process_finish` instead."""

        try:
            output = self.process_finish()
            
            # Write the output if needed
            self._save_output(output)

            return output

        except AttributeError:
            pass
{'fraction_non_alphanumeric': 0.0768258781905484, 'fraction_numerical': 0.002274450341167551, 'mean_word_length': 2.0797665369649807, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7205591', 'n_tokens_mistral': 2102, 'n_tokens_neox': 2005, 'n_words': 810}