<commit_before>""":class:`BasePreprocessor` define input and ouutput for processors."""

import abc
import typing
from pathlib import Path

import dill

from matchzoo import DataPack
from matchzoo import processor_units


def validate_context(func):
    """Validate context in the preprocessor."""
    def transform_wrapper(self, *args, **kwargs):
        if not self.context:
            raise ValueError(
                'Please fit parameters before transformation.')
        return func(self, *args, **kwargs)

    return transform_wrapper


class BasePreprocessor(metaclass=abc.ABCMeta):
    """:class:`BasePreprocessor` to input handle data."""

    DATA_FILENAME = 'preprocessor.dill'

    def __init__(self):
        """Initialization."""
        self._context = {}

    @property
    def context(self):
        """Return context."""
        return self._context

    @abc.abstractmethod
    def fit(self, inputs: list, verbose=1) -> 'BasePreprocessor':
        """
        Fit parameters on input data.

        This method is an abstract base method, need to be
        implemented in the child class.

        This method is expected to return itself as a callable
        object.

        :param inputs: List of text-left, text-right, label triples.
        :param verbose: Verbosity.
        """

    @abc.abstractmethod
    def transform(self, inputs: list, verbose=1) -> DataPack:
        """
        Transform input data to expected manner.

        This method is an abstract base method, need to be
        implemented in the child class.

        :param inputs: List of text-left, text-right, label triples,
        :param verbose: Verbosity.
            or list of text-left, text-right tuples.
        """

    def fit_transform(self, inputs: list) -> DataPack:
        """
        Call fit-transform.

        :param inputs: List of text-left, text-right, label triples.
        """
        return self.fit(inputs).transform(inputs)

    def save(self, dirpath: typing.Union[str, Path]):
        """
        Save the :class:`DSSMPreprocessor` object.

        A saved :class:`DSSMPreprocessor` is represented as a directory with
        the `context` object (fitted parameters on training data), it will
        be saved by `pickle`.

        :param dirpath: directory path of the saved :class:`DSSMPreprocessor`.
        """
        dirpath = Path(dirpath)
        data_file_path = dirpath.joinpath(self.DATA_FILENAME)

        if data_file_path.exists():
            raise FileExistsError
        elif not dirpath.exists():
            dirpath.mkdir()

        dill.dump(self, open(data_file_path, mode='wb'))

    @classmethod
    def _default_processor_units(cls) -> list:
        """Prepare needed process units."""
        return [
            processor_units.TokenizeUnit(),
            processor_units.LowercaseUnit(),
            processor_units.PuncRemovalUnit(),
            processor_units.StopRemovalUnit(),
        ]


def load_preprocessor(dirpath: typing.Union[str, Path]) -> DataPack:
    """
    Load the fitted `context`. The reverse function of :meth:`save`.

    :param dirpath: directory path of the saved model.
    :return: a :class:`DSSMPreprocessor` instance.
    """
    dirpath = Path(dirpath)

    data_file_path = dirpath.joinpath(BasePreprocessor.DATA_FILENAME)
    return dill.load(open(data_file_path, 'rb'))
<commit_msg>modify the data type in BaseProcessor<commit_after>""":class:`BasePreprocessor` define input and ouutput for processors."""

import abc
import typing
from pathlib import Path

import dill

from matchzoo import DataPack
from matchzoo import processor_units


def validate_context(func):
    """Validate context in the preprocessor."""
    def transform_wrapper(self, *args, **kwargs):
        if not self.context:
            raise ValueError(
                'Please fit parameters before transformation.')
        return func(self, *args, **kwargs)

    return transform_wrapper


class BasePreprocessor(metaclass=abc.ABCMeta):
    """:class:`BasePreprocessor` to input handle data."""

    DATA_FILENAME = 'preprocessor.dill'

    def __init__(self):
        """Initialization."""
        self._context = {}

    @property
    def context(self):
        """Return context."""
        return self._context

    @abc.abstractmethod
    def fit(self, data_pack: DataPack, verbose=1) -> 'BasePreprocessor':
        """
        Fit parameters on input data.

        This method is an abstract base method, need to be
        implemented in the child class.

        This method is expected to return itself as a callable
        object.

        :param data_pack: :class:`Datapack` object to be fitted.
        :param verbose: Verbosity.
        """

    @abc.abstractmethod
    def transform(self, data_pack: DataPack, verbose=1) -> DataPack:
        """
        Transform input data to expected manner.

        This method is an abstract base method, need to be
        implemented in the child class.

        :param data_pack: :class:`DataPack` object to be transformed.
        :param verbose: Verbosity.
            or list of text-left, text-right tuples.
        """

    def fit_transform(self, data_pack: DataPack) -> DataPack:
        """
        Call fit-transform.

        :param data_pack: :class:`DataPack` object to be processed.
        """
        return self.fit(data_pack).transform(data_pack)

    def save(self, dirpath: typing.Union[str, Path]):
        """
        Save the :class:`DSSMPreprocessor` object.

        A saved :class:`DSSMPreprocessor` is represented as a directory with
        the `context` object (fitted parameters on training data), it will
        be saved by `pickle`.

        :param dirpath: directory path of the saved :class:`DSSMPreprocessor`.
        """
        dirpath = Path(dirpath)
        data_file_path = dirpath.joinpath(self.DATA_FILENAME)

        if data_file_path.exists():
            raise FileExistsError
        elif not dirpath.exists():
            dirpath.mkdir()

        dill.dump(self, open(data_file_path, mode='wb'))

    @classmethod
    def _default_processor_units(cls) -> list:
        """Prepare needed process units."""
        return [
            processor_units.TokenizeUnit(),
            processor_units.LowercaseUnit(),
            processor_units.PuncRemovalUnit(),
            processor_units.StopRemovalUnit(),
        ]


def load_preprocessor(dirpath: typing.Union[str, Path]) -> DataPack:
    """
    Load the fitted `context`. The reverse function of :meth:`save`.

    :param dirpath: directory path of the saved model.
    :return: a :class:`DSSMPreprocessor` instance.
    """
    dirpath = Path(dirpath)

    data_file_path = dirpath.joinpath(BasePreprocessor.DATA_FILENAME)
    return dill.load(open(data_file_path, 'rb'))

{'fraction_non_alphanumeric': 0.09883806442123842, 'fraction_numerical': 0.0005883218120311811, 'mean_word_length': 2.6016949152542375, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3501860', 'n_tokens_mistral': 1907, 'n_tokens_neox': 1840, 'n_words': 570}