<commit_before>import copy
import functools
from typing import List
from typing import Tuple
from typing import Union

import torch

from syft.frameworks.torch.tensors.interpreters.abstract import AbstractTensor
from syft.generic import ObjectStorage
from syft.codes import MSGTYPE
import syft as sy

from syft.workers import AbstractWorker  #


def make_plan(plan_blueprint):
    """Creates a plan from a function.

    For folks who would prefer to not use a decorator, they can use this function
    to create a plan.
    """
    return func2plan(plan_blueprint)


class func2plan(object):
    """Converts a function to a plan.

    Converts a function containing sequential pytorch code into
    a plan object which can be sent to any arbitrary worker.

    This class should be used only as decorator.
    """

    def __init__(self, args_shape=None):
        self.args_shape = args_shape

    def __call__(self, plan_blueprint):
        plan = Plan(
            owner=sy.local_worker,
            id=sy.ID_PROVIDER.pop(),
            name=plan_blueprint.__name__,
            blueprint=plan_blueprint,
        )
        if self.args_shape:
            plan._auto_build(args_shape=self.args_shape)
        return plan


def method2plan(plan_blueprint):
    """Converts a method to a plan.
    Converts a method containing sequential pytorch code into
    a plan object which can be sent to any arbitrary worker.
    """
    plan = Plan(
        owner=sy.local_worker,
        id=sy.ID_PROVIDER.pop(),
        blueprint=plan_blueprint,
        name=plan_blueprint.__name__,
        is_method=True,
    )

    @property
    def method(self: object) -> Plan:
        """
        This property is a way to catch the self of the method and give it to the plan,
        it will be provided in the future calls as this is not automatic (the structure
        of @func2plan would not keep the self during the call)
        Args:
            self (object): an instance of a class
        Returns:
            the plan which is also a callable.
        Example:
            When you have your plan and that you do
            > plan(*args)
            First the property is call with the part "plan" and self is caught, plan is
            returned
            Then plan is called with "(*args)" and in the __call__ function of plan the
            self parameter is re-inserted
        """
        plan._self = self
        return plan

    return method


class Plan(ObjectStorage):
    """A Plan store a sequence of torch operations, just like a function.

    A Plan is intended to store a sequence of torch operations, just like a function,
    but it allows to send this sequence of operations to remote workers and to keep a
    reference to it. This way, to compute remotely this sequence of operations on some remote
    input referenced through pointers, instead of sending multiple messages you need now to send a
    single message with the references of the plan and the pointers.
    """

    def __init__(
        self,
        id: Union[str, int],
        owner: "sy.workers.BaseWorker",
        name: str = "",
        arg_ids: List[Union[str, int]] = None,
        result_ids: List[Union[str, int]] = None,
        blueprint: callable = None,
        readable_plan: List = None,
        is_method: bool = False,
        is_built: bool = False,
        *args,
        **kwargs,
    ):
        super().__init__()

        # Plan instance info
        self.id = id
        self.name = name
        self.owner = owner

        # Info about the plan stored
        self.plan = list()
        self.readable_plan = readable_plan if readable_plan is not None else []
        self.arg_ids = arg_ids if arg_ids is not None else []
        self.result_ids = result_ids if result_ids is not None else []
        self.owner_when_built = None
        self.is_built = is_built

        # Pointing info towards a remote plan
        self.locations = []
        self.ptr_plans = {}

        self.tags = None
        self.description = None
        self.blueprint = blueprint

        # For methods
        self.is_method = is_method
        self._self = None

    def _auto_build(self, args_shape: List[Tuple[int]] = None):
        if self.is_method:
            self.build([self._self] + self._create_placeholders(args_shape))
        else:
            self.build(self._create_placeholders(args_shape))

    def _create_placeholders(self, args_shape):
        # In order to support -1 value in shape to indicate any dimension
        # we map -1 to 1 for shape dimensions.
        mapped_shapes = []
        for shape in args_shape:
            if list(filter(lambda x: x < -1, shape)):
                raise ValueError("Invalid shape {}".format(shape))
            mapped_shapes.append(tuple(map(lambda y: 1 if y == -1 else y, shape)))

        return [torch.zeros(shape) for shape in mapped_shapes]

    @property
    def _known_workers(self):
        return self.owner._known_workers

    def send_msg(self, *args, **kwargs):
        return self.owner.send_msg(*args, **kwargs)

    def request_obj(self, *args, **kwargs):
        return self.owner.request_obj(*args, **kwargs)

    def _recv_msg(self, bin_message: bin):
        """Upon reception, a Plan stores all commands which can be executed lazily.
        Args:
            bin_message: the message of a command received.
        Returns:
            The None message serialized to specify the command was received.
        """
        (some_type, (msg_type, contents)) = sy.serde.deserialize(bin_message, details=False)
<commit_msg>Add todo about -1 to 1 mapping<commit_after>import copy
import functools
from typing import List
from typing import Tuple
from typing import Union

import torch

from syft.frameworks.torch.tensors.interpreters.abstract import AbstractTensor
from syft.generic import ObjectStorage
from syft.codes import MSGTYPE
import syft as sy

from syft.workers import AbstractWorker  #


def make_plan(plan_blueprint):
    """Creates a plan from a function.

    For folks who would prefer to not use a decorator, they can use this function
    to create a plan.
    """
    return func2plan(plan_blueprint)


class func2plan(object):
    """Converts a function to a plan.

    Converts a function containing sequential pytorch code into
    a plan object which can be sent to any arbitrary worker.

    This class should be used only as decorator.
    """

    def __init__(self, args_shape=None):
        self.args_shape = args_shape

    def __call__(self, plan_blueprint):
        plan = Plan(
            owner=sy.local_worker,
            id=sy.ID_PROVIDER.pop(),
            name=plan_blueprint.__name__,
            blueprint=plan_blueprint,
        )
        if self.args_shape:
            plan._auto_build(args_shape=self.args_shape)
        return plan


def method2plan(plan_blueprint):
    """Converts a method to a plan.
    Converts a method containing sequential pytorch code into
    a plan object which can be sent to any arbitrary worker.
    """
    plan = Plan(
        owner=sy.local_worker,
        id=sy.ID_PROVIDER.pop(),
        blueprint=plan_blueprint,
        name=plan_blueprint.__name__,
        is_method=True,
    )

    @property
    def method(self: object) -> Plan:
        """
        This property is a way to catch the self of the method and give it to the plan,
        it will be provided in the future calls as this is not automatic (the structure
        of @func2plan would not keep the self during the call)
        Args:
            self (object): an instance of a class
        Returns:
            the plan which is also a callable.
        Example:
            When you have your plan and that you do
            > plan(*args)
            First the property is call with the part "plan" and self is caught, plan is
            returned
            Then plan is called with "(*args)" and in the __call__ function of plan the
            self parameter is re-inserted
        """
        plan._self = self
        return plan

    return method


class Plan(ObjectStorage):
    """A Plan store a sequence of torch operations, just like a function.

    A Plan is intended to store a sequence of torch operations, just like a function,
    but it allows to send this sequence of operations to remote workers and to keep a
    reference to it. This way, to compute remotely this sequence of operations on some remote
    input referenced through pointers, instead of sending multiple messages you need now to send a
    single message with the references of the plan and the pointers.
    """

    def __init__(
        self,
        id: Union[str, int],
        owner: "sy.workers.BaseWorker",
        name: str = "",
        arg_ids: List[Union[str, int]] = None,
        result_ids: List[Union[str, int]] = None,
        blueprint: callable = None,
        readable_plan: List = None,
        is_method: bool = False,
        is_built: bool = False,
        *args,
        **kwargs,
    ):
        super().__init__()

        # Plan instance info
        self.id = id
        self.name = name
        self.owner = owner

        # Info about the plan stored
        self.plan = list()
        self.readable_plan = readable_plan if readable_plan is not None else []
        self.arg_ids = arg_ids if arg_ids is not None else []
        self.result_ids = result_ids if result_ids is not None else []
        self.owner_when_built = None
        self.is_built = is_built

        # Pointing info towards a remote plan
        self.locations = []
        self.ptr_plans = {}

        self.tags = None
        self.description = None
        self.blueprint = blueprint

        # For methods
        self.is_method = is_method
        self._self = None

    def _auto_build(self, args_shape: List[Tuple[int]] = None):
        if self.is_method:
            self.build([self._self] + self._create_placeholders(args_shape))
        else:
            self.build(self._create_placeholders(args_shape))

    def _create_placeholders(self, args_shape):
        # In order to support -1 value in shape to indicate any dimension
        # we map -1 to 1 for shape dimensions.
        # TODO: A more complex strategy could be used
        mapped_shapes = []
        for shape in args_shape:
            if list(filter(lambda x: x < -1, shape)):
                raise ValueError("Invalid shape {}".format(shape))
            mapped_shapes.append(tuple(map(lambda y: 1 if y == -1 else y, shape)))

        return [torch.zeros(shape) for shape in mapped_shapes]

    @property
    def _known_workers(self):
        return self.owner._known_workers

    def send_msg(self, *args, **kwargs):
        return self.owner.send_msg(*args, **kwargs)

    def request_obj(self, *args, **kwargs):
        return self.owner.request_obj(*args, **kwargs)

    def _recv_msg(self, bin_message: bin):
        """Upon reception, a Plan stores all commands which can be executed lazily.
        Args:
            bin_message: the message of a command received.
        Returns:
            The None message serialized to specify the command was received.
        """
        (some_type, (msg_type, contents)) = sy.serde.deserialize(bin_message, details=False)

{'fraction_non_alphanumeric': 0.07331378299120235, 'fraction_numerical': 0.0019550342130987292, 'mean_word_length': 2.3207435821776334, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7622887', 'n_tokens_mistral': 3111, 'n_tokens_neox': 3045, 'n_words': 1166}