<commit_before>#
# Copyright 2016 The BigDL Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from typing import Any, Union
from logging import warning

import torch
from torch import nn
from torch.optim import Optimizer
import pytorch_lightning.lite as lite
from pytorch_lightning.strategies import Strategy
from pytorch_lightning.lite.wrappers import _LiteModule, _LiteOptimizer

from bigdl.nano.utils.log4Error import invalidInputError
from bigdl.nano.pytorch.utils import TORCH_VERSION_LESS_1_10
from bigdl.nano.pytorch.strategies.ipex.ipex_api import ipex_optimize
from bigdl.nano.pytorch.strategies import create_IPEXStrategy, DDPSpawnStrategy, \
    DDPSubprocessStrategy, create_RayStrategy


class LightningLite(lite.LightningLite):
    """
    LightningLite for BigDL-Nano pytorch.

    This LightningLite extends PyTorch Lightning's LightningLite by adding
    various options to accelerate pytorch training.
    """

    def __init__(self, num_processes: int = 1,
                 use_ipex: bool = False,
                 enable_bf16: bool = False,
                 strategy: Union[str, Strategy] = "subprocess",
                 *args, **kwargs) -> None:
        """
        Create a LightningLite with nano acceleration.

        :param num_processes: number of processes in distributed training, defaults to 1
        :param use_ipex: whether use ipex acceleration, defaults to False
        :param enable_bf16: whether use bf16 acceleration, defaults to False
        :param strategy: use which backend in distributed mode, defaults to "subprocess"
        """
        # Check arguments
        invalidInputError(isinstance(strategy, str),
                          "strategy object will be created by bigdl-nano, "
                          "you should pass the name of strategy.")

        self.num_processes = num_processes
        self.use_ipex = use_ipex
        self.enable_bf16 = enable_bf16

        if self.num_processes == 1:
            if self.use_ipex:
                strategy = create_IPEXStrategy(enable_bf16=self.enable_bf16)
            else:
                strategy = None     # type: ignore
        elif strategy == "spawn":
            strategy = DDPSpawnStrategy(num_processes=self.num_processes,
                                        use_ipex=self.use_ipex,
                                        enable_bf16=self.enable_bf16)
        elif strategy == "subprocess":
            strategy = DDPSubprocessStrategy(num_processes=self.num_processes,
                                             use_ipex=self.use_ipex,
                                             enable_bf16=self.enable_bf16)
        elif strategy == "ray":
            strategy = create_RayStrategy(num_workers=self.num_processes,<commit_msg>fix ipex 1.9 and 1.10 avx512 check in Lite (#5186)<commit_after>#
# Copyright 2016 The BigDL Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from typing import Any, Union
from logging import warning

import torch
from torch import nn
from torch.optim import Optimizer
import pytorch_lightning.lite as lite
from pytorch_lightning.strategies import Strategy
from pytorch_lightning.lite.wrappers import _LiteModule, _LiteOptimizer

from bigdl.nano.common import check_avx512
from bigdl.nano.utils.log4Error import invalidInputError
from bigdl.nano.pytorch.utils import TORCH_VERSION_LESS_1_10, TORCH_VERSION_LESS_1_11
from bigdl.nano.pytorch.strategies.ipex.ipex_api import ipex_optimize
from bigdl.nano.pytorch.strategies import create_IPEXStrategy, DDPSpawnStrategy, \
    DDPSubprocessStrategy, create_RayStrategy


class LightningLite(lite.LightningLite):
    """
    LightningLite for BigDL-Nano pytorch.

    This LightningLite extends PyTorch Lightning's LightningLite by adding
    various options to accelerate pytorch training.
    """

    def __init__(self, num_processes: int = 1,
                 use_ipex: bool = False,
                 enable_bf16: bool = False,
                 strategy: Union[str, Strategy] = "subprocess",
                 *args, **kwargs) -> None:
        """
        Create a LightningLite with nano acceleration.

        :param num_processes: number of processes in distributed training, defaults to 1
        :param use_ipex: whether use ipex acceleration, defaults to False
        :param enable_bf16: whether use bf16 acceleration, defaults to False
        :param strategy: use which backend in distributed mode, defaults to "subprocess"
        """
        # Check arguments
        invalidInputError(isinstance(strategy, str),
                          "strategy object will be created by bigdl-nano, "
                          "you should pass the name of strategy.")

        self.num_processes = num_processes
        self.use_ipex = use_ipex
        self.enable_bf16 = enable_bf16

        if TORCH_VERSION_LESS_1_11 and use_ipex and not check_avx512():
            warning("Enable ipex<=1.10 in a cpu instruction set"
                    " without avx512 will crash."
                    "Fall back to regular pytorch.")
            self.use_ipex = False

        if self.num_processes == 1:
            if self.use_ipex:
                strategy = create_IPEXStrategy(enable_bf16=self.enable_bf16)
            else:
                strategy = None     # type: ignore
        elif strategy == "spawn":
            strategy = DDPSpawnStrategy(num_processes=self.num_processes,
                                        use_ipex=self.use_ipex,
                                        enable_bf16=self.enable_bf16)
        elif strategy == "subprocess":
            strategy = DDPSubprocessStrategy(num_processes=self.num_processes,
                                             use_ipex=self.use_ipex,
                                             enable_bf16=self.enable_bf16)
        elif strategy == "ray":
            strategy = create_RayStrategy(num_workers=self.num_processes,
{'fraction_non_alphanumeric': 0.06382361473745286, 'fraction_numerical': 0.01508558166521613, 'mean_word_length': 2.505338078291815, 'pattern_counts': {'":': 6, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2643462', 'n_tokens_mistral': 1944, 'n_tokens_neox': 1868, 'n_words': 586}