2023 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license minimum peft version supported for the integration a class containing all functions for loading and using adapters weights that are supported in peft library for more details about adapters and injecting them on a transformerbased model check out the documentation of peft library https huggingface codocspeftindex currently supported peft methods are all nonprefix tuning methods below is the list of supported peft methods that anyone can load train and run with this mixin class low rank adapters lora https huggingface codocspeftconceptualguideslora ia3 https huggingface codocspeftconceptualguidesia3 adalora https arxiv orgabs2303 10512 other peft models such as prompt tuning prompt learning are out of scope as these adapters are not injectable into a torch module for using these methods please refer to the usage guide of peft library with this mixin if the correct peft version is installed it is possible to load an adapter stored on a local path or in a remote hub repository and inject it in the model attach new adapters in the model and train them with trainer or by your own attach multiple adapters and iteratively activate deactivate them activate deactivate all adapters from the model get the statedict of the active adapter load adapter weights from file or remote hub folder if you are not familiar with adapters and peft methods we invite you to read more about them on peft official documentation https huggingface codocspeft requires peft as a backend to load the adapter weights args peftmodelid str optional the identifier of the model to look for on the hub or a local path to the saved adapter config file and adapter weights adaptername str optional the adapter name to use if not set will use the default adapter revision str optional defaults to main the specific model version to use it can be a branch name a tag name or a commit id since we use a gitbased system for storing models and other artifacts on huggingface co so revision can be any identifier allowed by git tip to test a pull request you made on the hub you can pass revisionrefsprprnumber tip token str optional whether to use authentication token to load the remote folder userful to load private repositories that are on huggingface hub you might need to call huggingfacecli login and paste your tokens to cache it devicemap str or dictstr unionint str torch device or int or torch device optional a map that specifies where each submodule should go it doesn t need to be refined to each parameterbuffer name once a given module name is inside every submodule of it will be sent to the same device if we only pass the device e g cpu cuda 1 mps or a gpu ordinal rank like 1 on which the model will be allocated the device map will map the entire model to this device passing devicemap 0 means put the whole model on gpu 0 to have accelerate compute the most optimized devicemap automatically set devicemapauto for more information about each option see designing a device maphttps hf codocsacceleratemainenusageguidesbigmodelingdesigningadevicemap maxmemory dict optional a dictionary device identifier to maximum memory will default to the maximum memory available for each gpu and the available cpu ram if unset offloadfolder str or os pathlike optional if the devicemap contains any value disk the folder where we will offload weights offloadindex int optional offloadindex argument to be passed to accelerate dispatchmodel method peftconfig dictstr any optional the configuration of the adapter to add supported adapters are nonprefix tuning and adaption prompts methods this argument is used in case users directly pass peft state dicts adapterstatedict dictstr torch tensor optional the state dict of the adapter to load this argument is used in case users directly pass peft state dicts adapterkwargs dictstr any optional additional keyword arguments passed along to the frompretrained method of the adapter config and findadapterconfigfile method we keep revision in the signature for backward compatibility override token with adapterkwargs token create and add fresh new adapters into the model we need to preprocess the state dict to remove unneeded prefixes for backward compatibility load state dict check only for unexpected keys redispatch model and hooks in case the model is offloaded to cpu disk checkpeftversionminversionminpeftversion from peft import peftconfig injectadapterinmodel adaptername adaptername or default if not self hfpeftconfigloaded self hfpeftconfigloaded true elif adaptername in self peftconfig raise valueerrorfadapter with name adaptername already exists please use a different name if not isinstanceadapterconfig peftconfig raise valueerror fadapterconfig should be an instance of peftconfig got typeadapterconfig instead retrieve the name or path of the model one could also use self config nameorpath but to be consistent with what we do in peft https github comhuggingfacepeftblob6e783780ca9df3a623992cc4d1d665001232eae0srcpeftmapping pyl100 adapterconfig basemodelnameorpath self dict getnameorpath none injectadapterinmodeladapterconfig self adaptername self setadapteradaptername def setadapterself adaptername unionliststr str none checkpeftversionminversionminpeftversion if not self hfpeftconfigloaded raise valueerrorno adapter loaded please load an adapter first elif isinstanceadaptername list missing setadaptername setself peftconfig if lenmissing 0 raise valueerror ffollowing adapters could not be found joinmissing make sure you are passing the correct adapter names f current loaded adapters are listself peftconfig keys elif adaptername not in self peftconfig raise valueerror fadapter with name adaptername not found please pass the correct adapter name among listself peftconfig keys from peft tuners tunersutils import basetunerlayer from peft utils import modulestosavewrapper adaptershasbeenset false for module in self namedmodules if isinstancemodule basetunerlayer modulestosavewrapper for backward compatbility with previous peft versions if hasattrmodule setadapter module setadapteradaptername else module activeadapter adaptername adaptershasbeenset true if not adaptershasbeenset raise valueerror did not succeeded in setting the adapter please make sure you are using a model that supports adapters def disableadaptersself none r if you are not familiar with adapters and peft methods we invite you to read more about them on the peft official documentation https huggingface codocspeft disable all adapters that are attached to the model this leads to inferring with the base model only the recent version of peft need to call enableadapters instead if you are not familiar with adapters and peft methods we invite you to read more about them on the peft official documentation https huggingface codocspeft enable adapters that are attached to the model the model will use self activeadapter the recent version of peft need to call enableadapters instead if you are not familiar with adapters and peft methods we invite you to read more about them on the peft official documentation https huggingface codocspeft gets the current active adapters of the model in case of multiadapter inference combining multiple adapters for inference returns the list of all active adapters so that users can deal with them accordingly for previous peft versions that does not support multiadapter inference module activeadapter will return a single string for previous peft versions if you are not familiar with adapters and peft methods we invite you to read more about them on the peft official documentation https huggingface codocspeft gets the adapter state dict that should only contain the weights tensors of the specified adaptername adapter if no adaptername is passed the active adapter is used args adaptername str optional the name of the adapter to get the state dict from if no name is passed the active adapter is used optional redispatch the model and attach new hooks to the model in case the model has been loaded with accelerate i e with devicemapxxx args devicemap str or dictstr unionint str torch device or int or torch device optional a map that specifies where each submodule should go it doesn t need to be refined to each parameterbuffer name once a given module name is inside every submodule of it will be sent to the same device if we only pass the device e g cpu cuda 1 mps or a gpu ordinal rank like 1 on which the model will be allocated the device map will map the entire model to this device passing devicemap 0 means put the whole model on gpu 0 to have accelerate compute the most optimized devicemap automatically set devicemapauto for more information about each option see designing a device maphttps hf codocsacceleratemainenusageguidesbigmodelingdesigningadevicemap maxmemory dict optional a dictionary device identifier to maximum memory will default to the maximum memory available for each gpu and the available cpu ram if unset offloadfolder str or os pathlike optional if the devicemap contains any value disk the folder where we will offload weights offloadindex int optional the offloadindex argument to be passed to accelerate dispatchmodel method safety checker for previous accelerate versions offloadindex was introduced in https github comhuggingfaceacceleratepull873 2023 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license minimum peft version supported for the integration a class containing all functions for loading and using adapters weights that are supported in peft library for more details about adapters and injecting them on a transformer based model check out the documentation of peft library https huggingface co docs peft index currently supported peft methods are all non prefix tuning methods below is the list of supported peft methods that anyone can load train and run with this mixin class low rank adapters lora https huggingface co docs peft conceptual_guides lora ia3 https huggingface co docs peft conceptual_guides ia3 adalora https arxiv org abs 2303 10512 other peft models such as prompt tuning prompt learning are out of scope as these adapters are not injectable into a torch module for using these methods please refer to the usage guide of peft library with this mixin if the correct peft version is installed it is possible to load an adapter stored on a local path or in a remote hub repository and inject it in the model attach new adapters in the model and train them with trainer or by your own attach multiple adapters and iteratively activate deactivate them activate deactivate all adapters from the model get the state_dict of the active adapter load adapter weights from file or remote hub folder if you are not familiar with adapters and peft methods we invite you to read more about them on peft official documentation https huggingface co docs peft requires peft as a backend to load the adapter weights args peft_model_id str optional the identifier of the model to look for on the hub or a local path to the saved adapter config file and adapter weights adapter_name str optional the adapter name to use if not set will use the default adapter revision str optional defaults to main the specific model version to use it can be a branch name a tag name or a commit id since we use a git based system for storing models and other artifacts on huggingface co so revision can be any identifier allowed by git tip to test a pull request you made on the hub you can pass revision refs pr pr_number tip token str optional whether to use authentication token to load the remote folder userful to load private repositories that are on huggingface hub you might need to call huggingface cli login and paste your tokens to cache it device_map str or dict str union int str torch device or int or torch device optional a map that specifies where each submodule should go it doesn t need to be refined to each parameter buffer name once a given module name is inside every submodule of it will be sent to the same device if we only pass the device e g cpu cuda 1 mps or a gpu ordinal rank like 1 on which the model will be allocated the device map will map the entire model to this device passing device_map 0 means put the whole model on gpu 0 to have accelerate compute the most optimized device_map automatically set device_map auto for more information about each option see designing a device map https hf co docs accelerate main en usage_guides big_modeling designing a device map max_memory dict optional a dictionary device identifier to maximum memory will default to the maximum memory available for each gpu and the available cpu ram if unset offload_folder str or os pathlike optional if the device_map contains any value disk the folder where we will offload weights offload_index int optional offload_index argument to be passed to accelerate dispatch_model method peft_config dict str any optional the configuration of the adapter to add supported adapters are non prefix tuning and adaption prompts methods this argument is used in case users directly pass peft state dicts adapter_state_dict dict str torch tensor optional the state dict of the adapter to load this argument is used in case users directly pass peft state dicts adapter_kwargs dict str any optional additional keyword arguments passed along to the from_pretrained method of the adapter config and find_adapter_config_file method we keep revision in the signature for backward compatibility override token with adapter_kwargs token create and add fresh new adapters into the model we need to pre process the state dict to remove unneeded prefixes for backward compatibility load state dict check only for unexpected keys re dispatch model and hooks in case the model is offloaded to cpu disk if you are not familiar with adapters and peft methods we invite you to read more about them on the peft official documentation https huggingface co docs peft adds a fresh new adapter to the current model for training purpose if no adapter name is passed a default name is assigned to the adapter to follow the convention of peft library in peft we use default as the default adapter name args adapter_config peft peftconfig the configuration of the adapter to add supported adapters are non prefix tuning and adaption prompts methods adapter_name str optional defaults to default the name of the adapter to add if no name is passed a default name is assigned to the adapter retrieve the name or path of the model one could also use self config _name_or_path but to be consistent with what we do in peft https github com huggingface peft blob 6e783780ca9df3a623992cc4d1d665001232eae0 src peft mapping py l100 if you are not familiar with adapters and peft methods we invite you to read more about them on the peft official documentation https huggingface co docs peft sets a specific adapter by forcing the model to use a that adapter and disable the other adapters args adapter_name union list str str the name of the adapter to set can be also a list of strings to set multiple adapters for backward compatbility with previous peft versions if you are not familiar with adapters and peft methods we invite you to read more about them on the peft official documentation https huggingface co docs peft disable all adapters that are attached to the model this leads to inferring with the base model only the recent version of peft need to call enable_adapters instead if you are not familiar with adapters and peft methods we invite you to read more about them on the peft official documentation https huggingface co docs peft enable adapters that are attached to the model the model will use self active_adapter the recent version of peft need to call enable_adapters instead if you are not familiar with adapters and peft methods we invite you to read more about them on the peft official documentation https huggingface co docs peft gets the current active adapters of the model in case of multi adapter inference combining multiple adapters for inference returns the list of all active adapters so that users can deal with them accordingly for previous peft versions that does not support multi adapter inference module active_adapter will return a single string for previous peft versions if you are not familiar with adapters and peft methods we invite you to read more about them on the peft official documentation https huggingface co docs peft gets the adapter state dict that should only contain the weights tensors of the specified adapter_name adapter if no adapter_name is passed the active adapter is used args adapter_name str optional the name of the adapter to get the state dict from if no name is passed the active adapter is used optional re dispatch the model and attach new hooks to the model in case the model has been loaded with accelerate i e with device_map xxx args device_map str or dict str union int str torch device or int or torch device optional a map that specifies where each submodule should go it doesn t need to be refined to each parameter buffer name once a given module name is inside every submodule of it will be sent to the same device if we only pass the device e g cpu cuda 1 mps or a gpu ordinal rank like 1 on which the model will be allocated the device map will map the entire model to this device passing device_map 0 means put the whole model on gpu 0 to have accelerate compute the most optimized device_map automatically set device_map auto for more information about each option see designing a device map https hf co docs accelerate main en usage_guides big_modeling designing a device map max_memory dict optional a dictionary device identifier to maximum memory will default to the maximum memory available for each gpu and the available cpu ram if unset offload_folder str or os pathlike optional if the device_map contains any value disk the folder where we will offload weights offload_index int optional the offload_index argument to be passed to accelerate dispatch_model method safety checker for previous accelerate versions offload_index was introduced in https github com huggingface accelerate pull 873
import inspect
import warnings
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union

from ..utils import (
    check_peft_version,
    find_adapter_config_file,
    is_accelerate_available,
    is_peft_available,
    is_torch_available,
    logging,
)


if is_accelerate_available():
    from accelerate import dispatch_model
    from accelerate.utils import get_balanced_memory, infer_auto_device_map


MIN_PEFT_VERSION = "0.5.0"

if TYPE_CHECKING:
    if is_torch_available():
        import torch


logger = logging.get_logger(__name__)


class PeftAdapterMixin:
    

    _hf_peft_config_loaded = False

    def load_adapter(
        self,
        peft_model_id: Optional[str] = None,
        adapter_name: Optional[str] = None,
        revision: Optional[str] = None,
        token: Optional[str] = None,
        device_map: Optional[str] = "auto",
        max_memory: Optional[str] = None,
        offload_folder: Optional[str] = None,
        offload_index: Optional[int] = None,
        peft_config: Dict[str, Any] = None,
        adapter_state_dict: Optional[Dict[str, "torch.Tensor"]] = None,
        adapter_kwargs: Optional[Dict[str, Any]] = None,
    ) -> None:
        
        check_peft_version(min_version=MIN_PEFT_VERSION)

        adapter_name = adapter_name if adapter_name is not None else "default"
        if adapter_kwargs is None:
            adapter_kwargs = {}

        from peft import PeftConfig, inject_adapter_in_model, load_peft_weights
        from peft.utils import set_peft_model_state_dict

        if self._hf_peft_config_loaded and adapter_name in self.peft_config:
            raise ValueError(f"Adapter with name {adapter_name} already exists. Please use a different name.")

        if peft_model_id is None and (adapter_state_dict is None and peft_config is None):
            raise ValueError(
                "You should either pass a `peft_model_id` or a `peft_config` and `adapter_state_dict` to load an adapter."
            )

        
        if revision is not None and "revision" not in adapter_kwargs:
            adapter_kwargs["revision"] = revision
        elif revision is not None and "revision" in adapter_kwargs and revision != adapter_kwargs["revision"]:
            logger.error(
                "You passed a `revision` argument both in `adapter_kwargs` and as a standalone argument. "
                "The one in `adapter_kwargs` will be used."
            )

        
        if "token" in adapter_kwargs:
            token = adapter_kwargs.pop("token")

        if peft_config is None:
            adapter_config_file = find_adapter_config_file(
                peft_model_id,
                token=token,
                **adapter_kwargs,
            )

            if adapter_config_file is None:
                raise ValueError(
                    f"adapter model file not found in {peft_model_id}. Make sure you are passing the correct path to the "
                    "adapter model."
                )

            peft_config = PeftConfig.from_pretrained(
                peft_model_id,
                token=token,
                **adapter_kwargs,
            )

        
        inject_adapter_in_model(peft_config, self, adapter_name)

        if not self._hf_peft_config_loaded:
            self._hf_peft_config_loaded = True

        if peft_model_id is not None:
            adapter_state_dict = load_peft_weights(peft_model_id, token=token, **adapter_kwargs)

        
        processed_adapter_state_dict = {}
        prefix = "base_model.model."
        for key, value in adapter_state_dict.items():
            if key.startswith(prefix):
                new_key = key[len(prefix) :]
            else:
                new_key = key
            processed_adapter_state_dict[new_key] = value

        
        incompatible_keys = set_peft_model_state_dict(self, processed_adapter_state_dict, adapter_name)

        if incompatible_keys is not None:
            
            if hasattr(incompatible_keys, "unexpected_keys") and len(incompatible_keys.unexpected_keys) > 0:
                logger.warning(
                    f"Loading adapter weights from {peft_model_id} led to unexpected keys not found in the model: "
                    f" {incompatible_keys.unexpected_keys}. "
                )

        
        if (
            (getattr(self, "hf_device_map", None) is not None)
            and (len(set(self.hf_device_map.values()).intersection({"cpu", "disk"})) > 0)
            and len(self.peft_config) == 1
        ):
            self._dispatch_accelerate_model(
                device_map=device_map,
                max_memory=max_memory,
                offload_folder=offload_folder,
                offload_index=offload_index,
            )

    def add_adapter(self, adapter_config, adapter_name: Optional[str] = None) -> None:
        r
        check_peft_version(min_version=MIN_PEFT_VERSION)

        from peft import PeftConfig, inject_adapter_in_model

        adapter_name = adapter_name or "default"

        if not self._hf_peft_config_loaded:
            self._hf_peft_config_loaded = True
        elif adapter_name in self.peft_config:
            raise ValueError(f"Adapter with name {adapter_name} already exists. Please use a different name.")

        if not isinstance(adapter_config, PeftConfig):
            raise ValueError(
                f"adapter_config should be an instance of PeftConfig. Got {type(adapter_config)} instead."
            )

        
        
        adapter_config.base_model_name_or_path = self.__dict__.get("name_or_path", None)
        inject_adapter_in_model(adapter_config, self, adapter_name)

        self.set_adapter(adapter_name)

    def set_adapter(self, adapter_name: Union[List[str], str]) -> None:
        
        check_peft_version(min_version=MIN_PEFT_VERSION)
        if not self._hf_peft_config_loaded:
            raise ValueError("No adapter loaded. Please load an adapter first.")
        elif isinstance(adapter_name, list):
            missing = set(adapter_name) - set(self.peft_config)
            if len(missing) > 0:
                raise ValueError(
                    f"Following adapter(s) could not be found: {', '.join(missing)}. Make sure you are passing the correct adapter name(s)."
                    f" current loaded adapters are: {list(self.peft_config.keys())}"
                )
        elif adapter_name not in self.peft_config:
            raise ValueError(
                f"Adapter with name {adapter_name} not found. Please pass the correct adapter name among {list(self.peft_config.keys())}"
            )

        from peft.tuners.tuners_utils import BaseTunerLayer
        from peft.utils import ModulesToSaveWrapper

        _adapters_has_been_set = False

        for _, module in self.named_modules():
            if isinstance(module, (BaseTunerLayer, ModulesToSaveWrapper)):
                
                if hasattr(module, "set_adapter"):
                    module.set_adapter(adapter_name)
                else:
                    module.active_adapter = adapter_name
                _adapters_has_been_set = True

        if not _adapters_has_been_set:
            raise ValueError(
                "Did not succeeded in setting the adapter. Please make sure you are using a model that supports adapters."
            )

    def disable_adapters(self) -> None:
        r
        check_peft_version(min_version=MIN_PEFT_VERSION)

        if not self._hf_peft_config_loaded:
            raise ValueError("No adapter loaded. Please load an adapter first.")

        from peft.tuners.tuners_utils import BaseTunerLayer
        from peft.utils import ModulesToSaveWrapper

        for _, module in self.named_modules():
            if isinstance(module, (BaseTunerLayer, ModulesToSaveWrapper)):
                
                if hasattr(module, "enable_adapters"):
                    module.enable_adapters(enabled=False)
                else:
                    module.disable_adapters = True

    def enable_adapters(self) -> None:
        
        check_peft_version(min_version=MIN_PEFT_VERSION)

        if not self._hf_peft_config_loaded:
            raise ValueError("No adapter loaded. Please load an adapter first.")

        from peft.tuners.tuners_utils import BaseTunerLayer

        for _, module in self.named_modules():
            if isinstance(module, BaseTunerLayer):
                
                if hasattr(module, "enable_adapters"):
                    module.enable_adapters(enabled=True)
                else:
                    module.disable_adapters = False

    def active_adapters(self) -> List[str]:
        
        check_peft_version(min_version=MIN_PEFT_VERSION)

        if not is_peft_available():
            raise ImportError("PEFT is not available. Please install PEFT to use this function: `pip install peft`.")

        if not self._hf_peft_config_loaded:
            raise ValueError("No adapter loaded. Please load an adapter first.")

        from peft.tuners.tuners_utils import BaseTunerLayer

        for _, module in self.named_modules():
            if isinstance(module, BaseTunerLayer):
                active_adapters = module.active_adapter
                break

        
        if isinstance(active_adapters, str):
            active_adapters = [active_adapters]

        return active_adapters

    def active_adapter(self) -> str:
        warnings.warn(
            "The `active_adapter` method is deprecated and will be removed in a future version.", FutureWarning
        )

        return self.active_adapters()[0]

    def get_adapter_state_dict(self, adapter_name: Optional[str] = None) -> dict:
        
        check_peft_version(min_version=MIN_PEFT_VERSION)

        if not self._hf_peft_config_loaded:
            raise ValueError("No adapter loaded. Please load an adapter first.")

        from peft import get_peft_model_state_dict

        if adapter_name is None:
            adapter_name = self.active_adapter()

        adapter_state_dict = get_peft_model_state_dict(self, adapter_name=adapter_name)
        return adapter_state_dict

    def _dispatch_accelerate_model(
        self,
        device_map: str,
        max_memory: Optional[int] = None,
        offload_folder: Optional[str] = None,
        offload_index: Optional[int] = None,
    ) -> None:
        
        dispatch_model_kwargs = {}
        
        
        if "offload_index" in inspect.signature(dispatch_model).parameters:
            dispatch_model_kwargs["offload_index"] = offload_index

        no_split_module_classes = self._no_split_modules

        if device_map != "sequential":
            max_memory = get_balanced_memory(
                self,
                max_memory=max_memory,
                no_split_module_classes=no_split_module_classes,
                low_zero=(device_map == "balanced_low_0"),
            )
        if isinstance(device_map, str):
            device_map = infer_auto_device_map(
                self, max_memory=max_memory, no_split_module_classes=no_split_module_classes
            )
        dispatch_model(
            self,
            device_map=device_map,
            offload_dir=offload_folder,
            **dispatch_model_kwargs,
        )