@inproceedings{
babakniya2023slora,
title={{SL}o{RA}: Federated Parameter Efficient Fine-Tuning of Language Models},
author={Sara Babakniya and Ahmed Elkordy and Yahya Ezzeldin and Qingfeng Liu and Kee-Bong Song and MOSTAFA EL-Khamy and Salman Avestimehr},
booktitle={International Workshop on Federated Learning in the Age of Foundation Models in Conjunction with NeurIPS 2023},
year={2023},
url={https://openreview.net/forum?id=06quMTmtRV}
}
Fine-tuning pre-trained models has gained significant success in delivering SOTA results across various NLP tasks. In the absence of centralized data, Federated Learning (FL) helps the model to benefit from clients' private data for fine-tuning. However, due to the limited communication, computation, and storage capabilities of edge devices and the huge sizes of popular pre-trained models, efficient fine-tuning is crucial. This work explores the opportunities and challenges of applying parameter efficient fine-tuning (PEFT) methods in FL for language tasks. Specifically, our investigations reveal that with increasing data heterogeneity across users, the gap between fully fine-tuning the model and employing PEFT methods widens. To bridge this performance gap, we propose a method, SLoRA, which overcomes the key limitations of LoRA in high heterogeneous data scenarios through a novel data-driven initialization technique. Our experimental results demonstrate that SLoRA achieves performance comparable to full fine-tuning, with significant sparse updates with $\sim 1\%$ density while reducing training time by up to $90\%$.
SLoRA: Federated Parameter Efficient Fine-Tuning of Language Models
Sara Babakniya Ahmed Roushdy Elkordy Yahya H. Ezzeldin Qingfeng Liu Kee-Bong Song MOSTAFA EL-Khamy Salman Avestimehr