@inproceedings{
zhang2023using,
title={Using Large Language Models for Hyperparameter Optimization},
author={Michael Zhang and Nishkrit Desai and Juhan Bae and Jonathan Lorraine and Jimmy Ba},
booktitle={NeurIPS 2023 Foundation Models for Decision Making Workshop},
year={2023},
url={https://openreview.net/forum?id=FUdZ6HEOre}
}
This paper studies using foundational large language models (LLMs) to make decisions during hyperparameter optimization (HPO). Empirical evaluations demonstrate that in settings with constrained search budgets, LLMs can perform comparably or better than traditional HPO methods like random search and Bayesian optimization on standard benchmarks. Furthermore, we propose to treat the code specifying our model as a hyperparameter, which the LLM outputs, going beyond the capabilities of existing HPO approaches. Our findings suggest that LLMs are a promising tool for improving efficiency in the traditional decision-making problem of hyperparameter optimization.
Using Large Language Models for Hyperparameter Optimization
Michael R. Zhang Nishkrit Desai Juhan Bae Jonathan Lorraine Jimmy Ba