A survey on LLM-as-a-judge
 |
 |
 |
abstract = {Accurate and consistent evaluation is crucial for decision-making across numerous fields, yet it remains challenging due to inherent subjectivity, variability, and scale. Large language models (LLMs) have achieved remarkable success, leading to "LLM-as-a-judge," where LLMs serve as evaluators for complex tasks. With their ability to process diverse data types and provide scalable assessments, LLMs present a compelling alternative to traditional expert-driven evaluations. However, ensuring the reliability of LLM-as-a-judge systems remains a significant challenge requiring careful design and standardization. This paper provides a comprehensive survey of LLM-as-a-judge, offering a formal definition and detailed classification while addressing the core question of how to build reliable LLM-as-a-judge systems. We explore strategies to enhance reliability, including improving consistency, mitigating biases, and adapting to diverse scenarios. We propose methodologies for evaluating reliability, supported by a novel benchmark. To advance development and deployment, we discuss practical applications, challenges, and future directions. Our contributions span multiple levels: we establish conceptual boundaries, reorganize fragmented literature into a unified framework, and propose a reliability-oriented benchmark. We articulate a forward-looking research agenda, offering theoretical foundations and practical guidance for constructing reliable and trustworthy LLM-as-a-judge systems.},
apice = {LlmasajudgeTheinnovation7},
articleno = 101253,
author = {Jiawei Gu and Xuhui Jiang and Zhichao Shi and Hexiang Tan and Xuehao Zhai and Chengjin Xu and Wei Li and Yinghan Shen and Shengjie Ma and Honghao Liu and Saizhuo Wang and Kun Zhang and Zhouchi Lin and Bowen Zhang and Lionel Ni and Wen Gao and Yuanzhuo Wang and Jian Guo},
doi = {10.1016/j.xinn.2025.101253},
issn = {2666-6758},
journal = {The Innovation},
keywords = {large language models, LLM-as-a-judge, automated evaluation, reliability assessment, trustworthy AI},
month = jun,
number = 6,
numpages = 30,
openalex = {W7120088840},
title = {A survey on LLM-as-a-judge},
url = {https://www.cell.com/the-innovation/fulltext/S2666-6758(25)00456-4},
urlopenaccess = {https://www.sciencedirect.com/science/article/pii/S2666675825004564/pdf},
volume = 7,
year = 2026
}
